diff --git a/Cargo.lock b/Cargo.lock index 9ba538d9d..e1c110fee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8462,6 +8462,7 @@ dependencies = [ "cpe", "criterion", "csaf", + "fixedbitset 0.5.7", "hex", "humantime", "itertools 0.13.0", diff --git a/Cargo.toml b/Cargo.toml index 923df6c67..254582a24 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,6 +61,7 @@ csaf = { version = "0.5.0", default-features = false } csaf-walker = { version = "0.10.0", default-features = false } cve = "0.3.1" env_logger = "0.11.0" +fixedbitset = "0.5.7" futures = "0.3.30" futures-util = "0.3" garage-door = "0.1.1" diff --git a/common/src/model.rs b/common/src/model.rs index a38474671..a40786c7e 100644 --- a/common/src/model.rs +++ b/common/src/model.rs @@ -98,9 +98,9 @@ impl PaginatedResults { }) } - pub fn map O>(mut self, f: F) -> PaginatedResults { + pub fn map O>(self, f: F) -> PaginatedResults { PaginatedResults { - items: self.items.drain(..).map(f).collect(), + items: self.items.into_iter().map(f).collect(), total: self.total, } } diff --git a/entity/src/relationship.rs b/entity/src/relationship.rs index 669221cb7..4cf9a2fa8 100644 --- a/entity/src/relationship.rs +++ b/entity/src/relationship.rs @@ -50,6 +50,34 @@ pub enum Relationship { #[sea_orm(num_value = 14)] PackageOf, #[sea_orm(num_value = 15)] + Contains, + #[sea_orm(num_value = 16)] + Dependency, + #[sea_orm(num_value = 17)] + DevDependency, + #[sea_orm(num_value = 18)] + OptionalDependency, + #[sea_orm(num_value = 19)] + ProvidedDependency, + #[sea_orm(num_value = 20)] + TestDependency, + #[sea_orm(num_value = 21)] + RuntimeDependency, + #[sea_orm(num_value = 22)] + Example, + #[sea_orm(num_value = 23)] + Generates, + #[sea_orm(num_value = 24)] + Variant, + #[sea_orm(num_value = 25)] + BuildTool, + #[sea_orm(num_value = 26)] + DevTool, + #[sea_orm(num_value = 27)] + Describes, + #[sea_orm(num_value = 28)] + Packages, + #[sea_orm(num_value = 29)] Undefined, } diff --git a/migration/src/lib.rs b/migration/src/lib.rs index 2f902e110..cbcf43901 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -101,6 +101,7 @@ mod m0000810_fix_get_purl; mod m0000820_create_conversation; mod m0000830_perf_indexes; mod m0000840_add_relationship_14_15; +mod m0000850_normalise_relationships; pub struct Migrator; @@ -209,6 +210,7 @@ impl MigratorTrait for Migrator { Box::new(m0000820_create_conversation::Migration), Box::new(m0000830_perf_indexes::Migration), Box::new(m0000840_add_relationship_14_15::Migration), + Box::new(m0000850_normalise_relationships::Migration), ] } } diff --git a/migration/src/m0000850_normalise_relationships.rs b/migration/src/m0000850_normalise_relationships.rs new file mode 100644 index 000000000..e132346c3 --- /dev/null +++ b/migration/src/m0000850_normalise_relationships.rs @@ -0,0 +1,57 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; +const DATA: [(i32, &str); 14] = [ + (16, "Contains"), + (17, "Dependency"), + (18, "DevDependency"), + (19, "OptionalDependency"), + (20, "ProvidedDependency"), + (21, "TestDependency"), + (22, "RuntimeDependency"), + (23, "Example"), + (24, "Generates"), + (25, "Variant"), + (26, "BuildTool"), + (27, "DevTool"), + (28, "Describes"), + (29, "Packages"), +]; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + for (id, description) in DATA { + let insert = Query::insert() + .into_table(Relationship::Table) + .columns([Relationship::Id, Relationship::Description]) + .values_panic([id.into(), description.into()]) + .to_owned(); + + manager.exec_stmt(insert).await?; + } + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + for (id, _) in DATA { + let insert = Query::delete() + .from_table(Relationship::Table) + .and_where(Expr::col(Relationship::Id).lt(id)) + .to_owned(); + + manager.exec_stmt(insert).await?; + } + + Ok(()) + } +} + +#[derive(DeriveIden)] +pub enum Relationship { + Table, + Id, + Description, +} diff --git a/modules/analysis/Cargo.toml b/modules/analysis/Cargo.toml index f8309a777..706b9bc02 100644 --- a/modules/analysis/Cargo.toml +++ b/modules/analysis/Cargo.toml @@ -14,6 +14,7 @@ actix-http = { workspace = true } actix-web = { workspace = true } anyhow = { workspace = true } cpe = { workspace = true } +fixedbitset = { workspace = true } log = { workspace = true } parking_lot = { workspace = true } petgraph = { workspace = true } diff --git a/modules/analysis/src/endpoints/mod.rs b/modules/analysis/src/endpoints/mod.rs index b5db2a773..b4592a16d 100644 --- a/modules/analysis/src/endpoints/mod.rs +++ b/modules/analysis/src/endpoints/mod.rs @@ -3,10 +3,10 @@ mod query; #[cfg(test)] mod test; -use super::service::AnalysisService; +use super::service::{AnalysisService, QueryOptions}; use crate::{ endpoints::query::OwnedComponentReference, - model::{AnalysisStatus, AncestorSummary, BaseSummary, DepSummary}, + model::{AnalysisStatus, BaseSummary, Node, RootTraces}, }; use actix_web::{get, web, HttpResponse, Responder}; use trustify_auth::{ @@ -20,9 +20,7 @@ use trustify_common::{ }; use utoipa_actix_web::service_config::ServiceConfig; -pub fn configure(config: &mut ServiceConfig, db: Database) { - let analysis = AnalysisService::new(); - +pub fn configure(config: &mut ServiceConfig, db: Database, analysis: AnalysisService) { config .app_data(web::Data::new(analysis)) .app_data(web::Data::new(db)) @@ -31,7 +29,8 @@ pub fn configure(config: &mut ServiceConfig, db: Database) { .service(get_component) .service(analysis_status) .service(search_component_deps) - .service(get_component_deps); + .service(get_component_deps) + .service(render_sbom_graph); } #[utoipa::path( @@ -61,7 +60,7 @@ pub async fn analysis_status( Paginated, ), responses( - (status = 200, description = "Search component(s) and return their root components.", body = PaginatedResults), + (status = 200, description = "Search component(s) and return their root components.", body = PaginatedResults), ), )] #[get("/v2/analysis/root-component")] @@ -74,8 +73,9 @@ pub async fn search_component_root_components( ) -> actix_web::Result { Ok(HttpResponse::Ok().json( service - .retrieve_root_components(&search, paginated, db.as_ref()) - .await?, + .retrieve(&search, QueryOptions::ancestors(), paginated, db.as_ref()) + .await? + .root_traces(), )) } @@ -86,7 +86,7 @@ pub async fn search_component_root_components( ("key" = String, Path, description = "provide component name, URL-encoded pURL, or CPE itself") ), responses( - (status = 200, description = "Retrieve component(s) root components by name, pURL, or CPE.", body = PaginatedResults), + (status = 200, description = "Retrieve component(s) root components by name, pURL, or CPE.", body = PaginatedResults), ), )] #[get("/v2/analysis/root-component/{key}")] @@ -101,8 +101,9 @@ pub async fn get_component_root_components( Ok(HttpResponse::Ok().json( service - .retrieve_root_components(&query, paginated, db.as_ref()) - .await?, + .retrieve(&query, QueryOptions::ancestors(), paginated, db.as_ref()) + .await? + .root_traces(), )) } @@ -128,7 +129,14 @@ pub async fn get_component( Ok(HttpResponse::Ok().json( service - .retrieve_components(&query, paginated, db.as_ref()) + .retrieve( + &query, + QueryOptions { + ..Default::default() + }, + paginated, + db.as_ref(), + ) .await?, )) } @@ -141,7 +149,7 @@ pub async fn get_component( Paginated, ), responses( - (status = 200, description = "Search component(s) and return their deps.", body = PaginatedResults), + (status = 200, description = "Search component(s) and return their deps.", body = PaginatedResults), ), )] #[get("/v2/analysis/dep")] @@ -154,7 +162,7 @@ pub async fn search_component_deps( ) -> actix_web::Result { Ok(HttpResponse::Ok().json( service - .retrieve_deps(&search, paginated, db.as_ref()) + .retrieve(&search, QueryOptions::descendants(), paginated, db.as_ref()) .await?, )) } @@ -166,7 +174,7 @@ pub async fn search_component_deps( ("key" = String, Path, description = "provide component name or URL-encoded pURL itself") ), responses( - (status = 200, description = "Retrieve component(s) dep components by name or pURL.", body = PaginatedResults), + (status = 200, description = "Retrieve component(s) dep components by name or pURL.", body = PaginatedResults), ), )] #[get("/v2/analysis/dep/{key}")] @@ -180,7 +188,34 @@ pub async fn get_component_deps( let query = OwnedComponentReference::try_from(key.as_str())?; Ok(HttpResponse::Ok().json( service - .retrieve_deps(&query, paginated, db.as_ref()) + .retrieve(&query, QueryOptions::descendants(), paginated, db.as_ref()) .await?, )) } + +#[utoipa::path( + tag = "analysis", + operation_id = "renderSbomGraph", + params( + ("sbom" = String, Path, description = "ID of the SBOM") + ), + responses( + (status = 200, description = "A graphwiz dot file of the SBOM graph", body = String), + (status = 404, description = "The SBOM was not found"), + ), +)] +#[get("/v2/analysis/sbom/{sbom}/render")] +pub async fn render_sbom_graph( + service: web::Data, + db: web::Data, + sbom: web::Path, + _: Require, +) -> actix_web::Result { + service.load_graph(db.as_ref(), &sbom).await; + + if let Some(data) = service.render_dot(&sbom) { + Ok(HttpResponse::Ok().body(data)) + } else { + Ok(HttpResponse::NotFound().finish()) + } +} diff --git a/modules/analysis/src/endpoints/test.rs b/modules/analysis/src/endpoints/test.rs index 6194ee312..f79d15988 100644 --- a/modules/analysis/src/endpoints/test.rs +++ b/modules/analysis/src/endpoints/test.rs @@ -181,12 +181,14 @@ async fn test_simple_dep_endpoint(ctx: &TrustifyContext) -> Result<(), anyhow::E let request: Request = TestRequest::get().uri(uri).to_request(); let response: Value = app.call_and_read_body_json(request).await; + log::debug!("Response: {:#?}", response); + assert_eq!( response["items"][0]["purl"], Value::from(["pkg:rpm/redhat/A@0.0.0?arch=src"]), ); - let purls = response["items"][0]["deps"] + let purls = response["items"][0]["descendent"] .as_array() .iter() .flat_map(|deps| *deps) @@ -606,7 +608,7 @@ async fn cdx_ancestor_of(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { // we're only looking for the parent node .filter(|m| m["node_id"] == parent) // flatten all dependencies of that parent node - .flat_map(|m| m["deps"].as_array().into_iter().flatten()) + .flat_map(|m| m["descendent"].as_array().into_iter().flatten()) // filter out all non-AncestorOf dependencies .filter(|m| m["relationship"] == "AncestorOf") .collect(); diff --git a/modules/analysis/src/model.rs b/modules/analysis/src/model.rs index fe613b1ed..0121f1950 100644 --- a/modules/analysis/src/model.rs +++ b/modules/analysis/src/model.rs @@ -1,10 +1,11 @@ +mod roots; + +pub use roots::*; + use petgraph::Graph; use serde::Serialize; -use std::{ - collections::HashMap, - fmt, - ops::{Deref, DerefMut}, -}; +use std::ops::{Deref, DerefMut}; +use std::{collections::HashMap, fmt}; use trustify_common::{cpe::Cpe, purl::Purl}; use trustify_entity::relationship::Relationship; use utoipa::ToSchema; @@ -36,30 +37,14 @@ pub struct PackageNode { pub product_name: String, pub product_version: String, } -impl fmt::Display for PackageNode { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self.purl) - } -} -#[derive(Debug, Clone, PartialEq, Eq, ToSchema, serde::Serialize)] -pub struct AncNode { - pub sbom_id: String, - pub node_id: String, - pub relationship: String, - pub purl: Vec, - pub cpe: Vec, - pub name: String, - pub version: String, -} - -impl fmt::Display for AncNode { +impl fmt::Display for PackageNode { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self.purl) } } -#[derive(Debug, Clone, Serialize, ToSchema)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, ToSchema)] pub struct BaseSummary { pub sbom_id: String, pub node_id: String, @@ -90,54 +75,27 @@ impl From<&PackageNode> for BaseSummary { } } -#[derive(Debug, Clone, Serialize, ToSchema)] -pub struct AncestorSummary { +#[derive(Debug, Clone, PartialEq, Eq, Serialize, ToSchema)] +pub struct Node { #[serde(flatten)] pub base: BaseSummary, - pub ancestors: Vec, -} -impl Deref for AncestorSummary { - type Target = BaseSummary; + /// The relationship the node has to it's containing node, if any. + #[serde(skip_serializing_if = "Option::is_none")] + pub relationship: Option, - fn deref(&self) -> &Self::Target { - &self.base - } -} - -impl DerefMut for AncestorSummary { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.base - } -} - -#[derive(Debug, Clone, PartialEq, Eq, ToSchema, serde::Serialize)] -pub struct DepNode { - pub sbom_id: String, - pub node_id: String, - pub relationship: String, - pub purl: Vec, - pub cpe: Vec, - pub name: String, - pub version: String, + /// All ancestors of this node. [`None`] if not requested on this level. + #[serde(skip_serializing_if = "Option::is_none")] #[schema(no_recursion)] - pub deps: Vec, -} + pub ancestor: Option>, -impl fmt::Display for DepNode { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self.purl) - } -} - -#[derive(Debug, Clone, Serialize, ToSchema)] -pub struct DepSummary { - #[serde(flatten)] - pub base: BaseSummary, - pub deps: Vec, + /// All descendents of this node. [`None`] if not requested on this level. + #[serde(skip_serializing_if = "Option::is_none")] + #[schema(no_recursion)] + pub descendent: Option>, } -impl Deref for DepSummary { +impl Deref for Node { type Target = BaseSummary; fn deref(&self) -> &Self::Target { @@ -145,7 +103,7 @@ impl Deref for DepSummary { } } -impl DerefMut for DepSummary { +impl DerefMut for Node { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.base } diff --git a/modules/analysis/src/model/roots.rs b/modules/analysis/src/model/roots.rs new file mode 100644 index 000000000..9a9e3400b --- /dev/null +++ b/modules/analysis/src/model/roots.rs @@ -0,0 +1,205 @@ +use crate::model::{BaseSummary, Node}; +use std::collections::HashMap; +use trustify_common::model::PaginatedResults; +use trustify_entity::relationship::Relationship; + +pub trait Roots { + /// Collect all top level ancestors. + fn roots(self) -> Self; +} + +impl Roots for PaginatedResults { + fn roots(self) -> PaginatedResults { + let items = self.items.roots(); + let total = items.len(); + Self { + items, + total: total as _, + } + } +} + +impl Roots for Vec { + fn roots(self) -> Vec { + fn roots_into( + nodes: impl IntoIterator, + result: &mut HashMap<(String, String), Node>, + ) { + for node in nodes.into_iter() { + roots_into(node.ancestor.clone().into_iter().flatten(), result); + + if let Some(true) = node.ancestor.as_ref().map(|a| a.is_empty()) { + result.insert((node.base.sbom_id.clone(), node.base.node_id.clone()), node); + } + } + } + + let mut result = HashMap::new(); + roots_into(self, &mut result); + + result.into_values().collect() + } +} + +pub trait RootTraces { + type Result; + + /// Collect all traces to the root nodes + fn root_traces(self) -> Self::Result; +} + +impl<'a> RootTraces for &'a PaginatedResults { + type Result = PaginatedResults>; + + fn root_traces(self) -> Self::Result { + let items = self.items.root_traces(); + let total = items.len(); + Self::Result { + items, + total: total as _, + } + } +} + +impl<'a> RootTraces for &'a Vec { + type Result = Vec>; + + fn root_traces(self) -> Self::Result { + fn roots_into<'a>( + nodes: impl IntoIterator, + parents: &Vec<(&'a BaseSummary, Relationship)>, + result: &mut Vec>, + ) { + for node in nodes.into_iter() { + let mut next = parents.clone(); + + // if we don't have a relationship to the parent node, we are the initial node + // and will be skipped + if let Some(relationship) = node.relationship { + next.push((&node.base, relationship)); + }; + + if let Some(true) = node.ancestor.as_ref().map(|a| a.is_empty()) { + result.push(next); + } else { + roots_into(node.ancestor.iter().flatten(), &next, result); + } + } + } + + let mut result = Vec::new(); + roots_into(self, &Vec::new(), &mut result); + + result + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::model::BaseSummary; + use trustify_entity::relationship::Relationship; + + fn base(node_id: &str) -> BaseSummary { + BaseSummary { + sbom_id: "".to_string(), + node_id: node_id.to_string(), + purl: vec![], + cpe: vec![], + name: "".to_string(), + version: "".to_string(), + published: "".to_string(), + document_id: "".to_string(), + product_name: "".to_string(), + product_version: "".to_string(), + } + } + + fn node(node_id: &str) -> Node { + Node { + base: base(node_id), + relationship: None, + ancestor: None, + descendent: None, + } + } + + #[test] + fn simple_roots() { + let result = vec![Node { + base: base("AA"), + relationship: None, + ancestor: Some(vec![Node { + ancestor: Some(vec![]), + relationship: Some(Relationship::DependencyOf), + ..node("A") + }]), + descendent: None, + }] + .roots(); + + assert_eq!( + result, + vec![Node { + base: base("A"), + relationship: Some(Relationship::DependencyOf), + ancestor: Some(vec![]), + descendent: None, + }] + ); + } + + #[test] + fn nested_roots() { + let result = vec![Node { + ancestor: Some(vec![Node { + base: base("AA"), + relationship: Some(Relationship::DependencyOf), + ancestor: Some(vec![Node { + ancestor: Some(vec![]), + relationship: Some(Relationship::DependencyOf), + ..node("A") + }]), + descendent: None, + }]), + ..node("AAA") + }] + .roots(); + + assert_eq!( + result, + vec![Node { + base: base("A"), + relationship: Some(Relationship::DependencyOf), + ancestor: Some(vec![]), + descendent: None, + }] + ); + } + + #[test] + fn nested_root_traces() { + let result = vec![Node { + ancestor: Some(vec![Node { + base: base("AA"), + relationship: Some(Relationship::DependencyOf), + ancestor: Some(vec![Node { + ancestor: Some(vec![]), + relationship: Some(Relationship::DependencyOf), + ..node("A") + }]), + descendent: None, + }]), + ..node("AAA") + }]; + let result = result.root_traces(); + + assert_eq!( + result, + vec![vec![ + (&base("AA"), Relationship::DependencyOf), + (&base("A"), Relationship::DependencyOf), + ]] + ); + } +} diff --git a/modules/analysis/src/service/mod.rs b/modules/analysis/src/service/mod.rs index eff7d26ae..abd664ebe 100644 --- a/modules/analysis/src/service/mod.rs +++ b/modules/analysis/src/service/mod.rs @@ -1,23 +1,25 @@ mod load; mod query; +mod render; +mod walk; pub use query::*; +pub use walk::*; #[cfg(test)] mod test; use crate::{ - model::{ - AnalysisStatus, AncNode, AncestorSummary, BaseSummary, DepNode, DepSummary, GraphMap, - PackageNode, - }, + model::{AnalysisStatus, BaseSummary, GraphMap, Node, PackageNode}, Error, }; +use fixedbitset::FixedBitSet; use parking_lot::RwLock; use petgraph::{ algo::is_cyclic_directed, graph::{Graph, NodeIndex}, - visit::{NodeIndexable, VisitMap, Visitable}, + prelude::EdgeRef, + visit::{VisitMap, Visitable}, Direction, }; use sea_orm::{prelude::ConnectionTrait, EntityOrSelect, EntityTrait, QueryOrder}; @@ -35,115 +37,88 @@ use trustify_common::{ use trustify_entity::{relationship::Relationship, sbom}; use uuid::Uuid; -#[derive(Clone, Default)] +#[derive(Clone)] pub struct AnalysisService { graph: Arc>, } -pub fn dep_nodes( +/// Collect related nodes in the provided direction. +/// +/// If the depth is zero, or the node was already processed, it will return [`None`], indicating +/// that the request was not processed. +fn collect( graph: &Graph, node: NodeIndex, - visited: &mut HashSet, -) -> Vec { - let mut depnodes = Vec::new(); - fn dfs( - graph: &Graph, - node: NodeIndex, - depnodes: &mut Vec, - visited: &mut HashSet, - ) { - if visited.contains(&node) { - return; - } - visited.insert(node); - for neighbor in graph.neighbors_directed(node, Direction::Incoming) { - if let Some(dep_packagenode) = graph.node_weight(neighbor).cloned() { - // Attempt to find the edge and get the relationship in a more elegant way - if let Some(relationship) = graph - .find_edge(neighbor, node) - .and_then(|edge_index| graph.edge_weight(edge_index)) - { - let dep_node = DepNode { - sbom_id: dep_packagenode.sbom_id, - node_id: dep_packagenode.node_id, - relationship: relationship.to_string(), - purl: dep_packagenode.purl.clone(), - cpe: dep_packagenode.cpe.clone(), - name: dep_packagenode.name.to_string(), - version: dep_packagenode.version.to_string(), - deps: dep_nodes(graph, neighbor, visited), - }; - depnodes.push(dep_node); - dfs(graph, neighbor, depnodes, visited); - } - } else { - log::warn!( - "Processing descendants node weight for neighbor {:?} not found", - neighbor - ); - } - } + direction: Direction, + depth: u64, + discovered: &mut FixedBitSet, +) -> Option> { + tracing::debug!(direction = ?direction, "collecting for {node:?}"); + + if depth == 0 { + log::debug!("depth is zero"); + // we ran out of depth + return None; } - dfs(graph, node, &mut depnodes, visited); - - depnodes -} + if !discovered.visit(node) { + log::debug!("node got visited already"); + // we've already seen this + return None; + } -pub fn ancestor_nodes( - graph: &Graph, - node: NodeIndex, -) -> Vec { - let mut discovered = graph.visit_map(); - let mut ancestor_nodes = Vec::new(); - let mut stack = Vec::new(); - - stack.push(graph.from_index(node.index())); - - while let Some(node) = stack.pop() { - if discovered.visit(node) { - for succ in graph.neighbors_directed(node, Direction::Outgoing) { - if !discovered.is_visited(&succ) { - if let Some(anc_packagenode) = graph.node_weight(succ).cloned() { - if let Some(edge) = graph.find_edge(node, succ) { - if let Some(relationship) = graph.edge_weight(edge) { - let anc_node = AncNode { - sbom_id: anc_packagenode.sbom_id, - node_id: anc_packagenode.node_id, - relationship: relationship.to_string(), - purl: anc_packagenode.purl, - cpe: anc_packagenode.cpe, - name: anc_packagenode.name, - version: anc_packagenode.version, - }; - ancestor_nodes.push(anc_node); - stack.push(succ); - } else { - log::warn!( - "Edge weight not found for edge between {:?} and {:?}", - node, - succ - ); - } - } else { - log::warn!("Edge not found between {:?} and {:?}", node, succ); - } - } else { - log::warn!("Processing ancestors, node value for {:?} not found", succ); - } - } - } - if graph.neighbors_directed(node, Direction::Outgoing).count() == 0 { - continue; // we are at the root - } - } + let mut result = Vec::new(); + + for edge in graph.edges_directed(node, direction) { + log::debug!("edge {edge:?}"); + + // we only recurse in one direction + let (ancestor, descendent, package_node) = match direction { + Direction::Incoming => ( + collect(graph, edge.source(), direction, depth - 1, discovered), + None, + graph.node_weight(edge.source()), + ), + Direction::Outgoing => ( + None, + collect(graph, edge.target(), direction, depth - 1, discovered), + graph.node_weight(edge.target()), + ), + }; + + let relationship = edge.weight(); + let Some(package_node) = package_node else { + continue; + }; + + result.push(Node { + base: BaseSummary::from(package_node), + relationship: Some(*relationship), + ancestor, + descendent, + }); } - ancestor_nodes + + Some(result) } impl AnalysisService { + /// Create a new analysis service instance. + /// + /// ## Caching + /// + /// A new instance will have a new cache. Instanced cloned from it, will share that cache. + /// + /// Therefore, it is ok to create a new instance. However, if you want to make use of the + /// caching, it is necessary to re-use that instance. + /// + /// Also, we do not implement default because of this. As a new instance has the implication + /// of having its own cache. So creating a new instance should be a deliberate choice. + #[allow(clippy::new_without_default)] pub fn new() -> Self { - Self::default() + Self { + graph: Default::default(), + } } #[instrument(skip_all, err)] @@ -191,7 +166,7 @@ impl AnalysisService { /// Collect nodes from the graph /// - /// Similar to [`Self::query_graph`], but manages the state of collecting. + /// Similar to [`Self::query_graphs`], but manages the state of collecting. #[instrument(skip(self, init, collector))] fn collect_graph<'a, T, I, C>( &self, @@ -202,179 +177,150 @@ impl AnalysisService { ) -> T where I: FnOnce() -> T, - C: Fn(&mut T, &Graph, NodeIndex, &PackageNode), + C: Fn(&mut T, &Graph, NodeIndex, &PackageNode, &mut FixedBitSet), { let mut value = init(); - self.query_graph(query, distinct_sbom_ids, |graph, index, node| { - collector(&mut value, graph, index, node); - }); + self.query_graphs( + query, + distinct_sbom_ids, + |graph, index, node, discovered| { + collector(&mut value, graph, index, node, discovered); + }, + ); value } /// Traverse the graph, call the function for every matching node. #[instrument(skip(self, f))] - fn query_graph<'a, F>( + fn query_graphs<'a, F>( &self, query: impl Into> + Debug, distinct_sbom_ids: Vec, mut f: F, ) where - F: FnMut(&Graph, NodeIndex, &PackageNode), + F: FnMut(&Graph, NodeIndex, &PackageNode, &mut FixedBitSet), { let query = query.into(); // RwLock for reading hashmap let graph_read_guard = self.graph.read(); for distinct_sbom_id in &distinct_sbom_ids { - if let Some(graph) = graph_read_guard.get(distinct_sbom_id.to_string().as_str()) { - if is_cyclic_directed(graph) { - log::warn!( - "analysis graph of sbom {} has circular references!", - distinct_sbom_id - ); - } - - let mut visited = HashSet::new(); - - // Iterate over matching node indices and process them directly - graph - .node_indices() - .filter(|&i| Self::filter(graph, &query, i)) - .for_each(|node_index| { - if !visited.contains(&node_index) { - visited.insert(node_index); - - if let Some(find_match_package_node) = graph.node_weight(node_index) { - log::debug!("matched!"); - f(graph, node_index, find_match_package_node); - } - } - }); - } + self.query_graph(&graph_read_guard, query, distinct_sbom_id, &mut f); } } - #[instrument(skip(self))] - pub fn query_ancestor_graph<'a>( - &self, - query: impl Into> + Debug, - distinct_sbom_ids: Vec, - ) -> Vec { - self.collect_graph( - query, - distinct_sbom_ids, - Vec::new, - |components, graph, node_index, node| { - components.push(AncestorSummary { - base: node.into(), - ancestors: ancestor_nodes(graph, node_index), - }); - }, - ) + #[instrument(skip(self, graph, f))] + fn query_graph(&self, graph: &GraphMap, query: GraphQuery<'_>, sbom_id: &str, f: &mut F) + where + F: FnMut(&Graph, NodeIndex, &PackageNode, &mut FixedBitSet), + { + let Some(graph) = graph.get(sbom_id) else { + // FIXME: we need a better strategy handling such errors + log::warn!("Unable to find SBOM: {sbom_id}"); + return; + }; + + if is_cyclic_directed(graph) { + // FIXME: we need a better strategy handling such errors + log::warn!( + "analysis graph of sbom {} has circular references!", + sbom_id + ); + return; + } + + let mut visited = HashSet::new(); + let mut discovered = graph.visit_map(); + + // Iterate over matching node indices and process them directly + graph + .node_indices() + .filter(|&i| Self::filter(graph, &query, i)) + .for_each(|node_index| { + if visited.insert(node_index) { + if let Some(find_match_package_node) = graph.node_weight(node_index) { + f(graph, node_index, find_match_package_node, &mut discovered); + } + } + }); } #[instrument(skip(self))] - pub async fn query_deps_graph( + pub fn run_graph_query<'a>( &self, - query: impl Into> + Debug, + query: impl Into> + Debug, + options: QueryOptions, distinct_sbom_ids: Vec, - ) -> Vec { + ) -> Vec { self.collect_graph( query, distinct_sbom_ids, Vec::new, - |components, graph, node_index, node| { - components.push(DepSummary { + |components, graph, node_index, node, _| { + log::debug!( + "Discovered node - sbom: {}, node: {}", + node.sbom_id, + node.node_id + ); + components.push(Node { base: node.into(), - deps: dep_nodes(graph, node_index, &mut HashSet::new()), + relationship: None, + ancestor: collect( + graph, + node_index, + Direction::Incoming, + options.ancestors, + &mut graph.visit_map(), + ), + descendent: collect( + graph, + node_index, + Direction::Outgoing, + options.descendants, + &mut graph.visit_map(), + ), }); }, ) } - pub async fn retrieve_all_sbom_roots_by_name( - &self, - sbom_id: Uuid, - component_name: String, - connection: &C, - ) -> Result, Error> { - // This function searches for a component(s) by name in a specific sbom, then returns that components - // root components. - - let distinct_sbom_ids = vec![sbom_id.to_string()]; - self.load_graphs(connection, &distinct_sbom_ids).await?; - - let components = self.query_ancestor_graph( - GraphQuery::Component(ComponentReference::Name(&component_name)), - distinct_sbom_ids, - ); - - let mut root_components = Vec::new(); - for component in components { - if let Some(last_ancestor) = component.ancestors.last() { - if !root_components.contains(last_ancestor) { - // we want a distinct list - root_components.push(last_ancestor.clone()); - } - } - } - - Ok(root_components) - } - - /// locate components, retrieve ancestor information + /// locate components, retrieve dependency information, from a single SBOM #[instrument(skip(self, connection), err)] - pub async fn retrieve_root_components( + pub async fn retrieve_single( &self, + sbom_id: Uuid, query: impl Into> + Debug, + options: impl Into + Debug, paginated: Paginated, connection: &C, - ) -> Result, Error> { - let query = query.into(); - - let distinct_sbom_ids = self.load_graphs_query(connection, query).await?; - let components = self.query_ancestor_graph(query, distinct_sbom_ids); - - Ok(paginated.paginate_array(&components)) - } + ) -> Result, Error> { + let distinct_sbom_ids = vec![sbom_id.to_string()]; - /// locate components, retrieve dependency information - #[instrument(skip(self, connection), err)] - pub async fn retrieve_deps( - &self, - query: impl Into> + Debug, - paginated: Paginated, - connection: &C, - ) -> Result, Error> { let query = query.into(); + let options = options.into(); - let distinct_sbom_ids = self.load_graphs_query(connection, query).await?; - let components = self.query_deps_graph(query, distinct_sbom_ids).await; + self.load_graphs(connection, &distinct_sbom_ids).await?; + let components = self.run_graph_query(query, options, distinct_sbom_ids); Ok(paginated.paginate_array(&components)) } - /// locate components, retrieve basic information only + /// locate components, retrieve dependency information #[instrument(skip(self, connection), err)] - pub async fn retrieve_components( + pub async fn retrieve( &self, query: impl Into> + Debug, + options: impl Into + Debug, paginated: Paginated, connection: &C, - ) -> Result, Error> { + ) -> Result, Error> { let query = query.into(); + let options = options.into(); let distinct_sbom_ids = self.load_graphs_query(connection, query).await?; - let components = self.collect_graph( - query, - distinct_sbom_ids, - Vec::new, - |components, _, _, node| { - components.push(BaseSummary::from(node)); - }, - ); + let components = self.run_graph_query(query, options, distinct_sbom_ids); Ok(paginated.paginate_array(&components)) } diff --git a/modules/analysis/src/service/query.rs b/modules/analysis/src/service/query.rs index fbe3f82dd..8ab534c49 100644 --- a/modules/analysis/src/service/query.rs +++ b/modules/analysis/src/service/query.rs @@ -1,4 +1,6 @@ +use std::collections::HashSet; use trustify_common::{cpe::Cpe, db::query::Query, purl::Purl}; +use trustify_entity::relationship::Relationship; #[derive(Copy, Clone, Debug)] pub enum ComponentReference<'a> { @@ -55,3 +57,44 @@ impl<'a> From<&'a Query> for GraphQuery<'a> { Self::Query(query) } } + +/// Options when querying the graph. +#[derive(Clone, Debug, Default, Eq, PartialEq, serde::Deserialize)] +pub struct QueryOptions { + #[serde(default)] + pub ancestors: u64, + #[serde(default)] + pub descendants: u64, + #[serde(default)] + pub relationships: HashSet, +} + +impl QueryOptions { + pub fn any() -> Self { + Self { + ancestors: u64::MAX, + descendants: u64::MAX, + ..Default::default() + } + } + + pub fn ancestors() -> Self { + Self { + ancestors: u64::MAX, + ..Default::default() + } + } + + pub fn descendants() -> Self { + Self { + descendants: u64::MAX, + ..Default::default() + } + } +} + +impl From<()> for QueryOptions { + fn from(_: ()) -> Self { + Self::default() + } +} diff --git a/modules/analysis/src/service/render.rs b/modules/analysis/src/service/render.rs new file mode 100644 index 000000000..384d25e9e --- /dev/null +++ b/modules/analysis/src/service/render.rs @@ -0,0 +1,90 @@ +use super::*; + +impl AnalysisService { + pub fn render_dot(&self, sbom: &str) -> Option { + use std::fmt::Write; + + struct Renderer<'a> { + data: &'a mut String, + } + + impl Visitor for Renderer<'_> { + fn node(&mut self, node: &PackageNode) { + let _ = writeln!( + self.data, + r#""{id}" [label="{label}"]"#, + id = escape(&node.node_id), + label = escape(&format!( + "{name} / {version}: {id}", + name = node.name, + version = node.version, + id = node.node_id + )) + ); + } + + fn edge( + &mut self, + source: &PackageNode, + relationship: Relationship, + target: &PackageNode, + ) { + let _ = writeln!( + self.data, + r#""{source}" -> "{target}" [label="{label}"]"#, + source = escape(&source.node_id), + target = escape(&target.node_id), + label = escape(&relationship.to_string()) + ); + } + } + + let mut data = String::new(); + + data.push_str( + r#" +digraph { +"#, + ); + + if self.walk(sbom, Renderer { data: &mut data }) { + data.push_str( + r#" +} +"#, + ); + + Some(data) + } else { + None + } + } +} + +fn escape(id: &str) -> String { + let mut escaped = String::with_capacity(id.len()); + + for ch in id.chars() { + match ch { + '"' => { + escaped.push('\\'); + escaped.push(ch); + } + '\n' => { + escaped.push_str("\\n"); + } + _ => escaped.push(ch), + } + } + + escaped +} + +#[cfg(test)] +mod test { + + #[test] + fn escape() { + assert_eq!(super::escape("foo\"bar\nbaz"), r#"foo\"bar\nbaz"#); + } +} diff --git a/modules/analysis/src/service/test.rs b/modules/analysis/src/service/test.rs index 8e8fa8f2f..daf998214 100644 --- a/modules/analysis/src/service/test.rs +++ b/modules/analysis/src/service/test.rs @@ -1,5 +1,8 @@ use super::*; -use crate::test::*; +use crate::{ + model::*, + test::{Node, *}, +}; use std::{str::FromStr, time::SystemTime}; use test_context::test_context; use test_log::test; @@ -17,38 +20,52 @@ async fn test_simple_analysis_service(ctx: &TrustifyContext) -> Result<(), anyho let service = AnalysisService::new(); let analysis_graph = service - .retrieve_root_components(&Query::q("DD"), Paginated::default(), &ctx.db) + .retrieve( + &Query::q("DD"), + QueryOptions::ancestors(), + Paginated::default(), + &ctx.db, + ) .await?; - assert_eq!( - analysis_graph - .items - .last() - .unwrap() - .ancestors - .last() - .unwrap() - .purl, - vec![Purl::from_str("pkg:rpm/redhat/AA@0.0.0?arch=src")?] - ); - assert_eq!( - analysis_graph - .items - .last() - .unwrap() - .ancestors - .last() - .unwrap() - .node_id, - "SPDXRef-AA".to_string() - ); + log::debug!("Before: {analysis_graph:#?}"); + let analysis_graph = analysis_graph.root_traces(); + log::debug!("After: {analysis_graph:#?}"); + + assert_ancestors(&analysis_graph.items, |ancestors| { + assert!( + matches!( + ancestors[..], + [[ + .., + Node { + id: "SPDXRef-AA", + purls: ["pkg:rpm/redhat/AA@0.0.0?arch=src"], + .. + } + ]] + ), + "doesn't match: {ancestors:#?}" + ); + }); assert_eq!(analysis_graph.total, 1); - // ensure we set implicit relationship on component with no defined relationships + // ensure we set implicit relationship on components with no defined relationships let analysis_graph = service - .retrieve_root_components(&Query::q("EE"), Paginated::default(), &ctx.db) + .retrieve( + &Query::q("EE"), + QueryOptions::ancestors(), + Paginated::default(), + &ctx.db, + ) .await?; + + log::debug!("Before: {analysis_graph:#?}"); + let analysis_graph = analysis_graph.roots(); + log::debug!("After: {analysis_graph:#?}"); + assert_eq!(analysis_graph.total, 1); + Ok(()) } @@ -63,36 +80,49 @@ async fn test_simple_analysis_cyclonedx_service( let service = AnalysisService::new(); let analysis_graph = service - .retrieve_root_components(&Query::q("DD"), Paginated::default(), &ctx.db) + .retrieve( + &Query::q("DD"), + QueryOptions::ancestors(), + Paginated::default(), + &ctx.db, + ) .await?; - assert_eq!( - analysis_graph - .items - .last() - .unwrap() - .ancestors - .last() - .unwrap() - .purl, - vec![Purl::from_str("pkg:rpm/redhat/AA@0.0.0?arch=src")?] - ); - let node = analysis_graph - .items - .last() - .unwrap() - .ancestors - .last() - .unwrap(); - assert_eq!(node.node_id, "aa".to_string()); - assert_eq!(node.name, "AA".to_string()); + let analysis_graph = analysis_graph.root_traces(); + + assert_ancestors(&analysis_graph.items, |ancestors| { + assert!( + matches!( + ancestors[..], + [[ + .., + Node { + id: "aa", + name: "AA", + purls: ["pkg:rpm/redhat/AA@0.0.0?arch=src"], + .. + } + ]] + ), + "doesn't match: {ancestors:#?}" + ); + }); assert_eq!(analysis_graph.total, 1); - // ensure we set implicit relationship on component with no defined relationships + // ensure we set implicit relationship on components with no defined relationships let analysis_graph = service - .retrieve_root_components(&Query::q("EE"), Paginated::default(), &ctx.db) + .retrieve( + &Query::q("EE"), + QueryOptions::ancestors(), + Paginated::default(), + &ctx.db, + ) .await?; + + let analysis_graph = analysis_graph.root_traces(); + assert_eq!(analysis_graph.total, 1); + Ok(()) } @@ -104,9 +134,16 @@ async fn test_simple_by_name_analysis_service(ctx: &TrustifyContext) -> Result<( let service = AnalysisService::new(); let analysis_graph = service - .retrieve_root_components(ComponentReference::Name("B"), Paginated::default(), &ctx.db) + .retrieve( + ComponentReference::Name("B"), + QueryOptions::ancestors(), + Paginated::default(), + &ctx.db, + ) .await?; + let analysis_graph = analysis_graph.root_traces(); + assert_ancestors(&analysis_graph.items, |ancestors| { assert_eq!( ancestors, @@ -144,9 +181,18 @@ async fn test_simple_by_purl_analysis_service(ctx: &TrustifyContext) -> Result<( let component_purl: Purl = Purl::from_str("pkg:rpm/redhat/B@0.0.0").map_err(Error::Purl)?; let analysis_graph = service - .retrieve_root_components(&component_purl, Paginated::default(), &ctx.db) + .retrieve( + &component_purl, + QueryOptions::ancestors(), + Paginated::default(), + &ctx.db, + ) .await?; + log::debug!("Before: {analysis_graph:#?}"); + let analysis_graph = analysis_graph.root_traces(); + log::debug!("After: {analysis_graph:#?}"); + assert_ancestors(&analysis_graph.items, |ancestors| { assert_eq!( ancestors, @@ -185,20 +231,23 @@ async fn test_quarkus_analysis_service(ctx: &TrustifyContext) -> Result<(), anyh let service = AnalysisService::new(); let analysis_graph = service - .retrieve_root_components(&Query::q("spymemcached"), Paginated::default(), &ctx.db) + .retrieve( + &Query::q("spymemcached"), + QueryOptions::ancestors(), + Paginated::default(), + &ctx.db, + ) .await?; + log::debug!("Before: {analysis_graph:#?}"); + let analysis_graph = analysis_graph.root_traces(); + log::debug!("After: {analysis_graph:#?}"); + assert_ancestors(&analysis_graph.items, |ancestors| { assert!( matches!(ancestors, [ [..], [ - Node { - id: "SPDXRef-DOCUMENT", - name: "quarkus-bom-3.2.12.Final-redhat-00002", - version: "", - .. - }, Node { id: "SPDXRef-e24fec28-1001-499c-827f-2e2e5f2671b5", name: "quarkus-bom", @@ -210,9 +259,15 @@ async fn test_quarkus_analysis_service(ctx: &TrustifyContext) -> Result<(), anyh "pkg:maven/com.redhat.quarkus.platform/quarkus-bom@3.2.12.Final-redhat-00002?repository_url=https://maven.repository.redhat.com/ga/&type=pom" ], }, + Node { + id: "SPDXRef-DOCUMENT", + name: "quarkus-bom-3.2.12.Final-redhat-00002", + version: "", + .. + }, ] ]), - "must match: {ancestors:#?}" + "doesn't match: {ancestors:#?}" ); }); @@ -257,15 +312,30 @@ async fn test_simple_deps_service(ctx: &TrustifyContext) -> Result<(), anyhow::E let service = AnalysisService::new(); let analysis_graph = service - .retrieve_deps(&Query::q("AA"), Paginated::default(), &ctx.db) + .retrieve( + &Query::q("AA"), + QueryOptions::descendants(), + Paginated::default(), + &ctx.db, + ) .await?; assert_eq!(analysis_graph.total, 1); - // ensure we set implicit relationship on component with no defined relationships + // ensure we set implicit relationship on components with no defined relationships let analysis_graph = service - .retrieve_root_components(&Query::q("EE"), Paginated::default(), &ctx.db) + .retrieve( + &Query::q("EE"), + QueryOptions::ancestors(), + Paginated::default(), + &ctx.db, + ) .await?; + + log::debug!("Before: {analysis_graph:#?}"); + let analysis_graph = analysis_graph.roots(); + log::debug!("After: {analysis_graph:#?}"); + assert_eq!(analysis_graph.total, 1); Ok(()) @@ -279,15 +349,26 @@ async fn test_simple_deps_cyclonedx_service(ctx: &TrustifyContext) -> Result<(), let service = AnalysisService::new(); let analysis_graph = service - .retrieve_deps(&Query::q("AA"), Paginated::default(), &ctx.db) + .retrieve( + &Query::q("AA"), + QueryOptions::descendants(), + Paginated::default(), + &ctx.db, + ) .await?; assert_eq!(analysis_graph.total, 1); // ensure we set implicit relationship on component with no defined relationships let analysis_graph = service - .retrieve_root_components(&Query::q("EE"), Paginated::default(), &ctx.db) - .await?; + .retrieve( + &Query::q("EE"), + QueryOptions::ancestors(), + Paginated::default(), + &ctx.db, + ) + .await? + .roots(); assert_eq!(analysis_graph.total, 1); Ok(()) @@ -301,7 +382,12 @@ async fn test_simple_by_name_deps_service(ctx: &TrustifyContext) -> Result<(), a let service = AnalysisService::new(); let analysis_graph = service - .retrieve_deps(ComponentReference::Name("A"), Paginated::default(), &ctx.db) + .retrieve( + ComponentReference::Name("A"), + QueryOptions::descendants(), + Paginated::default(), + &ctx.db, + ) .await?; assert_eq!(analysis_graph.items.len(), 1); @@ -330,7 +416,12 @@ async fn test_simple_by_purl_deps_service(ctx: &TrustifyContext) -> Result<(), a Purl::from_str("pkg:rpm/redhat/AA@0.0.0?arch=src").map_err(Error::Purl)?; let analysis_graph = service - .retrieve_deps(&component_purl, Paginated::default(), &ctx.db) + .retrieve( + &component_purl, + QueryOptions::descendants(), + Paginated::default(), + &ctx.db, + ) .await?; assert_eq!( @@ -355,7 +446,12 @@ async fn test_quarkus_deps_service(ctx: &TrustifyContext) -> Result<(), anyhow:: let service = AnalysisService::new(); let analysis_graph = service - .retrieve_deps(&Query::q("spymemcached"), Paginated::default(), &ctx.db) + .retrieve( + &Query::q("spymemcached"), + QueryOptions::descendants(), + Paginated::default(), + &ctx.db, + ) .await?; assert_eq!(analysis_graph.total, 2); @@ -372,14 +468,16 @@ async fn test_circular_deps_cyclonedx_service(ctx: &TrustifyContext) -> Result<( let service = AnalysisService::new(); let analysis_graph = service - .retrieve_deps( + .retrieve( ComponentReference::Name("junit-bom"), + QueryOptions::descendants(), Paginated::default(), &ctx.db, ) .await?; - assert_eq!(analysis_graph.total, 1); + // we should get zero, as we don't deal with circular dependencies and don't load such graphs + assert_eq!(analysis_graph.total, 0); Ok(()) } @@ -392,10 +490,16 @@ async fn test_circular_deps_spdx_service(ctx: &TrustifyContext) -> Result<(), an let service = AnalysisService::new(); let analysis_graph = service - .retrieve_deps(ComponentReference::Name("A"), Paginated::default(), &ctx.db) + .retrieve( + ComponentReference::Name("A"), + QueryOptions::descendants(), + Paginated::default(), + &ctx.db, + ) .await?; - assert_eq!(analysis_graph.total, 1); + // we should get zero, as we don't deal with circular dependencies and don't load such graphs + assert_eq!(analysis_graph.total, 0); Ok(()) } @@ -410,9 +514,16 @@ async fn test_retrieve_all_sbom_roots_by_name(ctx: &TrustifyContext) -> Result<( let component_name = "quarkus-vertx-http".to_string(); let analysis_graph = service - .retrieve_root_components(&Query::q(&component_name), Paginated::default(), &ctx.db) + .retrieve( + &Query::q(&component_name), + QueryOptions::ancestors(), + Paginated::default(), + &ctx.db, + ) .await?; + let analysis_graph = analysis_graph.roots(); + log::debug!("Result: {analysis_graph:#?}"); let sbom_id = analysis_graph @@ -423,13 +534,34 @@ async fn test_retrieve_all_sbom_roots_by_name(ctx: &TrustifyContext) -> Result<( .parse::()?; let roots = service - .retrieve_all_sbom_roots_by_name(sbom_id, component_name, &ctx.db) + .retrieve_single( + sbom_id, + ComponentReference::Name(&component_name), + QueryOptions::ancestors(), + Default::default(), + &ctx.db, + ) .await?; - assert_eq!( - roots.last().unwrap().name, - "quarkus-bom-3.2.11.Final-redhat-00001" - ); + log::debug!("Before: {roots:#?}"); + let roots = roots.root_traces(); + log::debug!("After: {roots:#?}"); + + assert_ancestors(&roots.items, |ancestors| { + assert!( + matches!( + ancestors, + [[ + .., + Node { + name: "quarkus-bom-3.2.11.Final-redhat-00001", + .. + } + ]] + ), + "doesn't match: {ancestors:#?}" + ); + }); Ok(()) } diff --git a/modules/analysis/src/service/walk.rs b/modules/analysis/src/service/walk.rs new file mode 100644 index 000000000..1f5abfd09 --- /dev/null +++ b/modules/analysis/src/service/walk.rs @@ -0,0 +1,35 @@ +use super::*; + +pub trait Visitor { + fn node(&mut self, node: &PackageNode); + fn edge(&mut self, source: &PackageNode, relationship: Relationship, target: &PackageNode); +} + +impl AnalysisService { + pub fn walk(&self, sbom: &str, mut v: V) -> bool + where + V: Visitor, + { + let graph = self.graph.read(); + let graph = graph.get(sbom); + + let Some(graph) = graph else { + return false; + }; + + for node in graph.node_weights() { + v.node(node); + } + + for edge in graph.raw_edges() { + let source = graph.node_weight(edge.source()); + let target = graph.node_weight(edge.target()); + + if let (Some(source), Some(target)) = (source, target) { + v.edge(source, edge.weight, target); + } + } + + true + } +} diff --git a/modules/analysis/src/test.rs b/modules/analysis/src/test.rs index 2a55e0caa..9ade9a118 100644 --- a/modules/analysis/src/test.rs +++ b/modules/analysis/src/test.rs @@ -1,15 +1,17 @@ use crate::{ endpoints::configure, - model::{AncNode, AncestorSummary}, + model::{BaseSummary, Node as GraphNode}, + service::AnalysisService, }; -use itertools::Itertools; +use trustify_entity::relationship::Relationship; use trustify_test_context::{ call::{self, CallService}, TrustifyContext, }; pub async fn caller(ctx: &TrustifyContext) -> anyhow::Result { - call::caller(|svc| configure(svc, ctx.db.clone())).await + let analysis = AnalysisService::new(); + call::caller(|svc| configure(svc, ctx.db.clone(), analysis)).await } #[derive(PartialEq, Eq, Debug, Copy, Clone)] @@ -42,8 +44,14 @@ struct RefNode<'a> { pub purls: Vec<&'a str>, } -impl<'a> From<&'a AncNode> for OwnedNode<'a> { - fn from(value: &'a AncNode) -> Self { +impl<'a> From<&'a GraphNode> for OwnedNode<'a> { + fn from(value: &'a GraphNode) -> Self { + (&value.base).into() + } +} + +impl<'a> From<&'a BaseSummary> for OwnedNode<'a> { + fn from(value: &'a BaseSummary) -> Self { Self { id: &value.node_id, name: &value.name, @@ -54,18 +62,15 @@ impl<'a> From<&'a AncNode> for OwnedNode<'a> { } } -pub fn assert_ancestors(ancestors: &[AncestorSummary], f: F) +pub fn assert_ancestors(ancestors: &[Vec<(&BaseSummary, Relationship)>], f: F) where F: for<'a> FnOnce(&'a [&'a [Node]]), { let ancestors = ancestors .iter() - .sorted_by_key(|a| &a.node_id) .map(|item| { - item.ancestors - .iter() - .map(OwnedNode::from) - .sorted_by_key(|n| n.id.to_string()) + item.iter() + .map(|(base, _)| OwnedNode::from(*base)) .collect::>() }) .collect::>(); diff --git a/modules/fundamental/tests/sbom/spdx/aliases.rs b/modules/fundamental/tests/sbom/spdx/aliases.rs index ee4f64b2c..49f91c48a 100644 --- a/modules/fundamental/tests/sbom/spdx/aliases.rs +++ b/modules/fundamental/tests/sbom/spdx/aliases.rs @@ -21,8 +21,9 @@ async fn cpe_purl(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { let service = AnalysisService::new(); let result = service - .retrieve_components( + .retrieve( ComponentReference::Id("SPDXRef-SRPM"), + (), Default::default(), &ctx.db, ) @@ -33,7 +34,8 @@ async fn cpe_purl(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { let item = &result.items[0]; assert_eq!( - item.cpe + item.base + .cpe .iter() .map(ToString::to_string) .sorted() @@ -44,7 +46,8 @@ async fn cpe_purl(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { ] ); assert_eq!( - item.purl + item.base + .purl .iter() .map(ToString::to_string) .sorted() diff --git a/modules/ingestor/src/graph/sbom/cyclonedx.rs b/modules/ingestor/src/graph/sbom/cyclonedx.rs index 7959626db..f6c66e6f7 100644 --- a/modules/ingestor/src/graph/sbom/cyclonedx.rs +++ b/modules/ingestor/src/graph/sbom/cyclonedx.rs @@ -9,7 +9,7 @@ use crate::graph::{ }; use sea_orm::ConnectionTrait; use serde_cyclonedx::cyclonedx::v_1_6::{ - Component, ComponentEvidenceIdentity, CycloneDx, LicenseChoiceUrl, RefLinkType, + Component, ComponentEvidenceIdentity, CycloneDx, LicenseChoiceUrl, }; use std::str::FromStr; use time::{format_description::well_known::Iso8601, OffsetDateTime}; @@ -137,9 +137,9 @@ impl SbomContext { // create a relationship creator.relate( - bom_ref, - Relationship::DescribedBy, CYCLONEDX_DOC_REF.to_string(), + Relationship::Describes, + bom_ref, ); } } @@ -151,12 +151,16 @@ impl SbomContext { // create relationships for left in sbom.dependencies.iter().flatten() { - creator.relate_all(&left.ref_, Relationship::DependencyOf, &left.depends_on); + for target in left.depends_on.iter().flatten() { + creator.relate(left.ref_.clone(), Relationship::Dependency, target.clone()); + } // https://github.com/trustification/trustify/issues/1131 // Do we need to qualify this so that only "arch=src" refs // get the GeneratedFrom relationship? - creator.relate_all(&left.ref_, Relationship::GeneratedFrom, &left.provides); + for target in left.depends_on.iter().flatten() { + creator.relate(left.ref_.clone(), Relationship::Generates, target.clone()); + } } // create @@ -208,17 +212,6 @@ impl<'a> Creator<'a> { self.relations.push((left, rel, right)); } - pub fn relate_all( - &mut self, - source: &RefLinkType, - rel: Relationship, - targets: &Option>, - ) { - for target in targets.iter().flatten() { - self.relate(target.clone(), rel, source.clone()); - } - } - pub async fn create(self, db: &impl ConnectionTrait) -> anyhow::Result<()> { let mut purls = PurlCreator::new(); let mut cpes = CpeCreator::new(); diff --git a/modules/ingestor/src/graph/sbom/spdx.rs b/modules/ingestor/src/graph/sbom/spdx.rs index 12a86780f..eb8d96797 100644 --- a/modules/ingestor/src/graph/sbom/spdx.rs +++ b/modules/ingestor/src/graph/sbom/spdx.rs @@ -251,31 +251,31 @@ impl<'spdx> TryFrom<(&'spdx str, &'spdx RelationshipType, &'spdx str)> for SpdxR ) -> Result { match rel { RelationshipType::AncestorOf => Ok((left, Relationship::AncestorOf, right)), - RelationshipType::BuildToolOf => Ok((left, Relationship::BuildToolOf, right)), - RelationshipType::ContainedBy => Ok((left, Relationship::ContainedBy, right)), - RelationshipType::Contains => Ok((right, Relationship::ContainedBy, left)), - RelationshipType::DependencyOf => Ok((left, Relationship::DependencyOf, right)), - RelationshipType::DependsOn => Ok((right, Relationship::DependencyOf, left)), + RelationshipType::BuildToolOf => Ok((right, Relationship::BuildTool, left)), + RelationshipType::ContainedBy => Ok((right, Relationship::Contains, left)), + RelationshipType::Contains => Ok((left, Relationship::Contains, right)), + RelationshipType::DependencyOf => Ok((right, Relationship::Dependency, left)), + RelationshipType::DependsOn => Ok((left, Relationship::Dependency, right)), RelationshipType::DescendantOf => Ok((right, Relationship::AncestorOf, left)), - RelationshipType::DescribedBy => Ok((left, Relationship::DescribedBy, right)), - RelationshipType::Describes => Ok((right, Relationship::DescribedBy, left)), - RelationshipType::DevDependencyOf => Ok((left, Relationship::DevDependencyOf, right)), - RelationshipType::DevToolOf => Ok((left, Relationship::DevToolOf, right)), - RelationshipType::ExampleOf => Ok((left, Relationship::ExampleOf, right)), - RelationshipType::GeneratedFrom => Ok((left, Relationship::GeneratedFrom, right)), - RelationshipType::Generates => Ok((right, Relationship::GeneratedFrom, left)), + RelationshipType::DescribedBy => Ok((right, Relationship::Describes, left)), + RelationshipType::Describes => Ok((left, Relationship::Describes, right)), + RelationshipType::DevDependencyOf => Ok((right, Relationship::DevDependency, left)), + RelationshipType::DevToolOf => Ok((right, Relationship::DevTool, left)), + RelationshipType::ExampleOf => Ok((right, Relationship::Example, left)), + RelationshipType::GeneratedFrom => Ok((right, Relationship::Generates, left)), + RelationshipType::Generates => Ok((left, Relationship::Generates, right)), RelationshipType::OptionalDependencyOf => { - Ok((left, Relationship::OptionalDependencyOf, right)) + Ok((right, Relationship::OptionalDependency, left)) } - RelationshipType::PackageOf => Ok((right, Relationship::PackageOf, left)), + RelationshipType::PackageOf => Ok((right, Relationship::Packages, left)), RelationshipType::ProvidedDependencyOf => { - Ok((left, Relationship::ProvidedDependencyOf, right)) + Ok((right, Relationship::ProvidedDependency, left)) } RelationshipType::RuntimeDependencyOf => { - Ok((left, Relationship::RuntimeDependencyOf, right)) + Ok((right, Relationship::RuntimeDependency, left)) } - RelationshipType::TestDependencyOf => Ok((left, Relationship::TestDependencyOf, right)), - RelationshipType::VariantOf => Ok((left, Relationship::VariantOf, right)), + RelationshipType::TestDependencyOf => Ok((right, Relationship::TestDependency, left)), + RelationshipType::VariantOf => Ok((right, Relationship::Variant, left)), _ => Err(()), } .map(|(left, rel, right)| Self(left, rel, right)) diff --git a/server/src/openapi.rs b/server/src/openapi.rs index 2b8bde4c1..c293c3d81 100644 --- a/server/src/openapi.rs +++ b/server/src/openapi.rs @@ -1,6 +1,7 @@ use crate::profile::api::{configure, default_openapi_info, Config, ModuleConfig}; use actix_web::App; use trustify_common::{config::Database, db}; +use trustify_module_analysis::service::AnalysisService; use trustify_module_storage::service::{dispatch::DispatchBackend, fs::FileSystemBackend}; use utoipa::{ openapi::security::{OpenIdConnect, SecurityScheme}, @@ -11,6 +12,7 @@ use utoipa_actix_web::AppExt; pub async fn create_openapi() -> anyhow::Result { let (db, postgresql) = db::embedded::create().await?; let (storage, _temp) = FileSystemBackend::for_test().await?; + let analysis = AnalysisService::new(); let (_, mut openapi) = App::new() .into_utoipa_app() @@ -22,6 +24,7 @@ pub async fn create_openapi() -> anyhow::Result { db, storage: storage.into(), auth: None, + analysis, with_graphql: true, }, ); diff --git a/server/src/profile/api.rs b/server/src/profile/api.rs index d2dbacca4..d17af5c0a 100644 --- a/server/src/profile/api.rs +++ b/server/src/profile/api.rs @@ -307,6 +307,7 @@ impl InitData { let ui = Arc::new(UiResources::new(&self.ui)?); let db = self.db.clone(); let storage = self.storage.clone(); + let analysis = AnalysisService::new(); let http = { HttpServerBuilder::try_from(self.http)? @@ -323,6 +324,7 @@ impl InitData { db: self.db.clone(), storage: self.storage.clone(), auth: self.authenticator.clone(), + analysis: analysis.clone(), with_graphql: self.with_graphql, }, @@ -369,6 +371,7 @@ pub(crate) struct Config { pub(crate) config: ModuleConfig, pub(crate) db: db::Database, pub(crate) storage: DispatchBackend, + pub(crate) analysis: AnalysisService, pub(crate) auth: Option>, pub(crate) with_graphql: bool, } @@ -382,6 +385,7 @@ pub(crate) fn configure(svc: &mut utoipa_actix_web::service_config::ServiceConfi db, storage, auth, + analysis, with_graphql, } = config; @@ -413,8 +417,6 @@ pub(crate) fn configure(svc: &mut utoipa_actix_web::service_config::ServiceConfi // register REST API & UI - let analysis = AnalysisService::new(); - svc.app_data(graph) .configure(|svc| { endpoints::configure(svc, auth.clone()); @@ -436,9 +438,9 @@ pub(crate) fn configure(svc: &mut utoipa_actix_web::service_config::ServiceConfi fundamental, db.clone(), storage, - analysis, + analysis.clone(), ); - trustify_module_analysis::endpoints::configure(svc, db.clone()); + trustify_module_analysis::endpoints::configure(svc, db.clone(), analysis); trustify_module_user::endpoints::configure(svc, db.clone()); }), ); @@ -497,6 +499,7 @@ mod test { let db = ctx.db; let (storage, _) = FileSystemBackend::for_test().await?; let ui = Arc::new(UiResources::new(&UI::default())?); + let analysis = AnalysisService::new(); let app = actix_web::test::init_service( App::new() .into_utoipa_app() @@ -509,6 +512,7 @@ mod test { db, storage: DispatchBackend::Filesystem(storage), auth: None, + analysis, with_graphql: true, }, );