5.4.13

inc/component.h

@@ -2,6 +2,9 @@
 #define __COMPONENT_H
 
 #include "scanoss.h"
+
+#define COMPONENT_DEFAULT_RANK 999 //default rank for components without rank information
+extern int component_rank_max;
 /**
  * @brief Component object definition.
  * 
@@ -43,9 +46,10 @@ typedef struct component_data_t
 	char * health_text; /* used in json output generation */
 	int hits; /*used in binary analysis*/
 	char * file_path_ref;
-	int path_rank;
-	int url_stats[5];
-	int health_stats[3];
+	int path_rank; /* Path ranking index*/
+	int url_stats[5]; /* url stats: quantity of file */
+	int health_stats[3]; /* health stats: forks, watchers, contributors */
+	int rank; /* purl ranking - optional*/
 } component_data_t;
 
 component_data_t * component_init(void);

inc/scanoss.h

@@ -40,7 +40,7 @@
 #define WFP_REC_LN 18
 
 /* Log files */
-#define SCANOSS_VERSION "5.4.12"
+#define SCANOSS_VERSION "5.4.13"
 #define SCAN_LOG "/tmp/scanoss_scan.log"
 #define MAP_DUMP "/tmp/scanoss_map.dump"
 #define SLOW_QUERY_LOG "/tmp/scanoss_slow_query.log"

inc/versions.h

@@ -10,7 +10,6 @@ typedef struct release_version
 } release_version;
 
 void normalise_version(char *version, char *component);
-void clean_versions(component_data_t *component);
 void add_versions(component_data_t *component, file_recordset *files, uint32_t records);
 void get_purl_version(release_version *release, char *purl, uint8_t *file_id);
 char * version_cleanup(char *  version, char * component);

src/component.c

@@ -230,6 +230,7 @@ bool fill_component(component_data_t *component, uint8_t *url_key, char *file_pa
 	char license[MAX_FIELD_LN];
 	char url[MAX_FILE_PATH];
 	char purl[MAX_FILE_PATH];
+	char rank[MAX_FIELD_LN];
 	// component->path_ln = 0;
 	if (!component)
 		return false;
@@ -251,6 +252,7 @@ bool fill_component(component_data_t *component, uint8_t *url_key, char *file_pa
 	extract_csv(license, (char *)url_record, 5, sizeof(license));
 	extract_csv(purl, (char *)url_record, 6, sizeof(purl));
 	extract_csv(url, (char *)url_record, 7, sizeof(url));
+	extract_csv(rank, (char *)url_record, 13, sizeof(url)); //extracts the rank field if available
 	/* Fill url stats if these are available*/
 	for (int i = 0; i < 5; i++) {
 		char stat[16] = "\0";
@@ -289,6 +291,11 @@ bool fill_component(component_data_t *component, uint8_t *url_key, char *file_pa
 		MD5((uint8_t *)component->purls[0], strlen(component->purls[0]), component->purls_md5[0]);
 	}
 	component->age = -1;
+	if (*rank)
+		component->rank = atoi(rank);
+	else
+		component->rank = COMPONENT_DEFAULT_RANK;
+
 	return true;
 }
 

src/main.c

@@ -42,7 +42,7 @@
 #include "scan.h"
 #include "scanoss.h"
 #include "util.h"
-
+#include "component.h"
 #include <decrypt.h>
 #include "hpsm.h"
 #include <dlfcn.h>
@@ -261,6 +261,8 @@ uint64_t read_flags()
 	return 0;
 }
 
+
+int component_rank_max = COMPONENT_DEFAULT_RANK; /*Used defined max component rank accepted*/
 /**
  * @brief //TODO
  * @param argc //TODO
@@ -291,7 +293,7 @@ int main(int argc, char **argv)
 	int option;
 	bool invalid_argument = false;
 	char * ldb_db_name = NULL;
-	while ((option = getopt(argc, argv, ":T:s:b:B:c:k:a:F:l:n:M:N:wtvhedqH")) != -1)
+	while ((option = getopt(argc, argv, ":r:T:s:b:B:c:k:a:F:l:n:M:N:wtvhedqH")) != -1)
 	{
 		/* Check valid alpha is entered */
 		if (optarg)
@@ -318,6 +320,9 @@ int main(int argc, char **argv)
 			case 'c':
 				component_hint = strdup(optarg);
 				break;
+			case 'r':
+				component_rank_max = atoi(optarg);
+				break;
 
 			case 'k':
 				initialize_ldb_tables(ldb_db_name);

src/match.c

@@ -315,6 +315,22 @@ static bool component_hint_date_comparation(component_data_t *a, component_data_
 		else if (a->path_rank < PATH_LEVEL_COMP_REF / 3 + 1)
 			return false;
 	}
+
+	//lower rank selection logic
+	if (b->rank < COMPONENT_DEFAULT_RANK)
+	{
+		if (b->rank < a->rank)
+		{
+			scanlog("%s wins %s by rank %d/%d\n", b->purls[0],  a->purls[0], b->rank, a->rank);
+			return true;
+		}
+		else if (b->rank > a->rank)
+		{
+			scanlog("%s rejected by rank %d\n", b->purls[0], b->rank);
+			return false;
+		}
+	}
+
 	if (!*b->release_date)
 		return false;
 	if (!*a->release_date)
@@ -403,14 +419,13 @@ bool add_component_from_urlid(component_list_t *component_list, uint8_t *url_id,
 	if (!component_list_add(component_list, new_comp, component_hint_date_comparation, true))
 	{
 		component_data_free(new_comp); /* Free if the componet was rejected */
-	}
-	else
-	{
-		char hex_url[MD5_LEN * 2 + 1];
-		ldb_bin_to_hex(new_comp->url_md5, MD5_LEN, hex_url);
-		scanlog("component accepted: %s@%s - pathrank: %d - %s - %s\n", new_comp->purls[0], new_comp->version, new_comp->path_rank, new_comp->file, hex_url);
+		return false;
 	}
 
+	char hex_url[MD5_LEN * 2 + 1];
+	ldb_bin_to_hex(new_comp->url_md5, MD5_LEN, hex_url);
+	scanlog("component accepted: %s@%s - pathrank: %d - %s - %s\n", new_comp->purls[0], new_comp->version, new_comp->path_rank, new_comp->file, hex_url);
+
 	return true;
 }
 
@@ -425,14 +440,16 @@ bool add_component_from_urlid(component_list_t *component_list, uint8_t *url_id,
  * @return true
  * @return false
  */
-
+/*Iterations must be doubled if high accuracy is enabled*/
+int iteration_max = FETCH_MAX_FILES;
 bool component_from_file(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *raw_data, uint32_t datalen, int iteration, void *ptr)
 {
 	/*Iterations must be doubled if high accuracy is enabled*/
-	int iteration_max = ((engine_flags & ENABLE_HIGH_ACCURACY) ? FETCH_MAX_FILES * 4 : FETCH_MAX_FILES);
+	if (iteration == 0)
+		iteration_max = ((engine_flags & ENABLE_HIGH_ACCURACY) ? FETCH_MAX_FILES * 4 : FETCH_MAX_FILES);
 
 	/*Return we high accuracy it is not enabled*/
-	if (iteration > iteration_max * 2 && !(engine_flags & ENABLE_HIGH_ACCURACY))
+	if (iteration > iteration_max)
 		return true;
 
 	/* Ignore path lengths over the limit */
@@ -456,7 +473,8 @@ bool component_from_file(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *
 	strncpy(path, decrypted, MAX_FILE_PATH);
 	//check the ignore list only if the match type is MATCH_SNIPPET. TODO: remove this after remine everything.
 	if (!(component_list->match_ref->type == MATCH_SNIPPET && ignored_extension(path)))
-		add_component_from_urlid(component_list, url_id, path);
+		if (!add_component_from_urlid(component_list, url_id, path))
+			iteration_max++; //allow one more iteration if the component was rejected.
 
 	free(decrypted);
 	return false;

src/report.c

@@ -224,13 +224,11 @@ bool print_json_component(component_data_t * component)
 	printf("\"vendor\": \"%s\",", component->vendor);
 	printf("\"component\": \"%s\",", component->component);
 
-	char * version_clean = version_cleanup(component->version, component->component);
+	char * version_clean = string_clean(component->version);
 	printf("\"version\": \"%s\",", version_clean ? version_clean : "");
-	free(version_clean);
 
-	char * lastest_clean = version_cleanup(component->latest_version, component->component);
+	char * lastest_clean = string_clean(component->latest_version);
 	printf("\"latest\": \"%s\",", lastest_clean ? lastest_clean : "");
-	free(lastest_clean);
 
 	printf("\"url\": \"%s\",", component->main_url ? component->main_url : component->url);
 

src/url.c

@@ -71,7 +71,7 @@ bool handle_url_record(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *ra
 	component_data_t * new_comp = calloc(1, sizeof(*new_comp));
 	bool result = fill_component(new_comp, NULL, NULL, (uint8_t*) data);
 	scanlog("URL MATCH: %s\n", data);
-	if (result)
+	if (result && new_comp->rank <= component_rank_max)
 	{
 		/* Save match component id */
 		memcpy(new_comp->url_md5, key, LDB_KEY_LN);
@@ -336,7 +336,7 @@ bool get_oldest_url(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *data,
 	{
 		component_data_t * comp = calloc(1, sizeof(*comp));
 		bool result = fill_component(comp, key, NULL, (uint8_t *)url);
-		if (!result)
+		if (!result || comp->rank > component_rank_max)
 		{
 			free(url);
 			component_data_free(comp);
@@ -362,6 +362,10 @@ bool get_oldest_url(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *data,
 		/* If it is older, then we copy to oldest */
 		else if(comp->identified == comp_oldest->identified)
 		{
+
+			if (comp->rank < comp_oldest->rank) //lowest rank is 1.
+				replace = true;
+
 			if ((!*comp_oldest->release_date && *comp->release_date) || 
 				(*comp->release_date && (strcmp(comp->release_date, comp_oldest->release_date) < 0)))
 				replace = true;

src/versions.c

@@ -75,28 +75,6 @@ void normalise_version(char *version, char *component)
 		strcpy(version, aux);
 }
 
-/**
- * @brief Normalize versions for a match
- * @param match match to be processed
- */
-void clean_versions(component_data_t *component)
-{
-	normalise_version(component->version, component->component);
-	normalise_version(component->latest_version, component->component);
-}
-
-char * version_cleanup(char *  version, char * component)
-{
-	if (!version)
-		return NULL;
-
-	char cleaned[MAX_FIELD_LN] = "\0";
-	strcpy(cleaned, version);
-	normalise_version(cleaned, component);
-
-	return strdup(cleaned);
-}
-
 static char * purl_indirection_reference[FETCH_MAX_FILES];
 static int purl_indirection_index = 0;
 static release_version * purl_version_list[FETCH_MAX_FILES];

src/vulnerability.c

@@ -260,7 +260,6 @@ int print_vulnerabilities(component_data_t *component)
 	memset(crclist, 0, sizeof(crclist));
 
 	component_data_t comp = *component;
-	clean_versions(&comp);
 	component->vulnerabilities_text = NULL;
 	component->vulnerabilities = 0;
 	component->crclist = crclist;