added github-ci

.github/workflows/release-publish.yml

@@ -18,6 +18,7 @@ jobs:
         os:
           - 'debian:buster'
           - 'debian:stretch'
+          - 'debian:bullseye'
           - 'ubuntu:bionic'
           - 'ubuntu:focal'
 

.github/workflows/static-checks.yml

@@ -59,7 +59,7 @@ jobs:
         run: |
           composer install --prefer-dist --working-dir=src
           src/vendor/bin/phpcs --standard=src/fossy-ruleset.xml -p
-  
+
   copy-paste:
     runs-on: ubuntu-20.04
     continue-on-error: true

.github/workflows/test-checks.yml

@@ -0,0 +1,70 @@
+# Copyright 2021 Anupam Ghosh <[email protected]>,
+#   Gaurav Mishra <[email protected]>
+# SPDX-License-Identifier: GPL-2.0 AND LGPL-2.1
+
+name: c tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  C-tests:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        php: ['7.4'] # Can be extended in future
+    services:
+      postgres:
+        image: postgres:12
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_HOST_AUTH_METHOD: trust
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 50
+
+      - name: Install Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libcppunit-dev
+          sudo ./utils/fo-installdeps -y -e
+          export PATH="/usr/lib/ccache/:$PATH"
+
+      - name: Install PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: ${{ matrix.php }}
+          extensions: mbstring, gd, xml, zip, json, sqlite3, curl
+        env:
+          fail-fast: true
+
+      - name: Setup test database
+        env:
+          PGHOST: 127.0.0.1
+          PGPORT: 5432
+        run: |
+          ./utils/prepare-test -afty
+
+          #remove test all as delagent is failing
+      - name: Unit tests
+        env:
+          PGHOST: 127.0.0.1
+          PGPORT: 5432
+        run: |
+          make all phpvendors test-cli test-copyright test-lib test-monk \
+          test-nomos test-ojo test-reuser test-scheduler test-spdx2 \
+          test-unifiedreport test-www

CONTRIBUTING.md

@@ -71,10 +71,10 @@ The cardinal rule for creating good commits is to ensure there is only one
   code changes entangled with the original commit.
 
 - When troubleshooting problems using Git's bisect capability, small well
-  defined changes will aid in isolating exactly where the code problem was
+  -defined changes will aid in isolating exactly where the code problem was
   introduced.
 
-- When browsing history using Git annotate/blame, small well defined changes
+- When browsing history using Git annotate/blame, small well-defined changes
   also aid in isolating exactly where & why a piece of code came from.
 
 Things to avoid when creating commits

README.md

@@ -11,7 +11,7 @@
 
 ## About
 
-FOSSology is an open source license compliance software system and toolkit. As a toolkit, you can run license, copyright and export control scans from the command line. As a system, a database and web UI are provided to give you a compliance workflow. In one click you can generate an SPDX file or a ReadMe with all the copyrights notices from your software. FOSSology deduplication means that you can scan an entire distro, rescan a new version, and only the changed files will get rescanned. This is a big time saver for large projects.
+FOSSology is an open source license compliance software system and toolkit. As a toolkit, you can run license, copyright, and export control scans from the command line. As a system, a database and web UI are provided to give you a compliance workflow. In one click you can generate an SPDX file or a ReadMe with all the copyrights notices from your software. FOSSology deduplication means that you can scan an entire distro, rescan a new version, and only the changed files will get rescanned. This is a big time saver for large projects.
 
 [Check out Who Uses FOSSology!](https://www.fossology.org)
 
@@ -98,7 +98,7 @@ and developer docs on [Github Wiki](https://github.com/fossology/fossology/wiki)
 
 ## Support
 
-Mailing lists, FAQs, Release Notes, and other useful info is available
+Mailing lists, FAQs, Release Notes, and other useful info are available
 by clicking the documentation tab on the project website. We encourage
 all users to join the mailing list and participate in discussions.
 There is also a #fossology IRC channel on the freenode IRC network if

src/cli/exportLicenseRefUsingSPDX.php

@@ -1,6 +1,6 @@
 <?php
 /***********************************************************
- Copyright (C) 2019 Siemens AG
+ Copyright (C) 2019,2021 Siemens AG
 
  This program is free software; you can redistribute it and/or
  modify it under the terms of the GNU General Public License
@@ -42,6 +42,10 @@ function startProcessingLicenseData()
     $updateExisting = '';
     $addNewLicense = '';
     $newLicenseRefData = array();
+    $scanList = array(
+      'licenses' => 'https://spdx.org/licenses/licenses.json',
+      'exceptions' => 'https://spdx.org/licenses/exceptions.json'
+    );
     $usage = "Usage: " . basename($argv[0]) . " [options]
 
       Create new licenseref.json file.  Options are:
@@ -64,10 +68,6 @@ function startProcessingLicenseData()
           For type 'licenses' URL is : $scanList[licenses]
 
           For type 'exceptions' URL is : $scanList[exceptions]";
-    $scanList = array(
-    'licenses' => 'https://spdx.org/licenses/licenses.json',
-    'exceptions' => 'https://spdx.org/licenses/exceptions.json'
-    );
 
     $options = getopt("hcEen", array("type:","url:"));
     /* get type and url if exists if not set them to empty */
@@ -203,10 +203,10 @@ function getListSPDX($type, $URL, $updateWithNew, $updateExisting, $addNewLicens
             !empty($updateExisting)
           )
          ) {
-        $existingLicenseRefData[$licenseIdCheck]['rf_fullname'] = $getCurrentData[$this->mapArrayData[$type][2]];
-        $existingLicenseRefData[$licenseIdCheck]['rf_text'] = $getCurrentData[$this->mapArrayData[$type][1]];
+        $existingLicenseRefData[$licenseIdCheck]['rf_fullname'] = $this->replaceUnicode($getCurrentData[$this->mapArrayData[$type][2]]);
+        $existingLicenseRefData[$licenseIdCheck]['rf_text'] = $this->replaceUnicode($getCurrentData[$this->mapArrayData[$type][1]]);
         $existingLicenseRefData[$licenseIdCheck]['rf_url'] = $getCurrentData['seeAlso'][0];
-        $existingLicenseRefData[$licenseIdCheck]['rf_notes'] = (array_key_exists("licenseComments", $getCurrentData) ? $getCurrentData['licenseComments'] : $existingLicenseRefData[$licenseIdCheck]['rf_notes']);
+        $existingLicenseRefData[$licenseIdCheck]['rf_notes'] = $this->replaceUnicode((array_key_exists("licenseComments", $getCurrentData) ? $getCurrentData['licenseComments'] : $existingLicenseRefData[$licenseIdCheck]['rf_notes']));
         echo "INFO: license ".$getCurrentData[$this->mapArrayData[$type][0]]." updated\n\n";
       }
       if (!is_numeric($licenseIdCheck) &&
@@ -218,7 +218,7 @@ function getListSPDX($type, $URL, $updateWithNew, $updateExisting, $addNewLicens
          ) {
         $existingLicenseRefData[] = array(
           'rf_shortname' => $getCurrentData[$this->mapArrayData[$type][0]],
-          'rf_text' =>  $getCurrentData[$this->mapArrayData[$type][1]],
+          'rf_text' =>  $this->replaceUnicode($getCurrentData[$this->mapArrayData[$type][1]]),
           'rf_url' =>  $getCurrentData['seeAlso'][0],
           'rf_add_date' => null,
           'rf_copyleft' => null,
@@ -227,7 +227,7 @@ function getListSPDX($type, $URL, $updateWithNew, $updateExisting, $addNewLicens
           'rf_FSFfree' => null,
           'rf_GPLv2compatible' => null,
           'rf_GPLv3compatible' => null,
-          'rf_notes' => (array_key_exists("licenseComments", $getCurrentData) ? $getCurrentData['licenseComments'] : null),
+          'rf_notes' => $this->replaceUnicode((array_key_exists("licenseComments", $getCurrentData) ? $getCurrentData['licenseComments'] : null)),
           'rf_Fedora' => null,
           'marydone' => "f",
           'rf_active' => "t",
@@ -243,6 +243,35 @@ function getListSPDX($type, $URL, $updateWithNew, $updateExisting, $addNewLicens
     }
     return $existingLicenseRefData;
   }
+
+  /**
+   * Replace common unicode characters with ASCII for consistent results.
+   *
+   * @param string $text Input text
+   * @return string Input with characters replaced
+   */
+  private function replaceUnicode($text)
+  {
+    $search = [
+      '\u00a0',  // no break space
+      '\u2018',  // Left single quote
+      '\u2019',  // Right single quote
+      '\u201c',  // Left double quote
+      '\u201d',  // Right double quote
+      '\u2013',  // em dash
+    ];
+
+    $replace = [
+      " ",
+      "'",
+      "'",
+      '"',
+      '"',
+      "-",
+    ];
+
+    return str_replace($search, $replace, $text);
+  }
 }
 $obj = new exportLicenseRef();
 echo $obj->startProcessingLicenseData();

src/copyright/mod_deps

@@ -99,8 +99,10 @@ if [[ $RUNTIME ]]; then
       case "$CODENAME" in
         stretch)
           apt-get $YesOpt install libjsoncpp1 libboost-filesystem1.62.0;;
-        buster|sid)
+        buster)
           apt-get $YesOpt install libjsoncpp1 libboost-filesystem1.67.0;;
+        bullseye|sid)
+          apt-get $YesOpt install libjsoncpp24 libboost-filesystem1.74.0;;
         bionic)
           apt-get $YesOpt install libjsoncpp1 libboost-filesystem1.65.1;;
         focal)

src/decider/agent/BulkReuser.php

@@ -68,8 +68,8 @@ public function rerunBulkAndDeciderOnUpload($uploadId, $groupId, $userId, $bulkI
      */
     $deciderPlugin = plugin_find("agent_deciderjob");
     $dependecies = array();
-    $sql = "INSERT INTO license_ref_bulk (user_fk,group_fk,rf_text,upload_fk,uploadtree_fk,ignore_irrelevant) "
-            . "SELECT $1 AS user_fk, $2 AS group_fk,rf_text,$3 AS upload_fk, $4 as uploadtree_fk, ignore_irrelevant
+    $sql = "INSERT INTO license_ref_bulk (user_fk,group_fk,rf_text,upload_fk,uploadtree_fk,ignore_irrelevant,bulk_delimiters) "
+            . "SELECT $1 AS user_fk, $2 AS group_fk,rf_text,$3 AS upload_fk, $4 as uploadtree_fk, ignore_irrelevant, bulk_delimiters
               FROM license_ref_bulk WHERE lrb_pk=$5 RETURNING lrb_pk, $5 as lrb_origin";
     $sqlLic = "INSERT INTO license_set_bulk (lrb_fk, rf_fk, removing, comment, reportinfo, acknowledgement) "
             ."SELECT $1 as lrb_fk, rf_fk, removing, comment, reportinfo, acknowledgement FROM license_set_bulk WHERE lrb_fk=$2";

src/lib/php/Dao/LicenseDao.php

@@ -551,16 +551,24 @@ public function getLicenseByShortName($licenseShortname, $groupId=null)
    * @param bool[] $licenseRemovals
    * @param string $refText
    * @param bool $ignoreIrrelevant Ignore irrelevant files while scanning
+   * @param string $delimiters Delimiters for bulk scan,
+   *                           null or "DEFAULT" for default values
    * @return int lrp_pk on success or -1 on fail
    */
-  public function insertBulkLicense($userId, $groupId, $uploadTreeId, $licenseRemovals, $refText, $ignoreIrrelevant=true)
+  public function insertBulkLicense($userId, $groupId, $uploadTreeId, $licenseRemovals, $refText, $ignoreIrrelevant=true, $delimiters=null)
   {
+    if (strcasecmp($delimiters, "DEFAULT") === 0) {
+      $delimiters = null;
+    } elseif ($delimiters !== null) {
+      $delimiters = StringOperation::replaceUnicodeControlChar($delimiters);
+    }
     $licenseRefBulkIdResult = $this->dbManager->getSingleRow(
-        "INSERT INTO license_ref_bulk (user_fk, group_fk, uploadtree_fk, rf_text, ignore_irrelevant)
-      VALUES ($1,$2,$3,$4,$5) RETURNING lrb_pk",
+        "INSERT INTO license_ref_bulk (user_fk, group_fk, uploadtree_fk, rf_text, ignore_irrelevant, bulk_delimiters)
+      VALUES ($1,$2,$3,$4,$5,$6) RETURNING lrb_pk",
         array($userId, $groupId, $uploadTreeId,
           StringOperation::replaceUnicodeControlChar($refText),
-          $this->dbManager->booleanToDb($ignoreIrrelevant)),
+          $this->dbManager->booleanToDb($ignoreIrrelevant),
+          $delimiters),
         __METHOD__ . '.getLrb'
     );
     if ($licenseRefBulkIdResult === false) {

src/lib/php/UI/template/include/upload.html.twig

@@ -34,7 +34,7 @@
       <li>
         <div class="form-group">
           <label for="{{ fileInputName }}">({{ 'Optional'|trans }}) {{ 'Enter a description of this file'| trans }}:</label>
-          <input type="text" class="form-control" style="width:40%;" name="{{ descriptionInputName }} value="{{ descriptionInputValue }}">
+          <input type="text" class="form-control" style="width:40%;" name="{{ descriptionInputName }}" value="{{ descriptionInputValue }}">
         </div>
       </li>
       <li>

src/monk/agent/match.c

@@ -105,15 +105,15 @@ static char* getFileName(MonkState* state, long pFileId) {
   return pFileName;
 }
 
-int matchPFileWithLicenses(MonkState* state, long pFileId, const Licenses* licenses, const MatchCallbacks* callbacks) {
+int matchPFileWithLicenses(MonkState* state, long pFileId, const Licenses* licenses, const MatchCallbacks* callbacks, char* delimiters) {
   File file;
   file.id = pFileId;
   int result = 0;
 
   file.fileName = getFileName(state, pFileId);
-  
+
   if (file.fileName != NULL) {
-    result = readTokensFromFile(file.fileName, &(file.tokens), DELIMITERS);
+    result = readTokensFromFile(file.fileName, &(file.tokens), delimiters);
 
     if (result) {
       result = matchFileWithLicenses(state, &file, licenses, callbacks);

src/monk/agent/match.h

@@ -62,7 +62,7 @@ size_t match_getEnd(const Match* match);
 
 GArray* findAllMatchesBetween(const File* file, const Licenses* licenses, unsigned maxAllowedDiff, unsigned minAdjacentMatches, unsigned maxLeadingDiff);
 
-int matchPFileWithLicenses(MonkState* state, long pFileId, const Licenses* licenses, const MatchCallbacks* callbacks);
+int matchPFileWithLicenses(MonkState* state, long pFileId, const Licenses* licenses, const MatchCallbacks* callbacks, char* delimiters);
 int matchFileWithLicenses(MonkState* state, const File* file, const Licenses* licenses, const MatchCallbacks* callbacks);
 
 void findDiffMatches(const File* file, const License* license,

src/monk/agent/monk.h

@@ -35,7 +35,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #define DIFF_TYPE_REMOVAL "M-"
 #define DIFF_TYPE_REPLACE "MR"
 
-#define DELIMITERS " \t\n\r\f#^%"
+#define DELIMITERS " \t\n\r\f#^%,*"
 
 #define MONK_CASE_INSENSITIVE
 #define MAX_ALLOWED_DIFF_LENGTH 256

src/monk/agent/monkbulk.c

@@ -85,7 +85,7 @@ int queryBulkArguments(MonkState* state, long bulkId) {
       state->dbManager,
       "queryBulkArguments",
       "SELECT ut.upload_fk, ut.uploadtree_pk, lrb.user_fk, lrb.group_fk, "
-      "lrb.rf_text, lrb.ignore_irrelevant "
+      "lrb.rf_text, lrb.ignore_irrelevant, lrb.bulk_delimiters "
       "FROM license_ref_bulk lrb INNER JOIN uploadtree ut "
       "ON ut.uploadtree_pk = lrb.uploadtree_fk "
       "WHERE lrb_pk = $1",
@@ -105,6 +105,15 @@ int queryBulkArguments(MonkState* state, long bulkId) {
       bulkArguments->groupId = atoi(PQgetvalue(bulkArgumentsResult, 0, column++));
       bulkArguments->refText = g_strdup(PQgetvalue(bulkArgumentsResult, 0, column++));
       bulkArguments->ignoreIrre = strcmp(PQgetvalue(bulkArgumentsResult, 0, column++), "t") == 0;
+      if (PQgetisnull(bulkArgumentsResult, 0, column) == 1)
+      {
+        bulkArguments->delimiters = g_strdup(DELIMITERS);
+        column++;
+      }
+      else
+      {
+        bulkArguments->delimiters = normalize_escape_string(PQgetvalue(bulkArgumentsResult, 0, column++));
+      }
       bulkArguments->bulkId = bulkId;
       bulkArguments->actions = queryBulkActions(state, bulkId);
       bulkArguments->jobId = fo_scheduler_jobId();
@@ -169,6 +178,7 @@ void bulkArguments_contents_free(BulkArguments* bulkArguments) {
   free(bulkActions);
 
   g_free(bulkArguments->refText);
+  g_free(bulkArguments->delimiters);
 
   free(bulkArguments);
 }
@@ -179,7 +189,7 @@ int bulk_identification(MonkState* state) {
   License license = (License){
     .refId = bulkArguments->licenseId,
   };
-  license.tokens = tokenize(bulkArguments->refText, DELIMITERS);
+  license.tokens = tokenize(bulkArguments->refText, bulkArguments->delimiters);
 
   GArray* licenseArray = g_array_new(FALSE, FALSE, sizeof (License));
   g_array_append_val(licenseArray, license);
@@ -217,7 +227,8 @@ int bulk_identification(MonkState* state) {
 
           long fileId = atol(PQgetvalue(filesResult, i, 0));
 
-          if (matchPFileWithLicenses(threadLocalState, fileId, licenses, &bulkCallbacks)) {
+          if (matchPFileWithLicenses(threadLocalState, fileId, licenses,
+            &bulkCallbacks, bulkArguments->delimiters)) {
             fo_scheduler_heart(1);
           } else {
             fo_scheduler_heart(0);

src/monk/agent/monkbulk.h

@@ -49,6 +49,7 @@ typedef struct {
   int groupId;
   char* refText;
   bool ignoreIrre;
+  char* delimiters;
   BulkAction** actions;
 } BulkArguments;
 

src/monk/agent/scheduler.c

@@ -69,7 +69,7 @@ int processUploadId(MonkState* state, int uploadId, const Licenses* licenses) {
           continue;
         }
 
-        if (matchPFileWithLicenses(threadLocalState, pFileId, licenses, &schedulerCallbacks)) {
+        if (matchPFileWithLicenses(threadLocalState, pFileId, licenses, &schedulerCallbacks, DELIMITERS)) {
           fo_scheduler_heart(1);
         } else {
           fo_scheduler_heart(0);