Merge pull request #250 from SAP/develop

upgrade to v4.9

.github/workflows/test.yml

@@ -77,5 +77,6 @@ jobs:
         POSTGRES_DB: credential_digger_tests
         DBHOST: localhost
         DBPORT: 5432
+        GIT_TOKEN: ${{ secrets.GIT_TOKEN }}
       run: |
         pytest tests/functional_tests

.pep8speaks.yml

@@ -6,6 +6,7 @@ scanner:
 
 flake8:  # Same as scanner.linter value
     ignore:  # Errors and warnings to ignore
+        - E704  # multiple statements on one line (def)
         - W292  # no newline at the end of file (introduces W391)
         - W503  # line break before binary operator
         - W504  # line break after binary operator

credentialdigger/cli/cli.py

@@ -6,7 +6,7 @@
 from dotenv import load_dotenv
 
 from . import (add_rules, get_discoveries, hook, scan, scan_path,
-               scan_snapshot, scan_user, scan_wiki)
+               scan_pr, scan_snapshot, scan_user, scan_wiki)
 
 logger = logging.getLogger(__name__)
 
@@ -100,6 +100,12 @@ def main(sys_argv):
         parents=[parser_dotenv, parser_sqlite, parser_scan_base])
     scan_snapshot.configure_parser(parser_scan_snapshot)
 
+    # scan_pr subparser configuration
+    parser_scan_pr = subparsers.add_parser(
+        'scan_pr', help='Scan a pull request',
+        parents=[parser_dotenv, parser_sqlite, parser_scan_base])
+    scan_pr.configure_parser(parser_scan_pr)
+
     # get_discoveries subparser configuration
     parser_get_discoveries = subparsers.add_parser(
         'get_discoveries', help='Get discoveries of a scanened repository',
@@ -129,6 +135,7 @@ def main(sys_argv):
         scan_user.run,
         scan_wiki.run,
         scan_path.run,
+        scan_pr.run,
         scan_snapshot.run
     ]:
         # Connect to db only when running commands that need it

credentialdigger/cli/scan_pr.py

@@ -0,0 +1,122 @@
+"""
+The 'scan' module can be used to scan a git repository on the fly from the
+terminal. It supports both the Sqlite and Postgres clients.
+
+NOTE: Postgres is used by default. Please make sure that the environment
+variables are exported and that the rules have already been added to the
+database.
+
+
+usage: credentialdigger scan_pr [-h] [--dotenv DOTENV] [--sqlite SQLITE]
+                                [--pr PULL_REQUEST_NUMBER]
+                                [--api_endpoint API_ENDPOINT]
+                                [--category CATEGORY]
+                                [--models MODELS [MODELS ...]]
+                                [--force] [--debug]
+                                [--git_token GIT_TOKEN]
+                                [--similarity]
+                                repo_url
+
+positional arguments:
+  repo_url              The location of a git repository
+  pr                    The number of the pull request to scan
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --dotenv DOTENV       The path to the .env file which will be used in all
+                        commands. If not specified, the one in the current
+                        directory will be used (if present).
+  --sqlite SQLITE       If specified, scan the repo using the sqlite client
+                        passing as argument the path of the db. Otherwise, use
+                        postgres (must be up and running)
+  --api_endpoint API_ENDPOINT
+                        API endpoint of the git server (default is the public
+                        github, i.e., `https://api.github.com`)
+  --category CATEGORY   If specified, scan the PR using all the rules of
+                        this category, otherwise use all the rules in the db
+  --models MODELS [MODELS ...]
+                        A list of models for the ML false positives detection.
+                        Cannot accept empty lists.
+  --force               Wipe previous scan results, in case this repository
+                        has already been scanned previously
+  --debug               Flag used to decide whether to visualize the
+                        progressbars during the scan (e.g., during the
+                        insertion of the detections in the db)
+  --git_token GIT_TOKEN
+                        Git personal access token to authenticate to the git
+                        server
+ --similarity           Build and use the similarity model to compute
+                        embeddings and allow for automatic update of similar
+                        snippets
+
+"""
+import logging
+import sys
+
+logger = logging.getLogger(__name__)
+
+
+def configure_parser(parser):
+    """
+    Configure arguments for command line parser.
+
+    Parameters
+    ----------
+    parser: `credentialdigger.cli.customParser`
+        Command line parser
+    """
+    parser.set_defaults(func=run)
+    parser.add_argument(
+        'repo_url', type=str,
+        help='The location of a git repository (an url if --local is not set, \
+            a local path otherwise)')
+    parser.add_argument(
+        '--pr', type=int, required=True,
+        help='The number of pull request to scan')
+    parser.add_argument(
+        '--api_endpoint', type=str, default='https://api.github.com',
+        help='API endpoint of the git server')
+    parser.add_argument(
+        '--force', action='store_true',
+        help='Force a complete re-scan of the repository, in case it has \
+            already been scanned previously')
+    parser.add_argument(
+        '--similarity', action='store_true',
+        help='Build and use the similarity model to compute embeddings \
+            and allow for automatic update of similar snippets')
+    parser.add_argument(
+        '--git_token', default=None, type=str,
+        help='Git personal access token to authenticate to the git server')
+
+
+def run(client, args):
+    """
+    Scan a pull request in a git repository.
+
+    Parameters
+    ----------
+    client: `credentialdigger.Client`
+        Instance of the client on which to save results
+    args: `argparse.Namespace`
+        Arguments from command line parser.
+
+    Returns
+    -------
+        While this function returns nothing of use to the scanner itself, it
+        gives an exit status (integer) that is equal to the number of
+        discoveries. If it exits with a value that is equal to 0, then it means
+        that the scan detected no leaks in this repo.
+    """
+    logger.info(f'Scan pull request number {args.pr}')
+    discoveries = client.scan_pull_request(
+        repo_url=args.repo_url,
+        pr_number=args.pr,
+        category=args.category,
+        models=args.models,
+        force=args.force,
+        debug=args.debug,
+        similarity=args.similarity,
+        git_token=args.git_token,
+        api_endpoint=args.api_endpoint)
+
+    sys.exit(len(discoveries))

credentialdigger/client.py

@@ -14,6 +14,7 @@
 from .models.model_manager import ModelManager
 from .scanners.file_scanner import FileScanner
 from .scanners.git_file_scanner import GitFileScanner
+from .scanners.git_pr_scanner import GitPRScanner
 from .scanners.git_scanner import GitScanner
 from .snippet_similarity import (build_embedding_model, compute_similarity,
                                  compute_snippet_embedding)
@@ -511,7 +512,7 @@ def get_discoveries(self, query, repo_url, file_name=None):
         """
         cursor = self.db.cursor()
         all_discoveries = []
-        params = (repo_url,) if file_name is None else (
+        params = (repo_url,) if not file_name else (
             repo_url, file_name)
         cursor.execute(query, params)
         result = cursor.fetchone()
@@ -787,6 +788,9 @@ def scan(self, repo_url, category=None, models=None, force=False,
                 repo_url = repo_url[:-1]
             if repo_url.endswith('.git'):
                 repo_url = repo_url[:-4]
+            # NB: removesuffix not supported with py<3.9
+            # repo_url = repo_url.removesuffix('/')
+            # repo_url = repo_url.removesuffix('.git')
 
         rules = self._get_scan_rules(category)
         scanner = GitScanner(rules)
@@ -841,7 +845,7 @@ def scan_snapshot(self, repo_url, branch_or_commit, category=None,
             The id of the discoveries detected by the scanner (excluded the
             ones classified as false positives).
         """
-        if self.get_repo(repo_url) != {}:
+        if self.get_repo(repo_url):
             logger.info(f'The repository \"{repo_url}\" has already been '
                         'scanned.')
             if force:
@@ -896,7 +900,7 @@ def scan_path(self, scan_path, category=None, models=None, force=False,
         """
         scan_path = os.path.abspath(scan_path)
 
-        if self.get_repo(scan_path) != {} and force is False:
+        if self.get_repo(scan_path) and not force:
             raise ValueError(f'The directory \"{scan_path}\" has already been '
                              'scanned. Please use \"force\" to rescan it.')
 
@@ -908,6 +912,71 @@ def scan_path(self, scan_path, category=None, models=None, force=False,
             debug=debug, similarity=similarity, max_depth=max_depth,
             ignore_list=ignore_list)
 
+    def scan_pull_request(self, repo_url, pr_number,
+                          api_endpoint='https://api.github.com',
+                          category=None, models=None, force=False, debug=False,
+                          similarity=False, git_token=None):
+        """ Launch the scan of a pull request.
+
+        Only the commits part of the pull request get scanned.
+
+        Parameters
+        ----------
+        repo_url: str
+            The url of the repo to scan
+        pr_number: int
+            The number of pull request
+        api_endpoint: str, default `https://api.github.com`
+            API endpoint of the git server (default is github.com)
+        category: str, optional
+            If specified, scan the repo using all the rules of this category,
+            otherwise use all the rules in the db
+        models: list, optional
+            A list of models for the ML false positives detection
+        force: bool, default `False`
+            Force a complete re-scan of the repository, in case it has already
+            been scanned previously
+        debug: bool, default `False`
+            Flag used to decide whether to visualize the progressbars during
+            the scan (e.g., during the insertion of the detections in the db)
+        similarity: bool, default `False`
+            Decide whether to build the embedding model and to compute and add
+            embeddings, to allow for updating of similar discoveries
+        git_token: str, optional
+            Git personal access token to authenticate to the git server
+
+        Returns
+        -------
+        list
+            The id of the discoveries detected by the scanner (excluded the
+            ones classified as false positives).
+        """
+        # Trim the tail of the repo's url by removing '/' and '.git'
+        if repo_url.endswith('/'):
+            repo_url = repo_url[:-1]
+        if repo_url.endswith('.git'):
+            repo_url = repo_url[:-4]
+
+        if self.get_repo(repo_url):
+            logger.info(f'The repository \"{repo_url}\" has already been '
+                        'scanned.')
+            if force:
+                logger.info(f'The pull request {pr_number} will be scanned, '
+                            'and the old discoveries will be deleted) due to '
+                            'force=True')
+            else:
+                logger.info('Impossible to scan this pull request. Consider '
+                            'relaunching the scan with force=True')
+                return []
+
+        rules = self._get_scan_rules(category)
+        scanner = GitPRScanner(rules)
+
+        return self._scan(
+            repo_url=repo_url, scanner=scanner, models=models, force=force,
+            debug=debug, similarity=similarity,
+            pr_number=pr_number, git_token=git_token)
+
     def scan_user(self, username, category=None, models=None, debug=False,
                   forks=False, similarity=False, git_token=None,
                   api_endpoint='https://api.github.com'):
@@ -1063,7 +1132,7 @@ def _scan(self, repo_url, scanner, models=None, force=False, debug=False,
         if debug:
             logger.setLevel(level=logging.DEBUG)
 
-        if models is None:
+        if not models:
             logger.debug('Don\'t use ML models')
             models = []
 

credentialdigger/client_postgres.py

@@ -549,9 +549,11 @@ def update_discoveries(self, discoveries_ids, new_state):
         return super().update_discoveries(
             discoveries_ids=discoveries_ids,
             new_state=new_state,
-            query='UPDATE discoveries SET state=%s WHERE id IN %s RETURNING true')
+            query='UPDATE discoveries SET state=%s WHERE id IN %s \
+                    RETURNING true')
 
-    def update_discovery_group(self, new_state, repo_url, file_name, snippet=None):
+    def update_discovery_group(self, new_state, repo_url, file_name,
+                               snippet=None):
         """ Change the state of a group of discoveries.
 
         A group of discoveries is identified by the url of their repository,

credentialdigger/client_sqlite.py

@@ -58,7 +58,7 @@ def __init__(self, path):
             );
 
             PRAGMA foreign_keys=ON;
-        """)
+        """)  # noqa: E501
         cursor.close()
         self.db.commit()
 
@@ -205,9 +205,9 @@ def add_embedding(self, discovery_id, repo_url, embedding=None):
         query = 'INSERT INTO embeddings (id, snippet, embedding, repo_url) \
                 VALUES (?, ?, ?, ?);'
         super().add_embedding(query,
-                                     discovery_id,
-                                     repo_url,
-                                     embedding)
+                              discovery_id,
+                              repo_url,
+                              embedding)
 
     def add_embeddings(self, repo_url):
         """ Bulk add embeddings.
@@ -261,7 +261,8 @@ def add_rule(self, regex, category, description=''):
             regex=regex,
             category=category,
             description=description,
-            query='INSERT INTO rules (regex, category, description) VALUES (?, ?, ?)'
+            query='INSERT INTO rules (regex, category, description) \
+                    VALUES (?, ?, ?)'
         )
 
     def delete_rule(self, ruleid):
@@ -588,7 +589,8 @@ def update_discoveries(self, discoveries_ids, new_state):
             query='UPDATE discoveries SET state=? WHERE id IN('
                   f'VALUES {", ".join(["?"]*len(discoveries_ids))})')
 
-    def update_discovery_group(self, new_state, repo_url, file_name, snippet=None):
+    def update_discovery_group(self, new_state, repo_url, file_name,
+                               snippet=None):
         """ Change the state of a group of discoveries.
 
         A group of discoveries is identified by the url of their repository,