Merge pull request #218 from SAP/develop

upgrade to v4.6.0

.gitignore

@@ -118,3 +118,7 @@ config.py
 # IDE
 .vscode
 workbench.py
+
+# DBs
+data.db
+data.db.*

README.md

@@ -133,6 +133,8 @@ sudo docker-compose up --build
 
 The UI is available at [http://localhost:5000/](http://localhost:5000/)
 
+> It is preferrable to have at least 8 GB of RAM free when using docker containers
+
 
 ## Advanced Installation
 

credentialdigger/client.py

@@ -768,7 +768,9 @@ def scan(self, repo_url, category=None, models=None, force=False,
             If True, get the repository from a local directory instead of the
             web
         git_username: str, optional
-            the username of the user to authenticate to the git server
+            the username of the user to authenticate to the git server. While
+            it is not needed for `github.com` and github enterprise, it is
+            needed for some private git instances and bitbucket
         git_token: str, optional
             Git personal access token to authenticate to the git server
 
@@ -821,7 +823,9 @@ def scan_snapshot(self, repo_url, branch_or_commit, category=None,
             Flag used to decide whether to visualize the progressbars during
             the scan (e.g., during the insertion of the detections in the db)
         git_username: str, optional
-            the username of the user to authenticate to the git server
+            the username of the user to authenticate to the git server. While
+            it is not needed for `github.com` and github enterprise, it is
+            needed for some private git instances and bitbucket
         git_token: str, optional
             Git personal access token to authenticate to the git server
         max_depth: int, optional

credentialdigger/scanners/git_file_scanner.py

@@ -68,7 +68,9 @@ def scan(self, repo_url, branch_or_commit, max_depth=-1, ignore_list=[],
             names, directory names, or whole paths. Wildcards are supported as
             per the fnmatch package.
         git_username: str, optional
-            The username of the user to authenticate to the git server
+            the username of the user to authenticate to the git server. While
+            it is not needed for `github.com` and github enterprise, it is
+            needed for some private git instances and bitbucket
         git_token: str, optional
             Git personal access token to authenticate to the git server
         debug: bool, optional

credentialdigger/scanners/git_scanner.py

@@ -149,7 +149,9 @@ def get_commit_timestamp(self, repo_url, branch_or_commit,
         branch_or_commit: str
             The branch name or commit id of the repo
         git_username: str, optional
-            The username of the user to authenticate to the git server
+            the username of the user to authenticate to the git server. While
+            it is not needed for `github.com` and github enterprise, it is
+            needed for some private git instances and bitbucket
         git_token: str, optional
             The personal user access token to access to this repo (needed for
             private repos)
@@ -195,7 +197,9 @@ def scan(self, repo_url, since_timestamp=0, max_depth=1000000,
         max_depth: int, optional
             The maximum number of commits to scan
         git_username: str, optional
-            The username of the user to authenticate to the git server
+            the username of the user to authenticate to the git server. While
+            it is not needed for `github.com` and github enterprise, it is
+            needed for some private git instances and bitbucket
         git_token: str, optional
             Git personal access token to authenticate to the git server
         local_repo: bool, optional

docker-compose.postgres.yml

@@ -2,6 +2,7 @@ version: "3"
 services:
   postgres:
     image: postgres:12-alpine
+    shm_size: '1gb'
     restart: always
     container_name: postgres
     volumes:
@@ -29,5 +30,11 @@ services:
         - USE_PG=True
     ports:
       - "5000:5000"
+    deploy:
+      resources:
+        limits:
+          memory: 16G
+        reservations:
+          memory: 6G
 volumes:
   creddig_db:

setup.py

@@ -13,7 +13,7 @@ def requirements():
 
 setuptools.setup(
     name='credentialdigger',
-    version='4.5.2',
+    version='4.6.0',
     author='SAP SE',
     maintainer='Marco Rosa, Slim Trabelsi',
     maintainer_email='[email protected], [email protected]',

ui/backend/client_ui.py

@@ -6,10 +6,6 @@
 
 import git
 from credentialdigger import Client
-from credentialdigger.snippet_similarity import (
-    build_embedding_model,
-    compute_similarity,
-    compute_snippet_embedding)
 from git import GitCommandError, InvalidGitRepositoryError, NoSuchPathError
 from git import Repo as GitRepo
 
@@ -107,15 +103,19 @@ def get_files_summary(self, query, repo_url):
             result = cursor.fetchone()
         return files
 
-    def check_repo(self, repo_url, git_token=None, local_repo=False,
-                   branch_or_commit=None):
+    def check_repo(self, repo_url, git_username=None, git_token=None,
+                   local_repo=False, branch_or_commit=None):
         """ Check git token validity for the repository.
 
         Parameters
         ----------
         repo_url: str
             The location of a git repository (an url if local_repo is False, a
             local path otherwise)
+        git_username: str, optional
+            Git username to authenticate to the git server. It is needed only
+            for some private git instances and bitbucket (`github.com` and
+            github enterprise do not require this field)
         git_token: str, optional
             Git personal access token to authenticate to the git server
         local_repo: bool, optional
@@ -140,9 +140,10 @@ def check_repo(self, repo_url, git_token=None, local_repo=False,
                 return False, 'NoSuchPathError'
         else:
             g = git.cmd.Git()
-            if git_token is not None and len(git_token) > 0:
+            if git_token:
+                username = git_username or 'oauth2'
                 repo_url = repo_url.replace('https://',
-                                            f'https://oauth2:{git_token}@')
+                                            f'https://{username}:{git_token}@')
             try:
                 remote_refs = g.ls_remote(repo_url)
                 if branch_or_commit and branch_or_commit not in remote_refs:

ui/server.py

@@ -271,15 +271,17 @@ def scan_repo():
     # If the form does not contain the 'Force' checkbox,
     # then 'forceScan' will be set to False; thus, ignored.
     force_scan = request.form.get('forceScan') == 'force'
+    git_username = request.form.get('gitUsername')
     git_token = request.form.get('gitToken')
     snapshot = request.form.get('repoSnapshot')
     local_repo = not (repo_link.startswith('http://')
                       or repo_link.startswith('https://'))
 
     url_is_valid, err_code = c.check_repo(
-        repo_link, git_token, local_repo, snapshot)
+        repo_link, git_username, git_token, local_repo, snapshot)
     if not url_is_valid:
         return err_code, 401
+    app.logger.debug('Repo has been verified')
 
     # Set up models
     models = []
@@ -294,11 +296,13 @@ def scan_repo():
         'repo_url': repo_link,
         'models': models,
         'force': force_scan,
+        'git_username': git_username,
         'git_token': git_token,
         'local_repo': local_repo,
         'similarity': True
     }
     if rules_to_use != 'all':
+        app.logger.debug(f'Use rules only from {rules_to_use} category')
         args['category'] = rules_to_use
     if snapshot:
         args['branch_or_commit'] = snapshot
@@ -358,23 +362,22 @@ def get_repos():
 def export_discoveries_csv():
     """ Export the discoveries of a repo in a csv file. """
     url = request.form.get('repo_url')
-    _, discoveries = c.get_discoveries(url)
 
-    states = []
     if request.form.get('checkAll') == 'all':
-        states = ['new', 'false_positive',
-                  'addressing', 'not_relevant', 'fixed']
+        app.logger.debug('Export all the discoveries')
+        _, discoveries = c.get_discoveries(url)
     else:
         states = request.form.getlist('check')
-
-    filtered_discoveries = list(
-        filter(lambda d: d.get('state') in states, discoveries))
+        app.logger.debug(f'Export discoveries of states {states}')
+        discoveries = []
+        for s in states:
+            discoveries.extend(c.get_discoveries(url, state_filter=s)[1])
 
     try:
         string_io = io.StringIO()
         csv_writer = csv.DictWriter(string_io, discoveries[0].keys())
         csv_writer.writeheader()
-        csv_writer.writerows(filtered_discoveries)
+        csv_writer.writerows(discoveries)
         response_csv = make_response(string_io.getvalue())
         report_name = f'report-{url.split("/")[-1]}.csv'
         response_csv.headers['Content-Disposition'] = f'attachment; \

ui/templates/rules.html

@@ -98,11 +98,6 @@ <h1 class="topicH1">Add rule</h1>
         <input id="regexInput" type="text" name="regex" class="textInput" placeholder="Regex">
       </div>
 
-      <!-- category field -->
-      <div class="form-item">
-        <input id="catInput" type="text" name="category" class="textInput" placeholder="category">
-      </div>
-
       <!-- description field -->
       <div class="form-item">
         <textarea  id="descInput" type="text" name="description" class="textInput" placeholder="Description"></textarea>

ui/templates/shared/_scanRepoModal.html

@@ -34,6 +34,15 @@ <h1 class="topicH1" id="title"></h1>
           >
         </div>
       </div>
+      <div class="form-item">
+        <input
+          id="gitUsernameInput"
+          type="text"
+          name="gitUsername"
+          class="textInput"
+          placeholder="Git username (to authenticate to private git servers, OPTIONAL)"
+        />
+      </div>
       <div class="form-item">
         <input
           id="gitTokenInput"