diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..b2509caa --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,107 @@ +name: Publish the package on pypi + +on: + pull_request: + types: + - closed + branches: + - main + +jobs: + check_version_update: + if: github.event.pull_request.merged == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Check pypi version + uses: maybe-hello-world/pyproject-check-version@v4 + id: versioncheck + with: + pyproject-path: "./pyproject.toml" + + - name: Check if local version has been updated 🕵️ + run: | + if ${{ steps.versioncheck.outputs.local_version_is_higher }}; then + echo "Version bumped." + else + echo "Local version must be greater than public one." + echo "Please bump package version." + exit 1 + fi + outputs: + new_pkg_version: ${{ steps.versioncheck.outputs.local_version }} + + build: + name: Build distribution 📦 + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python 🚧 + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Install pypa/build 🚧 + run: >- + python3 -m + pip install + build + --user + - name: Build a binary wheel and a source tarball 🛠️ + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v3 + with: + name: python-package-distributions + path: dist/ + + publish-to-pypi: + name: Publish Python 🐍 distribution 📦 to PyPI + #if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + if: github.event.pull_request.merged == 'true' # publish on PR merge + needs: + - check_version_update # Only publish if there is a version update + - build # Only publish if the package was built + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/credentialdigger + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + skip-existing: true + + docker: + name: Push Docker image to Docker Hub 🐳 + if: github.event.pull_request.merged == 'true' + needs: + - check_version_update + - publish-to-pypi + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v4 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push Docker 🐳 image + uses: docker/build-push-action@v5 + with: + context: . + file: ./ui/Dockerfile + push: true + tags: saposs/credentialdigger:${{ needs.check_version_update.outputs.new_pkg_version }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f92946d8..beb2f25c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,8 +16,8 @@ jobs: postgres: image: postgres:alpine env: - POSTGRES_USER: ${{ secrets.POSTGRES_USER }} POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} POSTGRES_DB: credential_digger_tests DBHOST: localhost DBPORT: 5432 @@ -30,14 +30,14 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Create PostgreSQL database run: | PGPASSWORD=${{ secrets.POSTGRES_PASSWORD }} psql -U ${{ secrets.POSTGRES_USER }} -h 127.0.0.1 -p 5432 -d credential_digger_tests -f sql/create_table.sql - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -46,7 +46,7 @@ jobs: sudo apt install -y build-essential python3-dev - name: Cache python dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/pip # This path is specific to Ubuntu key: ${{ runner.os }}-pip-${{ hashFiles('./requirements.txt') }}-${{ hashFiles('./tests/tests-requirements.txt') }} diff --git a/README.md b/README.md index 9aa27deb..15d9dfc7 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ [![REUSE status](https://api.reuse.software/badge/github.com/SAP/credential-digger)](https://api.reuse.software/info/github.com/SAP/credential-digger) ![GitHub release (latest by date)](https://img.shields.io/github/v/release/SAP/credential-digger?logo=github) ![PyPI](https://img.shields.io/pypi/v/credentialdigger?logo=pypi) -![PyPI - Python Version](https://img.shields.io/pypi/pyversions/credentialdigger) +![PyPI - Python Version](https://img.shields.io/pypi/pyversions/credentialdigger?logo=python) +[![Docker](https://badgen.net/badge/icon/docker?icon=docker&label&color=0db7ed)](https://hub.docker.com/r/saposs/credentialdigger) +[![Visual Studio Plugin](https://badgen.net/badge/icon/visualstudio?icon=visualstudio&label)](https://marketplace.visualstudio.com/items?itemName=SAPOSS.vs-code-extension-for-project-credential-digger) ![Logo](https://raw.githubusercontent.com/SAP/credential-digger/main/github_assets/Logo-CD-Mint_48.png) @@ -67,10 +69,9 @@ For the complete description of the approach of Credential Digger (versions <4.4 ## Requirements -Credential Digger supports Python >= 3.8 and < 3.12, and works only with Linux and MacOS systems. +Credential Digger supports Python >= 3.8 and < 3.13, and works only with Linux and MacOS systems. In case you don't meet these requirements, you may consider running a [Docker container](#docker) (that also includes a user interface). - ## Download and Installation First, you need to install some dependencies (namely, `build-essential` and `python3-dev`). No need to explicitely install hyperscan anymore. @@ -85,6 +86,9 @@ Then, you can install Credential Digger module using `pip`. pip install credentialdigger ``` +> For ARM machines (e.g., new MacBooks), installation is possible [following this guide](https://github.com/SAP/credential-digger/wiki/MacOS-ARM-Installation) + + ## How to run ### Add rules diff --git a/docker-compose.postgres.yml b/docker-compose.postgres.yml index 26a6e721..82df44bc 100644 --- a/docker-compose.postgres.yml +++ b/docker-compose.postgres.yml @@ -15,6 +15,7 @@ services: ports: - "5432:${DBPORT}" credential_digger: + platform: linux/amd64 build: context: . dockerfile: ./ui/Dockerfile diff --git a/pyproject.toml b/pyproject.toml index b8958ece..92746b82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,8 @@ build-backend = 'setuptools.build_meta' [project] name = 'credentialdigger' -dynamic = ['dependencies', 'version'] +version = '4.14.0' +dynamic = ['dependencies'] authors = [ {name = 'SAP SE'} ] @@ -15,13 +16,14 @@ maintainers = [ description = 'Credential Digger' readme = 'README.md' license = {file = 'LICENSE'} -requires-python = '>= 3.8, < 3.12' +requires-python = '>= 3.8, < 3.13' classifiers = [ 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Operating System :: OS Independent', 'Topic :: Security', ] @@ -31,7 +33,6 @@ credentialdigger = 'credentialdigger.__main__:main' [tool.setuptools.dynamic] dependencies = {file = ['requirements.txt']} -version = {file = ['version.py']} [project.urls] Homepage = 'https://github.com/sap/credential-digger' diff --git a/requirements.txt b/requirements.txt index 973d694e..e054a74f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ Flask flask_jwt_extended GitPython -hyperscan==0.6.0; python_version != "3.9" -hyperscan==0.5.0; python_version == "3.9" +hyperscan==0.7.7; python_version > "3.8" +hyperscan==0.7.0; python_version <= "3.8" numpy pandas psycopg2-binary @@ -11,11 +11,9 @@ python-dotenv pyyaml rich~=13.7 srsly>=2.4.0 -tensorflow==2.15.0; python_version > "3.8" +tensorflow==2.16.1; python_version > "3.8" tensorflow==2.13.1; python_version <= "3.8" -tensorflow-estimator==2.15.0; python_version > "3.8" -tensorflow-estimator==2.13.0; python_version <= "3.8" -tensorflow-text==2.15.0; python_version > "3.8" +tensorflow-text==2.16.1; python_version > "3.8" tensorflow-text==2.13.0; python_version <= "3.8" tf-models-official transformers diff --git a/src/credentialdigger/__init__.py b/src/credentialdigger/__init__.py index 301c57ab..60a107b9 100644 --- a/src/credentialdigger/__init__.py +++ b/src/credentialdigger/__init__.py @@ -1,3 +1,8 @@ +import importlib.metadata + from .client import Client from .client_postgres import PgClient from .client_sqlite import SqliteClient + + +__version__ = importlib.metadata.version('credentialdigger') diff --git a/src/credentialdigger/cli/scan_user.py b/src/credentialdigger/cli/scan_user.py index f09d809b..77caa06a 100644 --- a/src/credentialdigger/cli/scan_user.py +++ b/src/credentialdigger/cli/scan_user.py @@ -15,7 +15,7 @@ username positional arguments: - username The username as on github.com + username The username or organization name as on github.com optional arguments: -h, --help show this help message and exit diff --git a/src/credentialdigger/client.py b/src/credentialdigger/client.py index d8f03b9d..6eb62de3 100644 --- a/src/credentialdigger/client.py +++ b/src/credentialdigger/client.py @@ -991,16 +991,19 @@ def scan_pull_request(self, repo_url, pr_number, def scan_user(self, username, category=None, models=None, debug=False, forks=False, similarity=False, git_token=None, api_endpoint='https://api.github.com'): - """ Scan all the repositories of a user. + """ Scan all the repositories of a user or of an organization. Find all the repositories of a user, and scan them. Please note that git limits the list of repositories to maximum 100 (due to pagination). + It is possible to enter the name of an organization instead of a user, + and the scan will work on all the repos of that org. + Parameters ---------- username: str - The username as on github.com + The username (or organization name) as on github.com category: str, optional If specified, scan the repo using all the rules of this category, otherwise use all the rules in the db diff --git a/tests/tests-requirements.txt b/tests/tests-requirements.txt index 2b2a282c..5eb06229 100644 --- a/tests/tests-requirements.txt +++ b/tests/tests-requirements.txt @@ -1,2 +1,2 @@ parameterized~=0.9.0 -pytest~=7.4.3 \ No newline at end of file +pytest~=8.1.1 \ No newline at end of file diff --git a/ui/Dockerfile b/ui/Dockerfile index a8fd155e..2fb2d2c0 100644 --- a/ui/Dockerfile +++ b/ui/Dockerfile @@ -15,7 +15,7 @@ RUN git config --global http.sslverify false RUN git config --global core.autocrlf false # Install Credential Digger -RUN pip install credentialdigger +RUN pip install credentialdigger==4.13.0 # Keys args (optional, for HTTPS) ARG SSL_certificate diff --git a/version.py b/version.py deleted file mode 100644 index 813b83b6..00000000 --- a/version.py +++ /dev/null @@ -1 +0,0 @@ -4.13.0