ci: 🎡 try to fix pip-audit

see pypa/pip-audit#84 (comment) in particular
huggingface · Nov 24, 2022 · 18fbd97 · 18fbd97
1 parent 4a48536
commit 18fbd97
Show file tree

Hide file tree

Showing 12 changed files with 25 additions and 2 deletions.
diff --git a/.github/workflows/_quality-python.yml b/.github/workflows/_quality-python.yml
@@ -52,5 +52,9 @@ jobs:
         run: poetry run mypy tests src
       - name: Run bandit
         run: poetry run bandit -r src
+      - name: Run pip-audit (datasets worker)
+        if: ${{ inputs.is-datasets-worker == true }}
+        run: bash -c "poetry run pip-audit -r <(poetry export -f requirements.txt --with dev | sed '/^requests==2.28.1 ;/,+2 d' | sed '/^kenlm @/d' | sed '/^trec-car-tools @/d')"
       - name: Run pip-audit
+        if: ${{ inputs.is-datasets-worker == false }}
         run: bash -c 'poetry run pip-audit -r <(poetry export -f requirements.txt --with dev)'
diff --git a/e2e/Makefile b/e2e/Makefile
@@ -17,6 +17,7 @@ DOCKER_COMPOSE := ../tools/docker-compose-datasets-server-from-remote-images.yml
 DOCKER_IMAGES := ../chart/docker-images.yaml
 
 include ../tools/Python.mk
+include ../tools/PythonAudit.mk
 include ../tools/PythonTest.mk
 include ../tools/DockerRemoteImages.mk
 include ../tools/Docker.mk

diff --git a/jobs/mongodb_migration/Makefile b/jobs/mongodb_migration/Makefile
@@ -8,6 +8,7 @@ export MONGODB_MIGRATION_MONGO_URL := mongodb://localhost:${MONGO_PORT}
 DOCKER_COMPOSE := ../../tools/docker-compose-mongo.yml
 
 include ../../tools/Python.mk
+include ../../tools/PythonAudit.mk
 include ../../tools/PythonTest.mk
 include ../../tools/Docker.mk
 

diff --git a/libs/libcache/Makefile b/libs/libcache/Makefile
@@ -6,5 +6,6 @@ export COMPOSE_PROJECT_NAME := libcache
 DOCKER_COMPOSE := ../../tools/docker-compose-mongo.yml
 
 include ../../tools/Python.mk
+include ../../tools/PythonAudit.mk
 include ../../tools/PythonTest.mk
 include ../../tools/Docker.mk
diff --git a/libs/libcommon/Makefile b/libs/libcommon/Makefile
@@ -4,5 +4,6 @@ export COMPOSE_PROJECT_NAME := libcommon
 DOCKER_COMPOSE := ../../tools/docker-compose-empty.yml
 
 include ../../tools/Python.mk
+include ../../tools/PythonAudit.mk
 include ../../tools/PythonTest.mk
 include ../../tools/Docker.mk
diff --git a/libs/libqueue/Makefile b/libs/libqueue/Makefile
@@ -6,5 +6,6 @@ export QUEUE_MONGO_URL := mongodb://localhost:${MONGO_PORT}
 DOCKER_COMPOSE := ../../tools/docker-compose-mongo.yml
 
 include ../../tools/Python.mk
+include ../../tools/PythonAudit.mk
 include ../../tools/PythonTest.mk
 include ../../tools/Docker.mk
diff --git a/services/admin/Makefile b/services/admin/Makefile
@@ -7,6 +7,7 @@ export QUEUE_MONGO_URL := mongodb://localhost:${MONGO_PORT}
 DOCKER_COMPOSE := ../../tools/docker-compose-mongo.yml
 
 include ../../tools/Python.mk
+include ../../tools/PythonAudit.mk
 #include ../../tools/PythonTest.mk
 include ../../tools/Docker.mk
 

diff --git a/services/api/Makefile b/services/api/Makefile
@@ -6,8 +6,8 @@ export QUEUE_MONGO_URL := mongodb://localhost:${MONGO_PORT}
 # makefile variables
 DOCKER_COMPOSE := ../../tools/docker-compose-mongo.yml
 
-# Ensure to specify HF_TOKEN when calling make test, ie HF_TOKEN=hf_app_xxx make test
 include ../../tools/Python.mk
+include ../../tools/PythonAudit.mk
 #include ../../tools/PythonTest.mk
 include ../../tools/Docker.mk
 

diff --git a/tools/Python.mk b/tools/Python.mk
@@ -22,7 +22,7 @@ quality:
 	poetry run flake8 tests src
 	poetry run mypy tests src
 	poetry run bandit -r src
-	bash -c 'poetry run pip-audit -r <(poetry export -f requirements.txt --with dev)'
+	$(MAKE) pip-audit
 
 # Format source code automatically
 .PHONY: style

diff --git a/tools/PythonAudit.mk b/tools/PythonAudit.mk
@@ -0,0 +1,3 @@
+.PHONY: pip-audit
+pip-audit:
+	bash -c 'poetry run pip-audit -r <(poetry export -f requirements.txt --with dev)'
diff --git a/workers/first_rows/Makefile b/workers/first_rows/Makefile
@@ -13,3 +13,8 @@ include ../../tools/Docker.mk
 .PHONY: run
 run:
 	poetry run python src/first_rows/main.py
+
+.PHONY: pip-audit
+pip-audit:
+	bash -c "poetry run pip-audit -r <(poetry export -f requirements.txt --with dev | sed '/^requests==2.28.1 ;/,+2 d' | sed '/^kenlm @/d' | sed '/^trec-car-tools @/d' | sed '/^libcache @/,+1 d' | sed '/^libcommon @/,+1 d' | sed '/^libqueue @/,+1 d')"
+# ^ we remove problematic lines to have a working pip-audit. See https://github.com/pypa/pip-audit/issues/84#issuecomment-1326203111 for "requests"
diff --git a/workers/splits/Makefile b/workers/splits/Makefile
@@ -13,3 +13,8 @@ include ../../tools/Docker.mk
 .PHONY: run
 run:
 	poetry run python src/splits/main.py
+
+.PHONY: pip-audit
+pip-audit:
+	bash -c "poetry run pip-audit -r <(poetry export -f requirements.txt --with dev | sed '/^requests==2.28.1 ;/,+2 d' | sed '/^kenlm @/d' | sed '/^trec-car-tools @/d')"
+# ^ we remove problematic lines to have a working pip-audit. See https://github.com/pypa/pip-audit/issues/84#issuecomment-1326203111 for "requests"