From 1c0f0649ec82178651a9ccf7d02a2658af1182fd Mon Sep 17 00:00:00 2001 From: Marco Rosa Date: Mon, 6 Sep 2021 14:26:26 +0200 Subject: [PATCH 1/6] give names to docker images --- docker-compose.postgres.yml | 1 + docker-compose.yml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.postgres.yml b/docker-compose.postgres.yml index 76f19ad1..81e05aab 100644 --- a/docker-compose.postgres.yml +++ b/docker-compose.postgres.yml @@ -21,6 +21,7 @@ services: - path_model=$path_model - snippet_model=$snippet_model container_name: credential_digger_backend + image: credential-digger_pg restart: always env_file: - ./.env diff --git a/docker-compose.yml b/docker-compose.yml index 98afc695..f3057c55 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,8 +10,9 @@ services: - SSL_certificate=$SSL_certificate - SSL_private_key=$SSL_private_key container_name: credential_digger_sqlite + image: credential-digger_sqlite restart: always env_file: - ./.env ports: - - "5000:5000" \ No newline at end of file + - "5000:5000" From 55bdf780664dd84ae4800bd17a77b5d364124d19 Mon Sep 17 00:00:00 2001 From: Marco Rosa Date: Mon, 6 Sep 2021 14:27:14 +0200 Subject: [PATCH 2/6] upgrade tf deps --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index efea928e..46564d33 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ rich scikit-learn srsly>=2.4.0 tensorflow==2.5.1; python_version >= "3.8" -tensorflow==2.4.2; python_version < "3.8" +tensorflow==2.4.*; python_version < "3.8" tensorflow-text==2.5.0; python_version >= "3.8" -tensorflow-text==2.4.2; python_version < "3.8" -tf-models-official \ No newline at end of file +tensorflow-text==2.4.*; python_version < "3.8" +tf-models-official From 2c460d1652416900cde4176c3c3c35717f9dfa8e Mon Sep 17 00:00:00 2001 From: Marco Rosa Date: Mon, 6 Sep 2021 14:27:31 +0200 Subject: [PATCH 3/6] change base docker image --- ui/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/Dockerfile b/ui/Dockerfile index 0ecde17b..11c042df 100644 --- a/ui/Dockerfile +++ b/ui/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7 +FROM python:3.9-bullseye RUN pip install flask_jwt_extended Flask python-dotenv RUN apt-get update && apt-get install -y libhyperscan5 libpq-dev gunicorn3 From 1720d755edfcbaa78048f484b857952317d06904 Mon Sep 17 00:00:00 2001 From: Marco Rosa Date: Mon, 6 Sep 2021 16:17:02 +0200 Subject: [PATCH 4/6] fix embeddings queries --- credentialdigger/client.py | 2 +- credentialdigger/client_postgres.py | 8 ++++---- credentialdigger/client_sqlite.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/credentialdigger/client.py b/credentialdigger/client.py index c375bed8..0e45fb2f 100644 --- a/credentialdigger/client.py +++ b/credentialdigger/client.py @@ -145,8 +145,8 @@ def add_embedding(self, query, discovery_id, repo_url, embedding=None): cursor = self.db.cursor() try: cursor.execute(query, (discovery_id, - embedding, snippet, + embedding, repo_url)) self.db.commit() except self.Error: diff --git a/credentialdigger/client_postgres.py b/credentialdigger/client_postgres.py index cfd56820..3b83d9b7 100644 --- a/credentialdigger/client_postgres.py +++ b/credentialdigger/client_postgres.py @@ -208,7 +208,7 @@ def add_embedding(self, discovery_id, repo_url, embedding=None): embedding: list The embedding being added """ - query = 'INSERT INTO embeddings (id, embedding, snippet, repo_url) \ + query = 'INSERT INTO embeddings (id, snippet, embedding, repo_url) \ VALUES (%s, %s, %s, %s);' super().add_embedding(query, discovery_id, @@ -223,7 +223,7 @@ def add_embeddings(self, repo_url): repo_url: str The discoveries' repository url """ - query = 'INSERT INTO embeddings (id, embedding, snippet, repo_url) \ + query = 'INSERT INTO embeddings (id, snippet, embedding, repo_url) \ VALUES (%s, %s, %s, %s);' super().add_embeddings(query, repo_url) @@ -466,9 +466,9 @@ def get_embedding(self, discovery_id=None, snippet=None): snippet or id """ if discovery_id: - query = 'SELECT embedding FROM embeddings WHERE id=%s' + query = 'SELECT embedding FROM embeddings WHERE id=%s;' elif snippet: - query = 'SELECT embedding FROM embeddings WHERE snippet=%s' + query = 'SELECT embedding FROM embeddings WHERE snippet=%s;' else: return None return super().get_embedding(query=query, diff --git a/credentialdigger/client_sqlite.py b/credentialdigger/client_sqlite.py index 959d91e9..7c49c52d 100644 --- a/credentialdigger/client_sqlite.py +++ b/credentialdigger/client_sqlite.py @@ -202,7 +202,7 @@ def add_embedding(self, discovery_id, repo_url, embedding=None): repo_url: str The discovery's repository url """ - query = 'INSERT INTO embeddings (id, embedding, snippet, repo_url) \ + query = 'INSERT INTO embeddings (id, snippet, embedding, repo_url) \ VALUES (?, ?, ?, ?);' return super().add_embedding(query, discovery_id, From bb541398320a75500f6c0938fa6c50e0806f6164 Mon Sep 17 00:00:00 2001 From: Marco Rosa Date: Mon, 6 Sep 2021 16:34:50 +0200 Subject: [PATCH 5/6] fix embeddings table --- sql/create_table.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/create_table.sql b/sql/create_table.sql index c28473ca..e839476d 100644 --- a/sql/create_table.sql +++ b/sql/create_table.sql @@ -31,8 +31,8 @@ CREATE TABLE discoveries ( CREATE TABLE embeddings ( id INTEGER REFERENCES discoveries, - embedding TEXT, snippet TEXT, + embedding TEXT, repo_url TEXT REFERENCES repos, PRIMARY KEY (id) ); From 05c19ceb7ea091b2d592d71906c723b7cf0c8ee9 Mon Sep 17 00:00:00 2001 From: Marco Rosa Date: Mon, 6 Sep 2021 17:12:29 +0200 Subject: [PATCH 6/6] upgrade version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d35368bb..04e1893d 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ def requirements(): setuptools.setup( name='credentialdigger', - version='4.1.0', + version='4.2.0', author='SAP SE', maintainer='Marco Rosa, Slim Trabelsi', maintainer_email='marco.rosa@sap.com, slim.trabelsi@sap.com',