From 26f81df62f3f7ffe0153b94c222ca362b8852af2 Mon Sep 17 00:00:00 2001 From: Ulu Kagan Boelek Date: Fri, 23 Aug 2024 15:00:18 +0200 Subject: [PATCH 1/7] code migrated under gcs api service --- .../happtiq_commons_google_cloud/gcs_api_service.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/happtiq_commons_google_cloud/src/happtiq_commons_google_cloud/gcs_api_service.py b/packages/happtiq_commons_google_cloud/src/happtiq_commons_google_cloud/gcs_api_service.py index 33f66c9..3bfd75d 100644 --- a/packages/happtiq_commons_google_cloud/src/happtiq_commons_google_cloud/gcs_api_service.py +++ b/packages/happtiq_commons_google_cloud/src/happtiq_commons_google_cloud/gcs_api_service.py @@ -37,3 +37,13 @@ def upload_file(self, file_path: bytes, bucket_name: str, destination: str, con blob.upload_from_filename(file_path, content_type=content_type, timeout=self.timeout) self.logger.info(f"File uploaded to {destination}.") return f"gs://{bucket_name}/{destination}" + + def list_files_with_prefix(self, bucket_name: str, prefix: str): + self.logger.info(f"Listing files in bucket {bucket_name} with prefix {prefix}") + bucket = self.client.bucket(bucket_name) + blobs = bucket.list_blobs(prefix=prefix) + + file_list = [blob.name for blob in blobs] + self.logger.info(f"Files in bucket {bucket_name} with prefix {prefix}: {file_list}") + + return file_list From ec3188dbac2a4d4b1dd23b231a20ac14c9a72305 Mon Sep 17 00:00:00 2001 From: Ulu Kagan Boelek Date: Mon, 26 Aug 2024 15:30:31 +0200 Subject: [PATCH 2/7] debug log added --- .../src/happtiq_commons_google_cloud/gcs_api_service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/happtiq_commons_google_cloud/src/happtiq_commons_google_cloud/gcs_api_service.py b/packages/happtiq_commons_google_cloud/src/happtiq_commons_google_cloud/gcs_api_service.py index 3bfd75d..fa89619 100644 --- a/packages/happtiq_commons_google_cloud/src/happtiq_commons_google_cloud/gcs_api_service.py +++ b/packages/happtiq_commons_google_cloud/src/happtiq_commons_google_cloud/gcs_api_service.py @@ -39,7 +39,8 @@ def upload_file(self, file_path: bytes, bucket_name: str, destination: str, con return f"gs://{bucket_name}/{destination}" def list_files_with_prefix(self, bucket_name: str, prefix: str): - self.logger.info(f"Listing files in bucket {bucket_name} with prefix {prefix}") + self.logger.debug(f"Files in bucket {bucket_name} with prefix {prefix}") + self.logger.info(f"found {len(file_list)} files in bucket {bucket_name} with prefix {prefix}: {file_list}") bucket = self.client.bucket(bucket_name) blobs = bucket.list_blobs(prefix=prefix) From 41ed9ee408d2ec03d3826f7d9b31a6f3cad4fe8d Mon Sep 17 00:00:00 2001 From: Ulu Kagan Boelek Date: Mon, 26 Aug 2024 15:31:02 +0200 Subject: [PATCH 3/7] testing for list files with prefix added --- .../gcs_service_test.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packages/happtiq_commons_google_cloud/tests/happtiq_commons_google_cloud/gcs_service_test.py b/packages/happtiq_commons_google_cloud/tests/happtiq_commons_google_cloud/gcs_service_test.py index 20340bd..3c2841b 100644 --- a/packages/happtiq_commons_google_cloud/tests/happtiq_commons_google_cloud/gcs_service_test.py +++ b/packages/happtiq_commons_google_cloud/tests/happtiq_commons_google_cloud/gcs_service_test.py @@ -74,3 +74,21 @@ def test_upload_file_gzipped(gcs_api_service, monkeypatch): gcs_api_service.client.bucket().blob.assert_called_once_with(destination) gcs_api_service.client.bucket().blob(destination).upload_from_filename.assert_called_once_with(file_path, content_type=content_type, timeout=30) gcs_api_service.client.bucket().blob(destination).content_encoding = "gzip" + +def test_list_files_with_prefix(gcs_api_service, monkeypatch): + mock_client = MagicMock() + mock_bucket = MagicMock() + mock_blobs = [MagicMock(name="file1.txt"), MagicMock(name="file2.txt")] + + mock_client.bucket.return_value = mock_bucket + mock_bucket.list_blobs.return_value = mock_blobs + monkeypatch.setattr(gcs_api_service, "client", mock_client) + + bucket_name = "test-bucket" + prefix = "test-prefix/" + + result = gcs_api_service.list_files_with_prefix(bucket_name, prefix) + + assert result == ["file1.txt", "file2.txt"] + mock_client.bucket.assert_called_once_with(bucket_name) + mock_bucket.list_blobs.assert_called_once_with(prefix=prefix) From 8232e17ae0684891d690c186d87cb633c0ebceba Mon Sep 17 00:00:00 2001 From: Ulu Kagan Boelek Date: Tue, 27 Aug 2024 13:41:54 +0200 Subject: [PATCH 4/7] update debug on list files function --- .../src/happtiq_commons_google_cloud/gcs_api_service.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/happtiq_commons_google_cloud/src/happtiq_commons_google_cloud/gcs_api_service.py b/packages/happtiq_commons_google_cloud/src/happtiq_commons_google_cloud/gcs_api_service.py index fa89619..c7403b7 100644 --- a/packages/happtiq_commons_google_cloud/src/happtiq_commons_google_cloud/gcs_api_service.py +++ b/packages/happtiq_commons_google_cloud/src/happtiq_commons_google_cloud/gcs_api_service.py @@ -39,12 +39,12 @@ def upload_file(self, file_path: bytes, bucket_name: str, destination: str, con return f"gs://{bucket_name}/{destination}" def list_files_with_prefix(self, bucket_name: str, prefix: str): - self.logger.debug(f"Files in bucket {bucket_name} with prefix {prefix}") - self.logger.info(f"found {len(file_list)} files in bucket {bucket_name} with prefix {prefix}: {file_list}") + self.logger.info(f"Listing files in bucket {bucket_name} with prefix {prefix}") bucket = self.client.bucket(bucket_name) blobs = bucket.list_blobs(prefix=prefix) file_list = [blob.name for blob in blobs] - self.logger.info(f"Files in bucket {bucket_name} with prefix {prefix}: {file_list}") - + self.logger.debug(f"Files in bucket {bucket_name} with prefix {prefix}: {file_list}") + self.logger.info(f"found {len(file_list)} files in bucket {bucket_name} with prefix {prefix}") + return file_list From 201c6e91b9071e410d7f61067e98f528bc2d3743 Mon Sep 17 00:00:00 2001 From: mab Date: Wed, 28 Aug 2024 10:48:32 +0200 Subject: [PATCH 5/7] fix unit test --- .../tests/happtiq_commons_google_cloud/gcs_service_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/happtiq_commons_google_cloud/tests/happtiq_commons_google_cloud/gcs_service_test.py b/packages/happtiq_commons_google_cloud/tests/happtiq_commons_google_cloud/gcs_service_test.py index 3c2841b..2ef5db4 100644 --- a/packages/happtiq_commons_google_cloud/tests/happtiq_commons_google_cloud/gcs_service_test.py +++ b/packages/happtiq_commons_google_cloud/tests/happtiq_commons_google_cloud/gcs_service_test.py @@ -78,7 +78,11 @@ def test_upload_file_gzipped(gcs_api_service, monkeypatch): def test_list_files_with_prefix(gcs_api_service, monkeypatch): mock_client = MagicMock() mock_bucket = MagicMock() - mock_blobs = [MagicMock(name="file1.txt"), MagicMock(name="file2.txt")] + mock_blob1 = MagicMock() + mock_blob1.name = "file1.txt" + mock_blob2 = MagicMock() + mock_blob2.name = "file2.txt" + mock_blobs = [mock_blob1,mock_blob2] mock_client.bucket.return_value = mock_bucket mock_bucket.list_blobs.return_value = mock_blobs From e49fbacdaca77e96a8f684ef23cd76a2f3825969 Mon Sep 17 00:00:00 2001 From: Ulu Kagan Boelek Date: Wed, 28 Aug 2024 13:58:32 +0200 Subject: [PATCH 6/7] yaml for testpypi updated --- .github/workflows/publish_pypi.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 9080e1c..3a66781 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -85,3 +85,29 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | gh release upload '${{ github.ref_name }}' dist/** --repo '${{ github.repository }}' + + publish-to-testpypi: + name: Publish Python 🐍 distribution 📦 to TestPyPI + if: startsWith(github.ref, 'refs/tags/') + needs: + - build + runs-on: ubuntu-latest + + environment: + name: testpypi + url: https://test.pypi.org/p/${{ matrix.package }} + + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file From 00b11efacbdbf39669763d93e06ec9e3a169eb21 Mon Sep 17 00:00:00 2001 From: Ulu Kagan Boelek Date: Wed, 28 Aug 2024 14:18:42 +0200 Subject: [PATCH 7/7] api token test added --- .github/workflows/publish_pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 3a66781..427d281 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -110,4 +110,4 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ - password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file + password: ${{ secrets.TEST_PYPI_API_TOKEN }} \ No newline at end of file