From 5872585146bee5f70018362d76a2200d8aabd1c9 Mon Sep 17 00:00:00 2001 From: kiraksi Date: Mon, 13 Nov 2023 16:34:21 -0800 Subject: [PATCH 1/8] feat: introduce compatibility with native namespace packages --- google/__init__.py | 24 ----------------------- google/cloud/__init__.py | 24 ----------------------- setup.py | 8 ++------ tests/unit/test_packaging.py | 37 ++++++++++++++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 54 deletions(-) delete mode 100644 google/__init__.py delete mode 100644 google/cloud/__init__.py create mode 100644 tests/unit/test_packaging.py diff --git a/google/__init__.py b/google/__init__.py deleted file mode 100644 index 9a1b64a6..00000000 --- a/google/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -try: - import pkg_resources - - pkg_resources.declare_namespace(__name__) -except ImportError: - import pkgutil - - __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/google/cloud/__init__.py b/google/cloud/__init__.py deleted file mode 100644 index 9a1b64a6..00000000 --- a/google/cloud/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -try: - import pkg_resources - - pkg_resources.declare_namespace(__name__) -except ImportError: - import pkgutil - - __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/setup.py b/setup.py index 2bdaa0f1..cdee34c4 100644 --- a/setup.py +++ b/setup.py @@ -58,14 +58,10 @@ packages = [ package - for package in setuptools.PEP420PackageFinder.find() + for package in setuptools.find_namespace_packages() if package.startswith("google") ] -namespaces = ["google"] -if "google.cloud" in packages: - namespaces.append("google.cloud") - setuptools.setup( name=name, version=version, @@ -92,7 +88,7 @@ platforms="Posix; MacOS X; Windows", packages=packages, python_requires=">=3.7", - namespace_packages=namespaces, + install_requires=dependencies, extras_require=extras, scripts=["scripts/fixup_bigquery_storage_v1_keywords.py"], diff --git a/tests/unit/test_packaging.py b/tests/unit/test_packaging.py new file mode 100644 index 00000000..3bc4a37c --- /dev/null +++ b/tests/unit/test_packaging.py @@ -0,0 +1,37 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + + +def test_namespace_package_compat(tmp_path): + # The ``google`` namespace package should not be masked + # by the presence of ``google-cloud-bigquery-storage``. + google = tmp_path / "google" + google.mkdir() + google.joinpath("othermod.py").write_text("") + env = dict(os.environ, PYTHONPATH=str(tmp_path)) + cmd = [sys.executable, "-m", "google.othermod"] + subprocess.check_call(cmd, env=env, shell=True) + + # The ``google.cloud`` namespace package should not be masked + # by the presence of ``google-cloud-bigquery-storage``. + google_cloud = tmp_path / "google" / "cloud" + google_cloud.mkdir() + google_cloud.joinpath("othermod.py").write_text("") + env = dict(os.environ, PYTHONPATH=str(tmp_path)) + cmd = [sys.executable, "-m", "google.cloud.othermod"] + subprocess.check_call(cmd, env=env, shell=True) \ No newline at end of file From a85ddcac809c731e4745f00bfa1c013990478822 Mon Sep 17 00:00:00 2001 From: kiraksi Date: Mon, 13 Nov 2023 16:39:16 -0800 Subject: [PATCH 2/8] reformatted with black --- setup.py | 1 - tests/unit/test_packaging.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index cdee34c4..e4875f59 100644 --- a/setup.py +++ b/setup.py @@ -88,7 +88,6 @@ platforms="Posix; MacOS X; Windows", packages=packages, python_requires=">=3.7", - install_requires=dependencies, extras_require=extras, scripts=["scripts/fixup_bigquery_storage_v1_keywords.py"], diff --git a/tests/unit/test_packaging.py b/tests/unit/test_packaging.py index 3bc4a37c..b98148f5 100644 --- a/tests/unit/test_packaging.py +++ b/tests/unit/test_packaging.py @@ -34,4 +34,4 @@ def test_namespace_package_compat(tmp_path): google_cloud.joinpath("othermod.py").write_text("") env = dict(os.environ, PYTHONPATH=str(tmp_path)) cmd = [sys.executable, "-m", "google.cloud.othermod"] - subprocess.check_call(cmd, env=env, shell=True) \ No newline at end of file + subprocess.check_call(cmd, env=env, shell=True) From 92fd70b77e3db98b64c3525bd4d73b39a8297a7a Mon Sep 17 00:00:00 2001 From: kiraksi Date: Wed, 15 Nov 2023 00:54:02 -0800 Subject: [PATCH 3/8] removed pkg_resources --- setup.py | 8 +++++++- tests/unit/test_reader_v1_arrow.py | 21 +++++++++++---------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index e4875f59..b52d446a 100644 --- a/setup.py +++ b/setup.py @@ -42,6 +42,7 @@ "proto-plus >= 1.22.0, <2.0.0dev", "proto-plus >= 1.22.2, <2.0.0dev; python_version>='3.11'", "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", + "importlib_metadata<=4.13.0; python_version<'3.8'", ] extras = { "pandas": ["pandas>=0.21.1"], @@ -58,10 +59,14 @@ packages = [ package - for package in setuptools.find_namespace_packages() + for package in setuptools.PEP420PackageFinder.find() if package.startswith("google") ] +namespaces = ["google"] +if "google.cloud" in packages: + namespaces.append("google.cloud") + setuptools.setup( name=name, version=version, @@ -88,6 +93,7 @@ platforms="Posix; MacOS X; Windows", packages=packages, python_requires=">=3.7", + namespace_packages=namespaces, install_requires=dependencies, extras_require=extras, scripts=["scripts/fixup_bigquery_storage_v1_keywords.py"], diff --git a/tests/unit/test_reader_v1_arrow.py b/tests/unit/test_reader_v1_arrow.py index 63e84935..d8aaf785 100644 --- a/tests/unit/test_reader_v1_arrow.py +++ b/tests/unit/test_reader_v1_arrow.py @@ -20,7 +20,11 @@ import pandas import pandas.testing import pytest -import pkg_resources + +try: + import importlib.metadata as metadata +except ImportError: + import importlib_metadata as metadata import google.api_core.exceptions from google.cloud.bigquery_storage import types @@ -30,9 +34,10 @@ pyarrow = pytest.importorskip("pyarrow") if pandas is not None: # pragma: NO COVER - PANDAS_INSTALLED_VERSION = pkg_resources.get_distribution("pandas").parsed_version -else: # pragma: NO COVER - PANDAS_INSTALLED_VERSION = pkg_resources.parse_version("0.0.0") + try: + PANDAS_INSTALLED_VERSION = metadata.version("pandas") + except metadata.PackageNotFoundError: + PANDAS_INSTALLED_VERSION = None # This dictionary is duplicated in bigquery/google/cloud/bigquery/_pandas_helpers.py @@ -178,9 +183,7 @@ def test_to_arrow_w_scalars_arrow(class_under_test, mock_gapic_client): assert actual_table == expected_table -@pytest.mark.skipif( - PANDAS_INSTALLED_VERSION >= pkg_resources.parse_version("2.0.0"), reason="" -) +@pytest.mark.skipif(PANDAS_INSTALLED_VERSION >= "2.0.0", reason="") def test_to_dataframe_w_scalars_arrow(class_under_test, mock_gapic_client): arrow_schema = _bq_to_arrow_schema(SCALAR_COLUMNS) arrow_batches = _bq_to_arrow_batches(SCALAR_BLOCKS, arrow_schema) @@ -248,9 +251,7 @@ def test_to_dataframe_w_dtypes_arrow(class_under_test, mock_gapic_client): ) -@pytest.mark.skipif( - PANDAS_INSTALLED_VERSION >= pkg_resources.parse_version("2.0.0"), reason="" -) +@pytest.mark.skipif(PANDAS_INSTALLED_VERSION >= "2.0.0", reason="") def test_to_dataframe_empty_w_scalars_arrow(class_under_test, mock_gapic_client): arrow_schema = _bq_to_arrow_schema(SCALAR_COLUMNS) read_session = _generate_arrow_read_session(arrow_schema) From 63e32cb6a181d66f7c891fb7005b5358cc1e2bd4 Mon Sep 17 00:00:00 2001 From: kiraksi Date: Thu, 16 Nov 2023 09:53:31 -0800 Subject: [PATCH 4/8] Moved importlib to extras and resolved issues with using it in tests --- setup.py | 6 ++++-- tests/unit/test_reader_v1_arrow.py | 11 +++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index b52d446a..8e24d8d4 100644 --- a/setup.py +++ b/setup.py @@ -42,10 +42,12 @@ "proto-plus >= 1.22.0, <2.0.0dev", "proto-plus >= 1.22.2, <2.0.0dev; python_version>='3.11'", "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", - "importlib_metadata<=4.13.0; python_version<'3.8'", ] extras = { - "pandas": ["pandas>=0.21.1"], + "pandas": [ + "pandas>=0.21.1", + "importlib_metadata>=1.0.0; python_version<'3.8'" + ], "fastavro": ["fastavro>=0.21.2"], "pyarrow": ["pyarrow>=0.15.0"], } diff --git a/tests/unit/test_reader_v1_arrow.py b/tests/unit/test_reader_v1_arrow.py index d8aaf785..0ada259b 100644 --- a/tests/unit/test_reader_v1_arrow.py +++ b/tests/unit/test_reader_v1_arrow.py @@ -34,10 +34,9 @@ pyarrow = pytest.importorskip("pyarrow") if pandas is not None: # pragma: NO COVER - try: - PANDAS_INSTALLED_VERSION = metadata.version("pandas") - except metadata.PackageNotFoundError: - PANDAS_INSTALLED_VERSION = None + PANDAS_INSTALLED_VERSION = metadata.version("pandas") +else: + PANDAS_INSTALLED_VERSION = "0.0.0" # This dictionary is duplicated in bigquery/google/cloud/bigquery/_pandas_helpers.py @@ -183,7 +182,7 @@ def test_to_arrow_w_scalars_arrow(class_under_test, mock_gapic_client): assert actual_table == expected_table -@pytest.mark.skipif(PANDAS_INSTALLED_VERSION >= "2.0.0", reason="") +@pytest.mark.skipif(PANDAS_INSTALLED_VERSION not in ["0", "1"], reason="") def test_to_dataframe_w_scalars_arrow(class_under_test, mock_gapic_client): arrow_schema = _bq_to_arrow_schema(SCALAR_COLUMNS) arrow_batches = _bq_to_arrow_batches(SCALAR_BLOCKS, arrow_schema) @@ -251,7 +250,7 @@ def test_to_dataframe_w_dtypes_arrow(class_under_test, mock_gapic_client): ) -@pytest.mark.skipif(PANDAS_INSTALLED_VERSION >= "2.0.0", reason="") +@pytest.mark.skipif(PANDAS_INSTALLED_VERSION not in ["0", "1"], reason="") def test_to_dataframe_empty_w_scalars_arrow(class_under_test, mock_gapic_client): arrow_schema = _bq_to_arrow_schema(SCALAR_COLUMNS) read_session = _generate_arrow_read_session(arrow_schema) From d0ea0356f1514caaa2055a6d8ba82d2ee0f7a8ad Mon Sep 17 00:00:00 2001 From: kiraksi Date: Thu, 16 Nov 2023 10:00:49 -0800 Subject: [PATCH 5/8] reformatted with black --- setup.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 8e24d8d4..a799b7cb 100644 --- a/setup.py +++ b/setup.py @@ -44,10 +44,7 @@ "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", ] extras = { - "pandas": [ - "pandas>=0.21.1", - "importlib_metadata>=1.0.0; python_version<'3.8'" - ], + "pandas": ["pandas>=0.21.1", "importlib_metadata>=1.0.0; python_version<'3.8'"], "fastavro": ["fastavro>=0.21.2"], "pyarrow": ["pyarrow>=0.15.0"], } From 9413caebca28a16f8a1753345059a9d664400675 Mon Sep 17 00:00:00 2001 From: kiraksi Date: Thu, 16 Nov 2023 12:42:54 -0800 Subject: [PATCH 6/8] Fixed mistake of not checking only first 3 indexes of PANDAS_VERSION --- tests/unit/test_reader_v1_arrow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_reader_v1_arrow.py b/tests/unit/test_reader_v1_arrow.py index 0ada259b..14e1c56a 100644 --- a/tests/unit/test_reader_v1_arrow.py +++ b/tests/unit/test_reader_v1_arrow.py @@ -182,7 +182,7 @@ def test_to_arrow_w_scalars_arrow(class_under_test, mock_gapic_client): assert actual_table == expected_table -@pytest.mark.skipif(PANDAS_INSTALLED_VERSION not in ["0", "1"], reason="") +@pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0", "1"], reason="") def test_to_dataframe_w_scalars_arrow(class_under_test, mock_gapic_client): arrow_schema = _bq_to_arrow_schema(SCALAR_COLUMNS) arrow_batches = _bq_to_arrow_batches(SCALAR_BLOCKS, arrow_schema) @@ -250,7 +250,7 @@ def test_to_dataframe_w_dtypes_arrow(class_under_test, mock_gapic_client): ) -@pytest.mark.skipif(PANDAS_INSTALLED_VERSION not in ["0", "1"], reason="") +@pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0", "1"], reason="") def test_to_dataframe_empty_w_scalars_arrow(class_under_test, mock_gapic_client): arrow_schema = _bq_to_arrow_schema(SCALAR_COLUMNS) read_session = _generate_arrow_read_session(arrow_schema) From f7587d9bc3fcc5e33c443c8f4e414c8891c738fa Mon Sep 17 00:00:00 2001 From: kiraksi Date: Thu, 16 Nov 2023 12:53:29 -0800 Subject: [PATCH 7/8] Fixed another error in comparison strings --- tests/unit/test_reader_v1_arrow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_reader_v1_arrow.py b/tests/unit/test_reader_v1_arrow.py index 14e1c56a..e0d1142c 100644 --- a/tests/unit/test_reader_v1_arrow.py +++ b/tests/unit/test_reader_v1_arrow.py @@ -182,7 +182,7 @@ def test_to_arrow_w_scalars_arrow(class_under_test, mock_gapic_client): assert actual_table == expected_table -@pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0", "1"], reason="") +@pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0.", "1."], reason="") def test_to_dataframe_w_scalars_arrow(class_under_test, mock_gapic_client): arrow_schema = _bq_to_arrow_schema(SCALAR_COLUMNS) arrow_batches = _bq_to_arrow_batches(SCALAR_BLOCKS, arrow_schema) @@ -250,7 +250,7 @@ def test_to_dataframe_w_dtypes_arrow(class_under_test, mock_gapic_client): ) -@pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0", "1"], reason="") +@pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0.", "1."], reason="") def test_to_dataframe_empty_w_scalars_arrow(class_under_test, mock_gapic_client): arrow_schema = _bq_to_arrow_schema(SCALAR_COLUMNS) read_session = _generate_arrow_read_session(arrow_schema) From a112ec27c568089ef821c491d6aef1dfb592390b Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 1 Dec 2023 21:39:11 +0000 Subject: [PATCH 8/8] use setuptools.find_namespace_packages() --- setup.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/setup.py b/setup.py index a799b7cb..67171f0a 100644 --- a/setup.py +++ b/setup.py @@ -58,14 +58,10 @@ packages = [ package - for package in setuptools.PEP420PackageFinder.find() + for package in setuptools.find_namespace_packages() if package.startswith("google") ] -namespaces = ["google"] -if "google.cloud" in packages: - namespaces.append("google.cloud") - setuptools.setup( name=name, version=version, @@ -92,7 +88,6 @@ platforms="Posix; MacOS X; Windows", packages=packages, python_requires=">=3.7", - namespace_packages=namespaces, install_requires=dependencies, extras_require=extras, scripts=["scripts/fixup_bigquery_storage_v1_keywords.py"],