[MAINT]: Remove tests from wheel distro (#2277)

* Move files only used in tests to `pvlib/tests/data`

* Attemp # 1

* Fine-tune inclusion via no-automatic data inclusion

* Update v0.11.2.rst

* Fix flake8 warnings

* Update docs/sphinx/source/whatsnew/v0.11.2.rst

Co-authored-by: Adam R. Jensen <[email protected]>

* Flatten the tests dir out of pvlib

* fix CI

* revert back to the 💩 CRLF

* Update v0.7.2.rst

* whatsnews

* Update test_solcast.py

---------

Co-authored-by: Adam R. Jensen <[email protected]>
Co-authored-by: Kevin Anderson <[email protected]>

Author: 3 people

.github/workflows/pytest-remote-data.yml

@@ -100,7 +100,7 @@ jobs:
           SOLARANYWHERE_API_KEY: ${{ secrets.SOLARANYWHERE_API_KEY }}
           BSRN_FTP_USERNAME: ${{ secrets.BSRN_FTP_USERNAME }}
           BSRN_FTP_PASSWORD: ${{ secrets.BSRN_FTP_PASSWORD }}
-        run: pytest pvlib/tests/iotools --cov=./ --cov-report=xml --remote-data
+        run: pytest tests/iotools --cov=./ --cov-report=xml --remote-data
 
       - name: Upload coverage to Codecov
         if: matrix.python-version == 3.9 && matrix.suffix == ''

.github/workflows/pytest.yml

@@ -79,7 +79,7 @@ jobs:
         shell: bash -l {0}  # necessary for conda env to be active
         run: |
           # ignore iotools; those tests are run in a separate workflow
-          pytest pvlib --cov=./ --cov-report=xml --ignore=pvlib/tests/iotools
+          pytest tests --cov=./ --cov-report=xml --ignore=tests/iotools
 
       - name: Upload coverage to Codecov
         if: matrix.python-version == 3.9 && matrix.suffix == '' && matrix.os == 'ubuntu-latest' && matrix.environment-type == 'conda'

.gitignore

@@ -81,9 +81,9 @@ coverage.xml
 
 
 # Datafiles
-*.csv
-# Ignore some csv 
-!pvlib/data/*.csv 
+# Do not exclude data directories
+!pvlib/data/**
+!tests/data/**
 
 # vi
 *.swp

codecov.yml

@@ -14,14 +14,15 @@ flags:
   core:
     paths:
       - pvlib/
+      - tests/
       - '!pvlib/iotools/'
-      - '!pvlib/tests/iotools/'
+      - '!tests/iotools/'
     carryforward: false
 
   remote-data:
     paths:
       - pvlib/iotools/
-      - pvlib/tests/iotools
+      - tests/iotools
     carryforward: true  # if not run, use coverage from previous commit
 
 
@@ -47,15 +48,15 @@ coverage:
       tests-core:
         target: 95%
         paths:
-          - 'pvlib/tests/.*'
-          - '!pvlib/tests/iotools/.*'
+          - 'tests/.*'
+          - '!tests/iotools/.*'
         flags:
           - core
 
       tests-remote-data:
         target: 95%
         paths:
-          - 'pvlib/tests/iotools/.*'
+          - 'tests/iotools/.*'
         flags:
           - remote-data
 

docs/sphinx/source/contributing/testing.rst

@@ -85,14 +85,22 @@ all = ["pvlib[test,optional,doc]"]
 Documentation = "https://pvlib-python.readthedocs.io/"
 "Source Code" = "https://github.com/pvlib/pvlib-python"
 
+[tool.setuptools]
+# do not include all non-python files in the bdist by default, but select them manually
+# https://setuptools.pypa.io/en/latest/userguide/datafiles.html#interplay-between-these-keywords (2024-10-22)
+include-package-data = false  # true by default
+
 [tool.setuptools.packages.find]
 include = ["pvlib*"]
 
+[tool.setuptools.package-data]
+pvlib = ["data/*"]
+
 [tool.setuptools_scm]
 
 [tool.pytest]
 junit_family = "xunit2"
-testpaths = "pvlib/tests"
+testpaths = "tests"
 # warning messages to suppress from pytest output.  useful in cases
 # where a dependency hasn't addressed a deprecation yet, and there's
 # nothing we can do to fix it ourselves.

docs/sphinx/source/whatsnew/v0.11.3.rst

@@ -5,7 +5,7 @@
 import numpy as np
 import pandas as pd
 from pvlib.bifacial import infinite_sheds
-from ..conftest import assert_series_equal
+from tests.conftest import assert_series_equal
 
 import pytest
 

docs/sphinx/source/whatsnew/v0.7.2.rst

@@ -1,7 +1,7 @@
 import pandas as pd
 from datetime import datetime
 from pvlib.bifacial.pvfactors import pvfactors_timeseries
-from ..conftest import requires_pvfactors, assert_series_equal
+from tests.conftest import requires_pvfactors, assert_series_equal
 import pytest
 
 

pyproject.toml

@@ -70,7 +70,8 @@ def assert_frame_equal(left, right, **kwargs):
 
 # commonly used directories in the tests
 TEST_DIR = Path(__file__).parent
-DATA_DIR = TEST_DIR.parent / 'data'
+PVLIB_DATA_DIR = Path(pvlib.__path__[0]) / 'data'
+TESTS_DATA_DIR = TEST_DIR / 'data'
 
 
 # pytest-rerunfailures variables

pvlib/tests/__init__.py renamed to tests/__init__.py

@@ -9,7 +9,7 @@
     get_acis_prism, get_acis_nrcc, get_acis_mpe,
     get_acis_station_data, get_acis_available_stations
 )
-from ..conftest import RERUNS, RERUNS_DELAY, assert_frame_equal
+from tests.conftest import RERUNS, RERUNS_DELAY, assert_frame_equal
 from requests import HTTPError
 
 

pvlib/tests/bifacial/__init__.py renamed to tests/bifacial/__init__.py

@@ -7,8 +7,13 @@
 import os
 import tempfile
 from pvlib.iotools import read_bsrn, get_bsrn
-from ..conftest import (DATA_DIR, RERUNS, RERUNS_DELAY, assert_index_equal,
-                        requires_bsrn_credentials)
+from tests.conftest import (
+    TESTS_DATA_DIR,
+    RERUNS,
+    RERUNS_DELAY,
+    assert_index_equal,
+    requires_bsrn_credentials,
+)
 
 
 @pytest.fixture(scope="module")
@@ -33,7 +38,7 @@ def expected_index():
     ('bsrn-lr0100-pay0616.dat'),
 ])
 def test_read_bsrn(testfile, expected_index):
-    data, metadata = read_bsrn(DATA_DIR / testfile)
+    data, metadata = read_bsrn(TESTS_DATA_DIR / testfile)
     assert_index_equal(expected_index, data.index)
     assert 'ghi' in data.columns
     assert 'dni_std' in data.columns
@@ -45,7 +50,7 @@ def test_read_bsrn(testfile, expected_index):
 def test_read_bsrn_logical_records(expected_index):
     # Test if logical records 0300 and 0500 are correct parsed
     # and that 0100 is not passed when not specified
-    data, metadata = read_bsrn(DATA_DIR / 'bsrn-pay0616.dat.gz',
+    data, metadata = read_bsrn(TESTS_DATA_DIR / 'bsrn-pay0616.dat.gz',
                                logical_records=['0300', '0500'])
     assert_index_equal(expected_index, data.index)
     assert 'lwu' in data.columns
@@ -57,13 +62,13 @@ def test_read_bsrn_logical_records(expected_index):
 def test_read_bsrn_bad_logical_record():
     # Test if ValueError is raised if an unsupported logical record is passed
     with pytest.raises(ValueError, match='not in'):
-        read_bsrn(DATA_DIR / 'bsrn-lr0100-pay0616.dat',
+        read_bsrn(TESTS_DATA_DIR / 'bsrn-lr0100-pay0616.dat',
                   logical_records=['dummy'])
 
 
 def test_read_bsrn_logical_records_not_found():
     # Test if an empty dataframe is returned if specified LRs are not present
-    data, metadata = read_bsrn(DATA_DIR / 'bsrn-lr0100-pay0616.dat',
+    data, metadata = read_bsrn(TESTS_DATA_DIR / 'bsrn-lr0100-pay0616.dat',
                                logical_records=['0300', '0500'])
     assert data.empty  # assert that the dataframe is empty
     assert 'uva_global' in data.columns

pvlib/tests/bifacial/test_infinite_sheds.py renamed to tests/bifacial/test_infinite_sheds.py

@@ -3,7 +3,11 @@
 from numpy import dtype, nan
 import pytest
 from pvlib.iotools import crn
-from ..conftest import DATA_DIR, assert_frame_equal, assert_index_equal
+from tests.conftest import (
+    TESTS_DATA_DIR,
+    assert_frame_equal,
+    assert_index_equal,
+)
 
 
 @pytest.fixture
@@ -41,12 +45,12 @@ def dtypes():
 
 @pytest.fixture
 def testfile():
-    return DATA_DIR / 'CRNS0101-05-2019-AZ_Tucson_11_W.txt'
+    return TESTS_DATA_DIR / 'CRNS0101-05-2019-AZ_Tucson_11_W.txt'
 
 
 @pytest.fixture
 def testfile_problems():
-    return DATA_DIR / 'CRN_with_problems.txt'
+    return TESTS_DATA_DIR / 'CRN_with_problems.txt'
 
 
 def test_read_crn(testfile, columns_mapped, dtypes):

pvlib/tests/bifacial/test_losses_models.py renamed to tests/bifacial/test_losses_models.py

@@ -1,9 +1,9 @@
 import pytest
 
 from pvlib.iotools import epw
-from ..conftest import DATA_DIR, RERUNS, RERUNS_DELAY
+from tests.conftest import TESTS_DATA_DIR, RERUNS, RERUNS_DELAY
 
-epw_testfile = DATA_DIR / 'NLD_Amsterdam062400_IWEC.epw'
+epw_testfile = TESTS_DATA_DIR / 'NLD_Amsterdam062400_IWEC.epw'
 
 
 def test_read_epw():

pvlib/tests/bifacial/test_pvfactors.py renamed to tests/bifacial/test_pvfactors.py

@@ -3,7 +3,7 @@
 import pytz
 
 from pvlib.iotools import midc
-from ..conftest import DATA_DIR, RERUNS, RERUNS_DELAY
+from tests.conftest import TESTS_DATA_DIR, RERUNS, RERUNS_DELAY
 
 
 @pytest.fixture
@@ -17,10 +17,10 @@ def test_mapping():
     }
 
 
-MIDC_TESTFILE = DATA_DIR / 'midc_20181014.txt'
-MIDC_RAW_TESTFILE = DATA_DIR / 'midc_raw_20181018.txt'
+MIDC_TESTFILE = TESTS_DATA_DIR / 'midc_20181014.txt'
+MIDC_RAW_TESTFILE = TESTS_DATA_DIR / 'midc_raw_20181018.txt'
 MIDC_RAW_SHORT_HEADER_TESTFILE = (
-    DATA_DIR / 'midc_raw_short_header_20191115.txt')
+    TESTS_DATA_DIR / 'midc_raw_short_header_20191115.txt')
 
 # TODO: not used, remove?
 # midc_network_testfile = ('https://midcdmz.nrel.gov/apps/data_api.pl'

pvlib/tests/bifacial/test_utils.py renamed to tests/bifacial/test_utils.py

@@ -3,10 +3,11 @@
 """
 
 from pvlib.iotools import read_panond
-from pvlib.tests.conftest import DATA_DIR
 
-PAN_FILE = DATA_DIR / 'ET-M772BH550GL.PAN'
-OND_FILE = DATA_DIR / 'CPS SCH275KTL-DO-US-800-250kW_275kVA_1.OND'
+from tests.conftest import TESTS_DATA_DIR
+
+PAN_FILE = TESTS_DATA_DIR / 'ET-M772BH550GL.PAN'
+OND_FILE = TESTS_DATA_DIR / 'CPS SCH275KTL-DO-US-800-250kW_275kVA_1.OND'
 
 
 def test_read_panond():

pvlib/tests/conftest.py renamed to tests/conftest.py

@@ -2,21 +2,25 @@
 test iotools for PSM3
 """
 
-import os
 from pvlib.iotools import psm3
-from ..conftest import DATA_DIR, RERUNS, RERUNS_DELAY, assert_index_equal
+from tests.conftest import (
+    TESTS_DATA_DIR,
+    RERUNS,
+    RERUNS_DELAY,
+    assert_index_equal,
+)
 import numpy as np
 import pandas as pd
 import pytest
+import os
 from requests import HTTPError
 from io import StringIO
 import warnings
-from pvlib._deprecation import pvlibDeprecationWarning
 
-TMY_TEST_DATA = DATA_DIR / 'test_psm3_tmy-2017.csv'
-YEAR_TEST_DATA = DATA_DIR / 'test_psm3_2017.csv'
-YEAR_TEST_DATA_5MIN = DATA_DIR / 'test_psm3_2019_5min.csv'
-MANUAL_TEST_DATA = DATA_DIR / 'test_read_psm3.csv'
+TMY_TEST_DATA = TESTS_DATA_DIR / 'test_psm3_tmy-2017.csv'
+YEAR_TEST_DATA = TESTS_DATA_DIR / 'test_psm3_2017.csv'
+YEAR_TEST_DATA_5MIN = TESTS_DATA_DIR / 'test_psm3_2019_5min.csv'
+MANUAL_TEST_DATA = TESTS_DATA_DIR / 'test_read_psm3.csv'
 LATITUDE, LONGITUDE = 40.5137, -108.5449
 METADATA_FIELDS = [
     'Source', 'Location ID', 'City', 'State', 'Country', 'Latitude',