TEST: create a single folder for all v1 tests, add test to compare pr…

…efit/not prefit, remove it from v0 tests, add non-integration tests to CI

.gitignore

@@ -59,7 +59,7 @@ nosetests.xml
 coverage.xml
 *,cover
 .hypothesis/
-mapie_v1/integration_tests/mapie_v0_package
+tests_v1/integration_tests/mapie_v0_package
 
 # Translations
 *.mo

Makefile

@@ -1,5 +1,6 @@
 .PHONY: tests doc build
 
+integration_tests_folder_name = tests_v1/integration_tests
 mapie_v0_folder_name = mapie_v0_package
 
 lint:
@@ -13,28 +14,44 @@ v1-type-check:
 
 tests:
 	pytest -vs --doctest-modules mapie
-	pytest -vs --doctest-modules mapie_v1 --ignore=mapie_v1/integration_tests
+	$(MAKE) v1-tests
+	$(MAKE) v1-docstring-tests
 
-integration-tests-v1:
-	@pip install git+https://github.com/scikit-learn-contrib/MAPIE@master --no-dependencies --target=./mapie_v1/integration_tests/$(mapie_v0_folder_name) >/dev/null 2>&1
-	@mv ./mapie_v1/integration_tests/$(mapie_v0_folder_name)/mapie ./mapie_v1/integration_tests/$(mapie_v0_folder_name)/mapiev0
-	@- export PYTHONPATH="${PYTHONPATH}:./mapie_v1/integration_tests/$(mapie_v0_folder_name)"; pytest -vs mapie_v1/integration_tests/tests  -k $(pattern)
-	@mv ./mapie_v1/integration_tests/$(mapie_v0_folder_name)/mapiev0 ./mapie_v1/integration_tests/$(mapie_v0_folder_name)/mapie
+v1-tests:
+	pytest -vs tests_v1 --ignore=$(integration_tests_folder_name)
 
-checks-v1-not-in-ci:
+v1-docstring-tests:
+	pytest -vs --doctest-modules mapie_v1 --ignore=$(integration_tests_folder_name)
+
+v1-integration-tests:
+	@pip install git+https://github.com/scikit-learn-contrib/MAPIE@master --no-dependencies --target=./$(integration_tests_folder_name)/$(mapie_v0_folder_name) >/dev/null 2>&1
+	@mv ./$(integration_tests_folder_name)/$(mapie_v0_folder_name)/mapie ./$(integration_tests_folder_name)/$(mapie_v0_folder_name)/mapiev0
+	@- export PYTHONPATH="${PYTHONPATH}:./$(integration_tests_folder_name)/$(mapie_v0_folder_name)"; pytest -vs $(integration_tests_folder_name)/tests $(params)
+	@mv ./$(integration_tests_folder_name)/$(mapie_v0_folder_name)/mapiev0 ./$(integration_tests_folder_name)/$(mapie_v0_folder_name)/mapie
+
+v1-checks-not-in-ci:
 	$(MAKE) v1-type-check
-	$(MAKE) integration-tests-v1 pattern=test
+	$(MAKE) v1-integration-tests
+	$(MAKE) v1-coverage
 
 coverage:
+	# We may need to add the v1 test suite here if we remove some v0 tests, to keep a 100% coverage
 	pytest -vs \
-		--doctest-modules \
 		--cov-branch \
 		--cov=mapie \
 		--cov-report term-missing \
 		--pyargs mapie \
 		--cov-fail-under=100 \
 		--cov-config=.coveragerc
 
+v1-coverage:
+	pytest -vs \
+		--cov-branch \
+		--cov=mapie_v1 \
+		--cov-report term-missing \
+		--pyargs tests_v1 --ignore=$(integration_tests_folder_name) \
+		--cov-fail-under=100
+
 doc:
 	$(MAKE) html -C doc
 

mapie/tests/test_regression.py

@@ -417,33 +417,6 @@ def test_calibration_data_size_asymmetric_score(delta: float) -> None:
         mapie_reg.predict(Xt, alpha=1-delta)
 
 
-def test_same_results_prefit_split() -> None:
-    """
-    Test checking that if split and prefit method have exactly
-    the same data split, then we have exactly the same results.
-    """
-    X, y = make_regression(
-        n_samples=500, n_features=10, noise=1.0, random_state=1
-    )
-    cv = ShuffleSplit(n_splits=1, test_size=0.1, random_state=random_state)
-    train_index, val_index = list(cv.split(X))[0]
-    X_train, X_calib = X[train_index], X[val_index]
-    y_train, y_calib = y[train_index], y[val_index]
-
-    mapie_reg = MapieRegressor(method='base', cv=cv)
-    mapie_reg.fit(X, y)
-    y_pred_1, y_pis_1 = mapie_reg.predict(X, alpha=0.1)
-
-    model = LinearRegression().fit(X_train, y_train)
-    mapie_reg = MapieRegressor(estimator=model, method='base', cv="prefit")
-    mapie_reg.fit(X_calib, y_calib)
-    y_pred_2, y_pis_2 = mapie_reg.predict(X, alpha=0.1)
-
-    np.testing.assert_allclose(y_pred_1, y_pred_2)
-    np.testing.assert_allclose(y_pis_1[:, 0, 0], y_pis_2[:, 0, 0])
-    np.testing.assert_allclose(y_pis_1[:, 1, 0], y_pis_2[:, 1, 0])
-
-
 @pytest.mark.parametrize("strategy", [*STRATEGIES])
 def test_results_for_same_alpha(strategy: str) -> None:
     """

tests_v1/integration_tests/tests/pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+markers =
+    regression

mapie_v1/integration_tests/tests/test_regression.py → tests_v1/integration_tests/tests/test_regression.py

@@ -22,10 +22,12 @@
 
 from mapiev0.regression import MapieRegressor as MapieRegressorV0  # noqa
 from mapiev0.regression import MapieQuantileRegressor as MapieQuantileRegressorV0  # noqa
-from mapie_v1.integration_tests.utils import (filter_params,
+from tests_v1.integration_tests.utils import (filter_params,
                                               train_test_split_shuffle)
 from sklearn.model_selection import LeaveOneOut, GroupKFold
 
+pytestmark = pytest.mark.regression
+
 RANDOM_STATE = 1
 K_FOLDS = 3
 N_BOOTSTRAPS = 30

tests_v1/tests/test_regression.py

@@ -0,0 +1,56 @@
+import numpy as np
+import pytest
+from sklearn.datasets import make_regression
+from sklearn.linear_model import LinearRegression
+from sklearn.model_selection import train_test_split
+from mapie_v1.regression import SplitConformalRegressor
+
+RANDOM_STATE = 1
+
+
+@pytest.fixture(scope="module")
+def dataset():
+    X, y = make_regression(
+        n_samples=500, n_features=2, noise=1.0, random_state=RANDOM_STATE
+    )
+    X_train, X_conf_test, y_train, y_conf_test = train_test_split(
+        X, y, random_state=RANDOM_STATE
+    )
+    X_conformalize, X_test, y_conformalize, y_test = train_test_split(
+        X_conf_test, y_conf_test, random_state=RANDOM_STATE
+    )
+    return X_train, X_conformalize, X_test, y_train, y_conformalize, y_test
+
+
+@pytest.fixture
+def predictions_scr_prefit(dataset):
+    X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset
+    regressor = LinearRegression()
+    regressor.fit(X_train, y_train)
+    scr_prefit = SplitConformalRegressor(estimator=regressor, prefit=True)
+    scr_prefit.conformalize(X_conformalize, y_conformalize)
+    return scr_prefit.predict_interval(X_test)
+
+
+@pytest.fixture
+def predictions_scr_not_prefit(dataset):
+    X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset
+    scr_not_prefit = SplitConformalRegressor(estimator=LinearRegression(), prefit=False)
+    scr_not_prefit.fit(X_train, y_train).conformalize(X_conformalize, y_conformalize)
+    return scr_not_prefit.predict_interval(X_test)
+
+
+def test_scr_same_intervals_prefit_not_prefit(
+    predictions_scr_prefit, predictions_scr_not_prefit
+) -> None:
+    intervals_scr_prefit = predictions_scr_prefit[1]
+    intervals_scr_not_prefit = predictions_scr_not_prefit[1]
+    np.testing.assert_equal(intervals_scr_prefit, intervals_scr_not_prefit)
+
+
+def test_scr_same_predictions_prefit_not_prefit(
+    predictions_scr_prefit, predictions_scr_not_prefit
+) -> None:
+    predictions_scr_prefit = predictions_scr_prefit[0]
+    predictions_scr_not_prefit = predictions_scr_not_prefit[0]
+    np.testing.assert_equal(predictions_scr_prefit, predictions_scr_not_prefit)