diff --git a/.gitignore b/.gitignore
index 57070a8c..ed7caa96 100644
--- a/.gitignore
+++ b/.gitignore
@@ -59,7 +59,7 @@ nosetests.xml
coverage.xml
*,cover
.hypothesis/
-mapie_v1/integration_tests/mapie_v0_package
+tests_v1/integration_tests/mapie_v0_package
# Translations
*.mo
diff --git a/Makefile b/Makefile
index f471b461..20dbcdcf 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,6 @@
.PHONY: tests doc build
+integration_tests_folder_name = tests_v1/integration_tests
mapie_v0_folder_name = mapie_v0_package
lint:
@@ -13,21 +14,29 @@ v1-type-check:
tests:
pytest -vs --doctest-modules mapie
- pytest -vs --doctest-modules mapie_v1 --ignore=mapie_v1/integration_tests
+ $(MAKE) v1-tests
+ $(MAKE) v1-docstring-tests
-integration-tests-v1:
- @pip install git+https://github.com/scikit-learn-contrib/MAPIE@master --no-dependencies --target=./mapie_v1/integration_tests/$(mapie_v0_folder_name) >/dev/null 2>&1
- @mv ./mapie_v1/integration_tests/$(mapie_v0_folder_name)/mapie ./mapie_v1/integration_tests/$(mapie_v0_folder_name)/mapiev0
- @- export PYTHONPATH="${PYTHONPATH}:./mapie_v1/integration_tests/$(mapie_v0_folder_name)"; pytest -vs mapie_v1/integration_tests/tests -k $(pattern)
- @mv ./mapie_v1/integration_tests/$(mapie_v0_folder_name)/mapiev0 ./mapie_v1/integration_tests/$(mapie_v0_folder_name)/mapie
+v1-tests:
+ pytest -vs tests_v1 --ignore=$(integration_tests_folder_name)
-checks-v1-not-in-ci:
+v1-docstring-tests:
+ pytest -vs --doctest-modules mapie_v1 --ignore=$(integration_tests_folder_name)
+
+v1-integration-tests:
+ @pip install git+https://github.com/scikit-learn-contrib/MAPIE@master --no-dependencies --target=./$(integration_tests_folder_name)/$(mapie_v0_folder_name) >/dev/null 2>&1
+ @mv ./$(integration_tests_folder_name)/$(mapie_v0_folder_name)/mapie ./$(integration_tests_folder_name)/$(mapie_v0_folder_name)/mapiev0
+ @- export PYTHONPATH="${PYTHONPATH}:./$(integration_tests_folder_name)/$(mapie_v0_folder_name)"; pytest -vs $(integration_tests_folder_name)/tests $(params)
+ @mv ./$(integration_tests_folder_name)/$(mapie_v0_folder_name)/mapiev0 ./$(integration_tests_folder_name)/$(mapie_v0_folder_name)/mapie
+
+v1-checks-not-in-ci:
$(MAKE) v1-type-check
- $(MAKE) integration-tests-v1 pattern=test
+ $(MAKE) v1-integration-tests
+ $(MAKE) v1-coverage
coverage:
+ # We may need to add the v1 test suite here if we remove some v0 tests, to keep a 100% coverage
pytest -vs \
- --doctest-modules \
--cov-branch \
--cov=mapie \
--cov-report term-missing \
@@ -35,6 +44,15 @@ coverage:
--cov-fail-under=100 \
--cov-config=.coveragerc
+v1-coverage:
+ pytest -vsx \
+ --cov-branch \
+ --cov=mapie_v1 \
+ --cov-report term-missing \
+ --pyargs tests_v1 --ignore=$(integration_tests_folder_name) \
+ --cov-fail-under=100 \
+ --cov-config=.coveragerc
+
doc:
$(MAKE) html -C doc
diff --git a/mapie/tests/test_regression.py b/mapie/tests/test_regression.py
index f19c3021..bcf34c7e 100644
--- a/mapie/tests/test_regression.py
+++ b/mapie/tests/test_regression.py
@@ -417,33 +417,6 @@ def test_calibration_data_size_asymmetric_score(delta: float) -> None:
mapie_reg.predict(Xt, alpha=1-delta)
-def test_same_results_prefit_split() -> None:
- """
- Test checking that if split and prefit method have exactly
- the same data split, then we have exactly the same results.
- """
- X, y = make_regression(
- n_samples=500, n_features=10, noise=1.0, random_state=1
- )
- cv = ShuffleSplit(n_splits=1, test_size=0.1, random_state=random_state)
- train_index, val_index = list(cv.split(X))[0]
- X_train, X_calib = X[train_index], X[val_index]
- y_train, y_calib = y[train_index], y[val_index]
-
- mapie_reg = MapieRegressor(method='base', cv=cv)
- mapie_reg.fit(X, y)
- y_pred_1, y_pis_1 = mapie_reg.predict(X, alpha=0.1)
-
- model = LinearRegression().fit(X_train, y_train)
- mapie_reg = MapieRegressor(estimator=model, method='base', cv="prefit")
- mapie_reg.fit(X_calib, y_calib)
- y_pred_2, y_pis_2 = mapie_reg.predict(X, alpha=0.1)
-
- np.testing.assert_allclose(y_pred_1, y_pred_2)
- np.testing.assert_allclose(y_pis_1[:, 0, 0], y_pis_2[:, 0, 0])
- np.testing.assert_allclose(y_pis_1[:, 1, 0], y_pis_2[:, 1, 0])
-
-
@pytest.mark.parametrize("strategy", [*STRATEGIES])
def test_results_for_same_alpha(strategy: str) -> None:
"""
diff --git a/tests_v1/integration_tests/tests/pytest.ini b/tests_v1/integration_tests/tests/pytest.ini
new file mode 100644
index 00000000..6fcf120c
--- /dev/null
+++ b/tests_v1/integration_tests/tests/pytest.ini
@@ -0,0 +1,3 @@
+[pytest]
+markers =
+ regression
\ No newline at end of file
diff --git a/mapie_v1/integration_tests/tests/test_regression.py b/tests_v1/integration_tests/tests/test_regression.py
similarity index 99%
rename from mapie_v1/integration_tests/tests/test_regression.py
rename to tests_v1/integration_tests/tests/test_regression.py
index 8dea54ad..ec23315f 100644
--- a/mapie_v1/integration_tests/tests/test_regression.py
+++ b/tests_v1/integration_tests/tests/test_regression.py
@@ -22,10 +22,12 @@
from mapiev0.regression import MapieRegressor as MapieRegressorV0 # noqa
from mapiev0.regression import MapieQuantileRegressor as MapieQuantileRegressorV0 # noqa
-from mapie_v1.integration_tests.utils import (filter_params,
+from tests_v1.integration_tests.utils import (filter_params,
train_test_split_shuffle)
from sklearn.model_selection import LeaveOneOut, GroupKFold
+pytestmark = pytest.mark.regression
+
RANDOM_STATE = 1
K_FOLDS = 3
N_BOOTSTRAPS = 30
diff --git a/mapie_v1/integration_tests/utils.py b/tests_v1/integration_tests/utils.py
similarity index 100%
rename from mapie_v1/integration_tests/utils.py
rename to tests_v1/integration_tests/utils.py
diff --git a/tests_v1/tests/test_regression.py b/tests_v1/tests/test_regression.py
new file mode 100644
index 00000000..0dcdfd69
--- /dev/null
+++ b/tests_v1/tests/test_regression.py
@@ -0,0 +1,56 @@
+import numpy as np
+import pytest
+from sklearn.datasets import make_regression
+from sklearn.linear_model import LinearRegression
+from sklearn.model_selection import train_test_split
+from mapie_v1.regression import SplitConformalRegressor
+
+RANDOM_STATE = 1
+
+
+@pytest.fixture(scope="module")
+def dataset():
+ X, y = make_regression(
+ n_samples=500, n_features=2, noise=1.0, random_state=RANDOM_STATE
+ )
+ X_train, X_conf_test, y_train, y_conf_test = train_test_split(
+ X, y, random_state=RANDOM_STATE
+ )
+ X_conformalize, X_test, y_conformalize, y_test = train_test_split(
+ X_conf_test, y_conf_test, random_state=RANDOM_STATE
+ )
+ return X_train, X_conformalize, X_test, y_train, y_conformalize, y_test
+
+
+@pytest.fixture
+def predictions_scr_prefit(dataset):
+ X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset
+ regressor = LinearRegression()
+ regressor.fit(X_train, y_train)
+ scr_prefit = SplitConformalRegressor(estimator=regressor, prefit=True)
+ scr_prefit.conformalize(X_conformalize, y_conformalize)
+ return scr_prefit.predict_interval(X_test)
+
+
+@pytest.fixture
+def predictions_scr_not_prefit(dataset):
+ X_train, X_conformalize, X_test, y_train, y_conformalize, y_test = dataset
+ scr_not_prefit = SplitConformalRegressor(estimator=LinearRegression(), prefit=False)
+ scr_not_prefit.fit(X_train, y_train).conformalize(X_conformalize, y_conformalize)
+ return scr_not_prefit.predict_interval(X_test)
+
+
+def test_scr_same_intervals_prefit_not_prefit(
+ predictions_scr_prefit, predictions_scr_not_prefit
+) -> None:
+ intervals_scr_prefit = predictions_scr_prefit[1]
+ intervals_scr_not_prefit = predictions_scr_not_prefit[1]
+ np.testing.assert_equal(intervals_scr_prefit, intervals_scr_not_prefit)
+
+
+def test_scr_same_predictions_prefit_not_prefit(
+ predictions_scr_prefit, predictions_scr_not_prefit
+) -> None:
+ predictions_scr_prefit = predictions_scr_prefit[0]
+ predictions_scr_not_prefit = predictions_scr_not_prefit[0]
+ np.testing.assert_equal(predictions_scr_prefit, predictions_scr_not_prefit)