Document extra ML requirements for MLflow (mlflow#3495)

Author: smurching

.github/workflows/master.yml

@@ -90,7 +90,7 @@ jobs:
     - uses: actions/checkout@v2
     - name: Install dependencies
       env:
-        INSTALL_LARGE_PYTHON_DEPS: true
+        INSTALL_SMALL_PYTHON_DEPS: true
       run: |
         source ./dev/install-common-deps.sh
     - name: Run tests
@@ -106,6 +106,7 @@ jobs:
     - name: Install dependencies
       env:
         INSTALL_LARGE_PYTHON_DEPS: true
+        INSTALL_SMALL_PYTHON_DEPS: true
       run: |
         source ./dev/install-common-deps.sh
     - name: Run tests

EXTRA_DEPENDENCIES.rst

@@ -0,0 +1,19 @@
+=========================
+Extra MLflow Dependencies
+=========================
+
+When you `install the MLflow Python package <https://mlflow.org/docs/latest/quickstart.html#installing-mlflow>`_,
+a set of core dependencies needed to use most MLflow functionality (tracking, projects, models APIs)
+is also installed.
+
+However, in order to use certain framework-specific MLflow APIs or configuration options,
+you need to install additional, "extra" dependencies. For example, the model persistence APIs under
+the ``mlflow.sklearn`` module require scikit-learn to be installed. Some of the most common MLflow
+extra dependencies can be installed via ``pip install mlflow[extras]``.
+
+The full set of extra dependencies are documented, along with the modules that depend on them,
+in the following files:
+
+* extra-ml-requirements.txt: ML libraries needed to use model persistence and inference APIs
+* small-requirements.txt, large-requirements.txt: Libraries required to use non-default
+  artifact-logging and tracking server configurations

conftest.py

@@ -85,6 +85,9 @@ def pytest_ignore_collect(path, config):
             "tests/spacy",
             "tests/spark_autologging",
             "tests/fastai",
+            "tests/models",
+            "tests/shap",
+            "tests/utils/test_model_utils.py",
         ]
 
         relpath = os.path.relpath(str(path))

dev/extra-ml-requirements.txt

@@ -0,0 +1,38 @@
+## This file describes extra ML library dependencies that you, as an end user,
+## must install in order to use various MLflow Python modules.
+##
+# Required by mlflow.azureml
+azureml-sdk==1.2.0; python_version >= "3.0"
+# Required by mlflow.pyfunc
+cloudpickle==0.8.0
+# Required by mlflow.keras
+keras==2.3.1
+# Required by mlflow.sklearn
+scikit-learn==0.23.2
+# Required by mlflow.gluon
+mxnet==1.5.0
+# Required by mlflow.fastai
+fastai==1.0.60
+# Required by mlflow.spacy
+spacy==2.2.3
+# Required by mlflow.tensorflow
+tensorflow==1.15.2
+# Required by mlflow.pytorch
+torch==1.4.0
+torchvision==0.5.0
+# Required by mlflow.xgboost
+xgboost>=0.82
+# Required by mlflow.lightgbm
+lightgbm==2.3.0
+# Required by mlflow.h2o
+h2o==3.22.1.4
+# Required by mlflow.onnx
+onnx==1.4.1;
+onnxmltools==1.4.0;
+onnxruntime==0.3.0;
+# Required by mlflow.mleap, and in order to save SparkML models in
+# mleap format via ``mlflow.spark.log_model``, ``mlflow.spark.save_model``
+mleap==0.16.0
+# Required by mlflow.spark
+pyspark==2.4.0
+

dev/install-common-deps.sh

@@ -22,7 +22,6 @@ CONDA_DIR=/usr/share/miniconda
 export PATH="$CONDA_DIR/bin:$PATH"
 hash -r
 conda config --set always_yes yes --set changeps1 no
-
 # Useful for debugging any issues with conda
 conda info -a
 conda create -q -n test-environment python=3.6
@@ -40,6 +39,7 @@ if [[ "$INSTALL_SMALL_PYTHON_DEPS" == "true" ]]; then
 fi
 if [[ "$INSTALL_LARGE_PYTHON_DEPS" == "true" ]]; then
   retry-with-backoff pip install --quiet -r ./dev/large-requirements.txt
+  retry-with-backoff pip install --quiet -r ./dev/extra-ml-requirements.txt
   # Hack: make sure all spark-* scripts are executable.
   # Conda installs 2 version spark-* scripts and makes the ones spark
   # uses not executable. This is a temporary fix to unblock the tests.

dev/large-requirements.txt

@@ -1,39 +1,5 @@
-# Large test reqs
-azure-storage-blob==12.3.0
-google-cloud-storage==1.14.0
-botocore==1.12.84
-boto3==1.9.84
-moto==1.3.7
-h2o==3.22.1.4
-onnx==1.4.1;
-onnxmltools==1.4.0;
-onnxruntime==0.3.0;
-mleap==0.16.0
-mxnet==1.5.0
-pyarrow==0.17.0
-pyspark==2.4.0
-pytest==3.2.1
-pytest-cov==2.6.0
-scikit-learn==0.23.2
-scipy==1.2.1
-spacy==2.2.3
-tensorflow==1.15.2
-tf2onnx==1.5.4;
-torch==1.4.0
-torchvision==0.5.0
-xgboost>=0.82
-lightgbm==2.3.0
-# Install typing to fix torch on Python 2.7 (see https://github.com/pytorch/pytorch/issues/16775)
-typing==3.6.6
-pysftp==0.2.9
-keras==2.3.1
-attrdict==2.0.0
-azureml-sdk==1.2.0; python_version >= "3.0"
-cloudpickle==0.8.0
-pytest-localserver==0.5.0
-sqlalchemy==1.3.0
-kubernetes==9.0.0
-fastai==1.0.60
+## Large test reqs
+## Test-only dependencies
 matplotlib==3.2.1
-# test plugin
-tests/resources/mlflow-test-plugin/
+tf2onnx==1.5.4
+pyarrow==0.17.0

dev/run-large-python-tests.sh

@@ -7,6 +7,6 @@ trap 'err=1' ERR
 export MLFLOW_HOME=$(pwd)
 
 # NB: Also add --ignore'd tests to run-small-python-tests.sh
-pytest tests --large --ignore-flavors --ignore=tests/examples --ignore=tests/models
+pytest tests --large --ignore-flavors --ignore=tests/examples
 
 test $err = 0

dev/run-python-flavor-tests.sh

@@ -21,5 +21,7 @@ pytest --verbose tests/gluon --large
 pytest --verbose tests/gluon_autolog --large
 pytest --verbose tests/spacy --large
 pytest --verbose tests/fastai --large
+pytest --verbose tests/utils/test_model_utils.py --large
+
 
 test $err = 0

dev/small-requirements.txt

@@ -1,21 +1,26 @@
-# Small test reqs
+## Small test reqs
+# Required
 azure-storage-blob==12.3.0
+# Required to log artifacts and models to GCS artifact locations
 google-cloud-storage==1.14.0
+# Required to log artifacts and models to AWS S3 artifact locations
 botocore==1.12.84  # pinned for moto
 boto3==1.9.84      # pinned for moto
-moto==1.3.7
-mxnet==1.5.0
-scikit-learn==0.23.2
-matplotlib==3.2.1
-scipy==1.2.1
+# Required to log artifacts and models to HDFS artifact locations
 pyarrow==0.17.0
+# Required to log artifacts to SFTP artifact locations
 pysftp==0.2.9
+# Required to run the MLflow server against SQL-backed storage
+sqlalchemy==1.3.0
+# Required by the mlflow.projects module, when running projects against
+# a remote Kubernetes cluster
+kubernetes==9.0.0
+## Test-only dependencies
 attrdict==2.0.0
-cloudpickle==0.8.0
 pytest==3.2.1
 pytest-cov==2.6.0
 pytest-localserver==0.5.0
-sqlalchemy==1.3.0
-kubernetes==9.0.0
-# test plugin
+scipy==1.2.1
+moto==1.3.7
+# Test plugin, used to verify correctness of MLflow plugin APIs
 tests/resources/mlflow-test-plugin/

docs/source/quickstart.rst

@@ -25,9 +25,10 @@ You install MLflow by running:
     installing Python 3 through the `Homebrew <https://brew.sh/>`_ package manager using
     ``brew install python``. (In this case, installing MLflow is now ``pip3 install mlflow``).
 
-In order to use some of MLflow features (ML librairies, storage options, ...), you may need to install extra libraries.
-For example, the ``mlflow.tensorflow`` module requires TensorFlow to be installed.
-Please refer to this `requirements.txt <https://github.com/mlflow/mlflow/blob/master/dev/large-requirements.txt>`_ for a complete list of dependencies.
+To use certain MLflow modules and functionality (ML model persistence/inference, artifact storage options, etc),
+you may need to install extra libraries. For example, the ``mlflow.tensorflow`` module requires TensorFlow to be installed.
+See https://github.com/mlflow/mlflow/blob/master/EXTRA_DEPENDENCIES.rst for more details
+
 At this point we recommend you follow the :doc:`tutorial<tutorials-and-examples/tutorial>` for a walk-through on how you
 can leverage MLflow in your daily workflow.
 

test-requirements.txt

@@ -1,3 +1,4 @@
 -r dev/small-requirements.txt
 -r dev/large-requirements.txt
+-r dev/extra-ml-requirements.txt
 -r dev/lint-requirements.txt

tests/models/test_cli.py

@@ -64,6 +64,7 @@ def sk_model(iris_data):
     return knn_model
 
 
+@pytest.mark.large
 def test_predict_with_old_mlflow_in_conda_and_with_orient_records(iris_data):
     if no_conda:
         pytest.skip("This test needs conda.")
@@ -114,6 +115,7 @@ def test_predict_with_old_mlflow_in_conda_and_with_orient_records(iris_data):
         assert all(expected == actual)
 
 
+@pytest.mark.large
 def test_mlflow_is_not_installed_unless_specified():
     if no_conda:
         pytest.skip("This test requires conda.")
@@ -138,6 +140,7 @@ def test_mlflow_is_not_installed_unless_specified():
             assert "ImportError: No module named mlflow.pyfunc.scoring_server" in stderr
 
 
+@pytest.mark.large
 def test_model_with_no_deployable_flavors_fails_pollitely():
     from mlflow.models import Model