From ed970ee543fb41440cbacf6bbcf9495e57424415 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 29 Jul 2024 11:58:02 +0200
Subject: [PATCH 01/25] Start migration to mkdocs
---
docs/api.md | 224 +++
docs/contributing.md | 29 +
docs/examples/20_basic/index.rst.md | 65 +
.../20_basic/introduction_tutorial.rst.md | 229 +++
.../20_basic/sg_execution_times.rst.md | 21 +
.../20_basic/simple_datasets_tutorial.rst.md | 173 +++
.../simple_flows_and_runs_tutorial.rst.md | 174 +++
.../20_basic/simple_suites_tutorial.rst.md | 188 +++
.../30_extended/configure_logging.rst.md | 107 ++
.../30_extended/create_upload_tutorial.rst.md | 460 ++++++
docs/examples/30_extended/custom_flow_.rst.md | 294 ++++
.../30_extended/datasets_tutorial.rst.md | 1040 +++++++++++++
.../fetch_evaluations_tutorial.rst.md | 360 +++++
.../fetch_runtimes_tutorial.rst.md | 1135 +++++++++++++++
.../30_extended/flow_id_tutorial.rst.md | 208 +++
.../flows_and_runs_tutorial.rst.md | 604 ++++++++
docs/examples/30_extended/index.rst.md | 205 +++
.../plot_svm_hyperparameters_tutorial.rst.md | 235 +++
.../30_extended/run_setup_tutorial.rst.md | 215 +++
.../30_extended/sg_execution_times.rst.md | 51 +
.../30_extended/study_tutorial.rst.md | 402 +++++
.../30_extended/suites_tutorial.rst.md | 291 ++++
.../task_manual_iteration_tutorial.rst.md | 522 +++++++
.../30_extended/tasks_tutorial.rst.md | 1292 +++++++++++++++++
.../2015_neurips_feurer_example.rst.md | 167 +++
.../40_paper/2018_ida_strang_example.rst.md | 191 +++
docs/extensions.md | 183 +++
.../openml.datasets.OpenMLDataFeature.rst.md | 13 +
.../openml.datasets.OpenMLDataset.rst.md | 13 +
...ml.datasets.attributes_arff_from_df.rst.md | 11 +
...enml.datasets.check_datasets_active.rst.md | 11 +
.../openml.datasets.create_dataset.rst.md | 11 +
.../openml.datasets.delete_dataset.rst.md | 11 +
.../openml.datasets.edit_dataset.rst.md | 11 +
.../openml.datasets.fork_dataset.rst.md | 11 +
.../openml.datasets.get_dataset.rst.md | 11 +
.../openml.datasets.get_datasets.rst.md | 11 +
.../openml.datasets.list_datasets.rst.md | 11 +
.../openml.datasets.list_qualities.rst.md | 11 +
.../openml.datasets.status_update.rst.md | 11 +
...openml.evaluations.OpenMLEvaluation.rst.md | 13 +
...valuations.list_evaluation_measures.rst.md | 11 +
...openml.evaluations.list_evaluations.rst.md | 11 +
...evaluations.list_evaluations_setups.rst.md | 11 +
.../openml.extensions.Extension.rst.md | 13 +
...ml.extensions.get_extension_by_flow.rst.md | 11 +
...l.extensions.get_extension_by_model.rst.md | 11 +
...penml.extensions.register_extension.rst.md | 11 +
...extensions.sklearn.SklearnExtension.rst.md | 13 +
docs/generated/openml.flows.OpenMLFlow.rst.md | 13 +
.../openml.flows.assert_flows_equal.rst.md | 11 +
.../generated/openml.flows.delete_flow.rst.md | 11 +
.../generated/openml.flows.flow_exists.rst.md | 11 +
docs/generated/openml.flows.get_flow.rst.md | 11 +
docs/generated/openml.flows.list_flows.rst.md | 11 +
docs/generated/openml.runs.OpenMLRun.rst.md | 13 +
docs/generated/openml.runs.delete_run.rst.md | 11 +
docs/generated/openml.runs.get_run.rst.md | 11 +
.../openml.runs.get_run_trace.rst.md | 11 +
docs/generated/openml.runs.get_runs.rst.md | 11 +
...enml.runs.initialize_model_from_run.rst.md | 11 +
...ml.runs.initialize_model_from_trace.rst.md | 11 +
docs/generated/openml.runs.list_runs.rst.md | 11 +
docs/generated/openml.runs.run_exists.rst.md | 11 +
.../openml.runs.run_flow_on_task.rst.md | 11 +
.../openml.runs.run_model_on_task.rst.md | 11 +
.../openml.setups.OpenMLParameter.rst.md | 13 +
.../openml.setups.OpenMLSetup.rst.md | 13 +
docs/generated/openml.setups.get_setup.rst.md | 11 +
.../openml.setups.initialize_model.rst.md | 11 +
.../openml.setups.list_setups.rst.md | 11 +
.../openml.setups.setup_exists.rst.md | 11 +
.../openml.study.OpenMLBenchmarkSuite.rst.md | 13 +
.../generated/openml.study.OpenMLStudy.rst.md | 13 +
.../openml.study.attach_to_study.rst.md | 11 +
.../openml.study.attach_to_suite.rst.md | 11 +
...openml.study.create_benchmark_suite.rst.md | 11 +
.../openml.study.create_study.rst.md | 11 +
.../openml.study.delete_study.rst.md | 11 +
.../openml.study.delete_suite.rst.md | 11 +
.../openml.study.detach_from_study.rst.md | 11 +
.../openml.study.detach_from_suite.rst.md | 11 +
docs/generated/openml.study.get_study.rst.md | 11 +
docs/generated/openml.study.get_suite.rst.md | 11 +
.../openml.study.list_studies.rst.md | 11 +
.../generated/openml.study.list_suites.rst.md | 11 +
.../openml.study.update_study_status.rst.md | 11 +
.../openml.study.update_suite_status.rst.md | 11 +
...enml.tasks.OpenMLClassificationTask.rst.md | 13 +
.../openml.tasks.OpenMLClusteringTask.rst.md | 13 +
...penml.tasks.OpenMLLearningCurveTask.rst.md | 13 +
.../openml.tasks.OpenMLRegressionTask.rst.md | 13 +
.../generated/openml.tasks.OpenMLSplit.rst.md | 13 +
.../openml.tasks.OpenMLSupervisedTask.rst.md | 13 +
docs/generated/openml.tasks.OpenMLTask.rst.md | 13 +
docs/generated/openml.tasks.TaskType.rst.md | 13 +
.../generated/openml.tasks.create_task.rst.md | 11 +
.../generated/openml.tasks.delete_task.rst.md | 11 +
docs/generated/openml.tasks.get_task.rst.md | 11 +
docs/generated/openml.tasks.get_tasks.rst.md | 11 +
docs/generated/openml.tasks.list_tasks.rst.md | 11 +
docs/index.md | 99 ++
docs/progress.md | 493 +++++++
docs/usage.md | 206 +++
mkdocs.yml | 13 +
pyproject.toml | 2 +
106 files changed, 10730 insertions(+)
create mode 100644 docs/api.md
create mode 100644 docs/contributing.md
create mode 100644 docs/examples/20_basic/index.rst.md
create mode 100644 docs/examples/20_basic/introduction_tutorial.rst.md
create mode 100644 docs/examples/20_basic/sg_execution_times.rst.md
create mode 100644 docs/examples/20_basic/simple_datasets_tutorial.rst.md
create mode 100644 docs/examples/20_basic/simple_flows_and_runs_tutorial.rst.md
create mode 100644 docs/examples/20_basic/simple_suites_tutorial.rst.md
create mode 100644 docs/examples/30_extended/configure_logging.rst.md
create mode 100644 docs/examples/30_extended/create_upload_tutorial.rst.md
create mode 100644 docs/examples/30_extended/custom_flow_.rst.md
create mode 100644 docs/examples/30_extended/datasets_tutorial.rst.md
create mode 100644 docs/examples/30_extended/fetch_evaluations_tutorial.rst.md
create mode 100644 docs/examples/30_extended/fetch_runtimes_tutorial.rst.md
create mode 100644 docs/examples/30_extended/flow_id_tutorial.rst.md
create mode 100644 docs/examples/30_extended/flows_and_runs_tutorial.rst.md
create mode 100644 docs/examples/30_extended/index.rst.md
create mode 100644 docs/examples/30_extended/plot_svm_hyperparameters_tutorial.rst.md
create mode 100644 docs/examples/30_extended/run_setup_tutorial.rst.md
create mode 100644 docs/examples/30_extended/sg_execution_times.rst.md
create mode 100644 docs/examples/30_extended/study_tutorial.rst.md
create mode 100644 docs/examples/30_extended/suites_tutorial.rst.md
create mode 100644 docs/examples/30_extended/task_manual_iteration_tutorial.rst.md
create mode 100644 docs/examples/30_extended/tasks_tutorial.rst.md
create mode 100644 docs/examples/40_paper/2015_neurips_feurer_example.rst.md
create mode 100644 docs/examples/40_paper/2018_ida_strang_example.rst.md
create mode 100644 docs/extensions.md
create mode 100644 docs/generated/openml.datasets.OpenMLDataFeature.rst.md
create mode 100644 docs/generated/openml.datasets.OpenMLDataset.rst.md
create mode 100644 docs/generated/openml.datasets.attributes_arff_from_df.rst.md
create mode 100644 docs/generated/openml.datasets.check_datasets_active.rst.md
create mode 100644 docs/generated/openml.datasets.create_dataset.rst.md
create mode 100644 docs/generated/openml.datasets.delete_dataset.rst.md
create mode 100644 docs/generated/openml.datasets.edit_dataset.rst.md
create mode 100644 docs/generated/openml.datasets.fork_dataset.rst.md
create mode 100644 docs/generated/openml.datasets.get_dataset.rst.md
create mode 100644 docs/generated/openml.datasets.get_datasets.rst.md
create mode 100644 docs/generated/openml.datasets.list_datasets.rst.md
create mode 100644 docs/generated/openml.datasets.list_qualities.rst.md
create mode 100644 docs/generated/openml.datasets.status_update.rst.md
create mode 100644 docs/generated/openml.evaluations.OpenMLEvaluation.rst.md
create mode 100644 docs/generated/openml.evaluations.list_evaluation_measures.rst.md
create mode 100644 docs/generated/openml.evaluations.list_evaluations.rst.md
create mode 100644 docs/generated/openml.evaluations.list_evaluations_setups.rst.md
create mode 100644 docs/generated/openml.extensions.Extension.rst.md
create mode 100644 docs/generated/openml.extensions.get_extension_by_flow.rst.md
create mode 100644 docs/generated/openml.extensions.get_extension_by_model.rst.md
create mode 100644 docs/generated/openml.extensions.register_extension.rst.md
create mode 100644 docs/generated/openml.extensions.sklearn.SklearnExtension.rst.md
create mode 100644 docs/generated/openml.flows.OpenMLFlow.rst.md
create mode 100644 docs/generated/openml.flows.assert_flows_equal.rst.md
create mode 100644 docs/generated/openml.flows.delete_flow.rst.md
create mode 100644 docs/generated/openml.flows.flow_exists.rst.md
create mode 100644 docs/generated/openml.flows.get_flow.rst.md
create mode 100644 docs/generated/openml.flows.list_flows.rst.md
create mode 100644 docs/generated/openml.runs.OpenMLRun.rst.md
create mode 100644 docs/generated/openml.runs.delete_run.rst.md
create mode 100644 docs/generated/openml.runs.get_run.rst.md
create mode 100644 docs/generated/openml.runs.get_run_trace.rst.md
create mode 100644 docs/generated/openml.runs.get_runs.rst.md
create mode 100644 docs/generated/openml.runs.initialize_model_from_run.rst.md
create mode 100644 docs/generated/openml.runs.initialize_model_from_trace.rst.md
create mode 100644 docs/generated/openml.runs.list_runs.rst.md
create mode 100644 docs/generated/openml.runs.run_exists.rst.md
create mode 100644 docs/generated/openml.runs.run_flow_on_task.rst.md
create mode 100644 docs/generated/openml.runs.run_model_on_task.rst.md
create mode 100644 docs/generated/openml.setups.OpenMLParameter.rst.md
create mode 100644 docs/generated/openml.setups.OpenMLSetup.rst.md
create mode 100644 docs/generated/openml.setups.get_setup.rst.md
create mode 100644 docs/generated/openml.setups.initialize_model.rst.md
create mode 100644 docs/generated/openml.setups.list_setups.rst.md
create mode 100644 docs/generated/openml.setups.setup_exists.rst.md
create mode 100644 docs/generated/openml.study.OpenMLBenchmarkSuite.rst.md
create mode 100644 docs/generated/openml.study.OpenMLStudy.rst.md
create mode 100644 docs/generated/openml.study.attach_to_study.rst.md
create mode 100644 docs/generated/openml.study.attach_to_suite.rst.md
create mode 100644 docs/generated/openml.study.create_benchmark_suite.rst.md
create mode 100644 docs/generated/openml.study.create_study.rst.md
create mode 100644 docs/generated/openml.study.delete_study.rst.md
create mode 100644 docs/generated/openml.study.delete_suite.rst.md
create mode 100644 docs/generated/openml.study.detach_from_study.rst.md
create mode 100644 docs/generated/openml.study.detach_from_suite.rst.md
create mode 100644 docs/generated/openml.study.get_study.rst.md
create mode 100644 docs/generated/openml.study.get_suite.rst.md
create mode 100644 docs/generated/openml.study.list_studies.rst.md
create mode 100644 docs/generated/openml.study.list_suites.rst.md
create mode 100644 docs/generated/openml.study.update_study_status.rst.md
create mode 100644 docs/generated/openml.study.update_suite_status.rst.md
create mode 100644 docs/generated/openml.tasks.OpenMLClassificationTask.rst.md
create mode 100644 docs/generated/openml.tasks.OpenMLClusteringTask.rst.md
create mode 100644 docs/generated/openml.tasks.OpenMLLearningCurveTask.rst.md
create mode 100644 docs/generated/openml.tasks.OpenMLRegressionTask.rst.md
create mode 100644 docs/generated/openml.tasks.OpenMLSplit.rst.md
create mode 100644 docs/generated/openml.tasks.OpenMLSupervisedTask.rst.md
create mode 100644 docs/generated/openml.tasks.OpenMLTask.rst.md
create mode 100644 docs/generated/openml.tasks.TaskType.rst.md
create mode 100644 docs/generated/openml.tasks.create_task.rst.md
create mode 100644 docs/generated/openml.tasks.delete_task.rst.md
create mode 100644 docs/generated/openml.tasks.get_task.rst.md
create mode 100644 docs/generated/openml.tasks.get_tasks.rst.md
create mode 100644 docs/generated/openml.tasks.list_tasks.rst.md
create mode 100644 docs/index.md
create mode 100644 docs/progress.md
create mode 100644 docs/usage.md
create mode 100644 mkdocs.yml
diff --git a/docs/api.md b/docs/api.md
new file mode 100644
index 000000000..5ce9448e6
--- /dev/null
+++ b/docs/api.md
@@ -0,0 +1,224 @@
+orphan
+
+:
+
+# API
+
+## Modules
+
+### `openml.datasets`{.interpreted-text role="mod"}
+
+::: {.automodule no-members="" no-inherited-members=""}
+openml.datasets
+:::
+
+#### Dataset Classes
+
+::: currentmodule
+openml.datasets
+:::
+
+::: {.autosummary toctree="generated/" template="class.rst"}
+OpenMLDataFeature OpenMLDataset
+:::
+
+#### Dataset Functions
+
+::: currentmodule
+openml.datasets
+:::
+
+::: {.autosummary toctree="generated/" template="function.rst"}
+attributes_arff_from_df check_datasets_active create_dataset
+delete_dataset get_dataset get_datasets list_datasets list_qualities
+status_update edit_dataset fork_dataset
+:::
+
+### `openml.evaluations`{.interpreted-text role="mod"}
+
+::: {.automodule no-members="" no-inherited-members=""}
+openml.evaluations
+:::
+
+#### Evaluations Classes
+
+::: currentmodule
+openml.evaluations
+:::
+
+::: {.autosummary toctree="generated/" template="class.rst"}
+OpenMLEvaluation
+:::
+
+#### Evaluations Functions
+
+::: currentmodule
+openml.evaluations
+:::
+
+::: {.autosummary toctree="generated/" template="function.rst"}
+list_evaluations list_evaluation_measures list_evaluations_setups
+:::
+
+### `openml.flows`{.interpreted-text role="mod"}: Flow Functions
+
+::: {.automodule no-members="" no-inherited-members=""}
+openml.flows
+:::
+
+#### Flow Classes
+
+::: currentmodule
+openml.flows
+:::
+
+::: {.autosummary toctree="generated/" template="class.rst"}
+OpenMLFlow
+:::
+
+#### Flow Functions
+
+::: currentmodule
+openml.flows
+:::
+
+::: {.autosummary toctree="generated/" template="function.rst"}
+assert_flows_equal delete_flow flow_exists get_flow list_flows
+:::
+
+### `openml.runs`{.interpreted-text role="mod"}: Run Functions
+
+::: {.automodule no-members="" no-inherited-members=""}
+openml.runs
+:::
+
+#### Run Classes
+
+::: currentmodule
+openml.runs
+:::
+
+::: {.autosummary toctree="generated/" template="class.rst"}
+OpenMLRun
+:::
+
+#### Run Functions
+
+::: currentmodule
+openml.runs
+:::
+
+::: {.autosummary toctree="generated/" template="function.rst"}
+delete_run get_run get_runs get_run_trace initialize_model_from_run
+initialize_model_from_trace list_runs run_model_on_task run_flow_on_task
+run_exists
+:::
+
+### `openml.setups`{.interpreted-text role="mod"}: Setup Functions
+
+::: {.automodule no-members="" no-inherited-members=""}
+openml.setups
+:::
+
+#### Setup Classes
+
+::: currentmodule
+openml.setups
+:::
+
+::: {.autosummary toctree="generated/" template="class.rst"}
+OpenMLParameter OpenMLSetup
+:::
+
+#### Setup Functions
+
+::: currentmodule
+openml.setups
+:::
+
+::: {.autosummary toctree="generated/" template="function.rst"}
+get_setup initialize_model list_setups setup_exists
+:::
+
+### `openml.study`{.interpreted-text role="mod"}: Study Functions
+
+::: {.automodule no-members="" no-inherited-members=""}
+openml.study
+:::
+
+#### Study Classes
+
+::: currentmodule
+openml.study
+:::
+
+::: {.autosummary toctree="generated/" template="class.rst"}
+OpenMLBenchmarkSuite OpenMLStudy
+:::
+
+#### Study Functions
+
+::: currentmodule
+openml.study
+:::
+
+::: {.autosummary toctree="generated/" template="function.rst"}
+attach_to_study attach_to_suite create_benchmark_suite create_study
+delete_study delete_suite detach_from_study detach_from_suite get_study
+get_suite list_studies list_suites update_study_status
+update_suite_status
+:::
+
+### `openml.tasks`{.interpreted-text role="mod"}: Task Functions
+
+::: {.automodule no-members="" no-inherited-members=""}
+openml.tasks
+:::
+
+#### Task Classes
+
+::: currentmodule
+openml.tasks
+:::
+
+::: {.autosummary toctree="generated/" template="class.rst"}
+OpenMLClassificationTask OpenMLClusteringTask OpenMLLearningCurveTask
+OpenMLRegressionTask OpenMLSplit OpenMLSupervisedTask OpenMLTask
+TaskType
+:::
+
+#### Task Functions
+
+::: currentmodule
+openml.tasks
+:::
+
+::: {.autosummary toctree="generated/" template="function.rst"}
+create_task delete_task get_task get_tasks list_tasks
+:::
+
+## Extensions {#api_extensions}
+
+::: {.automodule no-members="" no-inherited-members=""}
+openml.extensions
+:::
+
+### Extension Classes
+
+::: currentmodule
+openml.extensions
+:::
+
+::: {.autosummary toctree="generated/" template="class.rst"}
+Extension sklearn.SklearnExtension
+:::
+
+### Extension Functions
+
+::: currentmodule
+openml.extensions
+:::
+
+::: {.autosummary toctree="generated/" template="function.rst"}
+get_extension_by_flow get_extension_by_model register_extension
+:::
diff --git a/docs/contributing.md b/docs/contributing.md
new file mode 100644
index 000000000..93cbbe9d9
--- /dev/null
+++ b/docs/contributing.md
@@ -0,0 +1,29 @@
+orphan
+
+:
+
+# Contributing
+
+Contribution to the OpenML package is highly appreciated in all forms.
+In particular, a few ways to contribute to openml-python are:
+
+> - A direct contribution to the package, by means of improving the
+> code, documentation or examples. To get started, see [this
+> file](https://github.com/openml/openml-python/blob/main/CONTRIBUTING.md)
+> with details on how to set up your environment to develop for
+> openml-python.
+> - A contribution to an openml-python extension. An extension package
+> allows OpenML to interface with a machine learning package (such
+> as scikit-learn or keras). These extensions are hosted in separate
+> repositories and may have their own guidelines. For more
+> information, see the `extensions`{.interpreted-text role="ref"}
+> below.
+> - Bug reports. If something doesn\'t work for you or is cumbersome,
+> please open a new issue to let us know about the problem. See
+> [this
+> section](https://github.com/openml/openml-python/blob/main/CONTRIBUTING.md).
+> - [Cite OpenML](https://www.openml.org/cite) if you use it in a
+> scientific publication.
+> - Visit one of our [hackathons](https://www.openml.org/meet).
+> - Contribute to another OpenML project, such as [the main OpenML
+> project](https://github.com/openml/OpenML/blob/master/CONTRIBUTING.md).
diff --git a/docs/examples/20_basic/index.rst.md b/docs/examples/20_basic/index.rst.md
new file mode 100644
index 000000000..2f29baa0f
--- /dev/null
+++ b/docs/examples/20_basic/index.rst.md
@@ -0,0 +1,65 @@
+# Introductory Examples {#sphx_glr_examples_20_basic}
+
+Introductory examples to the usage of the OpenML python connector.
+
+<div class="sphx-glr-thumbnails">
+<div class="sphx-glr-thumbcontainer" tooltip="This is a brief showcase of OpenML benchmark suites, which were introduced by Bischl et al. (20...">
+
+::: only
+html
+
+
+
+`sphx_glr_examples_20_basic_simple_suites_tutorial.py`{.interpreted-text
+role="ref"}
+:::
+
+<div class="sphx-glr-thumbnail-title">Benchmark suites</div>
+</div>
+<div class="sphx-glr-thumbcontainer" tooltip="A simple tutorial on how to train/run a model and how to upload the results.">
+
+::: only
+html
+
+
+
+`sphx_glr_examples_20_basic_simple_flows_and_runs_tutorial.py`{.interpreted-text
+role="ref"}
+:::
+
+<div class="sphx-glr-thumbnail-title">Flows and Runs</div>
+</div>
+<div class="sphx-glr-thumbcontainer" tooltip="An example how to set up OpenML-Python followed up by a simple example.">
+
+::: only
+html
+
+
+
+`sphx_glr_examples_20_basic_introduction_tutorial.py`{.interpreted-text
+role="ref"}
+:::
+
+<div class="sphx-glr-thumbnail-title">Introduction tutorial & Setup</div>
+</div>
+<div class="sphx-glr-thumbcontainer" tooltip="A basic tutorial on how to list, load and visualize datasets.">
+
+::: only
+html
+
+
+
+`sphx_glr_examples_20_basic_simple_datasets_tutorial.py`{.interpreted-text
+role="ref"}
+:::
+
+<div class="sphx-glr-thumbnail-title">Datasets</div>
+</div>
+</div>
+
+::: {.toctree hidden=""}
+/examples/20_basic/simple_suites_tutorial
+/examples/20_basic/simple_flows_and_runs_tutorial
+/examples/20_basic/introduction_tutorial
+/examples/20_basic/simple_datasets_tutorial
+:::
diff --git a/docs/examples/20_basic/introduction_tutorial.rst.md b/docs/examples/20_basic/introduction_tutorial.rst.md
new file mode 100644
index 000000000..56903b1f0
--- /dev/null
+++ b/docs/examples/20_basic/introduction_tutorial.rst.md
@@ -0,0 +1,229 @@
+::::: only
+html
+
+:::: {.note .sphx-glr-download-link-note}
+::: title
+Note
+:::
+
+`Go to the end <sphx_glr_download_examples_20_basic_introduction_tutorial.py>`{.interpreted-text
+role="ref"} to download the full example code
+::::
+:::::
+
+::: rst-class
+sphx-glr-example-title
+:::
+
+# Introduction tutorial & Setup {#sphx_glr_examples_20_basic_introduction_tutorial.py}
+
+An example how to set up OpenML-Python followed up by a simple example.
+
+OpenML is an online collaboration platform for machine learning which
+allows you to:
+
+- Find or share interesting, well-documented datasets
+- Define research / modelling goals (tasks)
+- Explore large amounts of machine learning algorithms, with APIs in
+ Java, R, Python
+- Log and share reproducible experiments, models, results
+- Works seamlessly with scikit-learn and other libraries
+- Large scale benchmarking, compare to state of the art
+
+## Installation
+
+Installation is done via `pip`:
+
+``` bash
+pip install openml
+```
+
+For further information, please check out the installation guide at
+`installation`{.interpreted-text role="ref"}.
+
+## Authentication
+
+The OpenML server can only be accessed by users who have signed up on
+the OpenML platform. If you don't have an account yet, sign up now. You
+will receive an API key, which will authenticate you to the server and
+allow you to download and upload datasets, tasks, runs and flows.
+
+- Create an OpenML account (free) on <https://www.openml.org>.
+- After logging in, open your account page (avatar on the top right)
+- Open \'Account Settings\', then \'API authentication\' to find your
+ API key.
+
+There are two ways to permanently authenticate:
+
+- Use the `openml` CLI tool with `openml configure apikey MYKEY`,
+ replacing **MYKEY** with your API key.
+- Create a plain text file **\~/.openml/config** with the line
+ **\'apikey=MYKEY\'**, replacing **MYKEY** with your API key. The
+ config file must be in the directory \~/.openml/config and exist
+ prior to importing the openml module.
+
+Alternatively, by running the code below and replacing \'YOURKEY\' with
+your API key, you authenticate for the duration of the python process.
+
+``` default
+# License: BSD 3-Clause
+
+import openml
+from sklearn import neighbors
+```
+
+:::: warning
+::: title
+Warning
+:::
+
+.. include:: ../../test_server_usage_warning.txt
+::::
+
+``` default
+openml.config.start_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!
+ warnings.warn(
+```
+:::
+
+When using the main server instead, make sure your apikey is configured.
+This can be done with the following line of code (uncomment it!). Never
+share your apikey with others.
+
+``` default
+# openml.config.apikey = 'YOURKEY'
+```
+
+## Caching
+
+When downloading datasets, tasks, runs and flows, they will be cached to
+retrieve them without calling the server later. As with the API key, the
+cache directory can be either specified through the config file or
+through the API:
+
+- Add the line **cachedir = \'MYDIR\'** to the config file, replacing
+ \'MYDIR\' with the path to the cache directory. By default, OpenML
+ will use **\~/.openml/cache** as the cache directory.
+- Run the code below, replacing \'YOURDIR\' with the path to the cache
+ directory.
+
+``` default
+# Uncomment and set your OpenML cache directory
+# import os
+# openml.config.cache_directory = os.path.expanduser('YOURDIR')
+```
+
+## Simple Example
+
+Download the OpenML task for the eeg-eye-state.
+
+``` default
+task = openml.tasks.get_task(403)
+data = openml.datasets.get_dataset(task.dataset_id)
+clf = neighbors.KNeighborsClassifier(n_neighbors=5)
+run = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False)
+# Publish the experiment on OpenML (optional, requires an API key).
+# For this tutorial, our configuration publishes to the test server
+# as to not crowd the main server with runs created by examples.
+myrun = run.publish()
+print(f"kNN on {data.name}: {myrun.openml_url}")
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+kNN on eeg-eye-state: https://test.openml.org/r/7327
+```
+:::
+
+``` default
+openml.config.stop_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-timing
+
+**Total running time of the script:** ( 0 minutes 8.000 seconds)
+:::
+
+::::::: {#sphx_glr_download_examples_20_basic_introduction_tutorial.py}
+:::::: only
+html
+
+::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
+::: {.container .sphx-glr-download .sphx-glr-download-python}
+`Download Python source code: introduction_tutorial.py <introduction_tutorial.py>`{.interpreted-text
+role="download"}
+:::
+
+::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
+`Download Jupyter notebook: introduction_tutorial.ipynb <introduction_tutorial.ipynb>`{.interpreted-text
+role="download"}
+:::
+:::::
+::::::
+:::::::
+
+:::: only
+html
+
+::: rst-class
+sphx-glr-signature
+
+[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
+:::
+::::
diff --git a/docs/examples/20_basic/sg_execution_times.rst.md b/docs/examples/20_basic/sg_execution_times.rst.md
new file mode 100644
index 000000000..5017d8cee
--- /dev/null
+++ b/docs/examples/20_basic/sg_execution_times.rst.md
@@ -0,0 +1,21 @@
+orphan
+
+:
+
+# Computation times {#sphx_glr_examples_20_basic_sg_execution_times}
+
+**00:27.928** total execution time for **examples_20_basic** files:
+
+ ---------------------------------------------------------------------------------- ------------ -----
+ `sphx_glr_examples_20_basic_simple_suites_tutorial.py`{.interpreted-text 00:09.728 0.0
+ role="ref"} (`simple_suites_tutorial.py`) MB
+
+ `sphx_glr_examples_20_basic_introduction_tutorial.py`{.interpreted-text 00:07.1000 0.0
+ role="ref"} (`introduction_tutorial.py`) MB
+
+ `sphx_glr_examples_20_basic_simple_datasets_tutorial.py`{.interpreted-text 00:06.402 0.0
+ role="ref"} (`simple_datasets_tutorial.py`) MB
+
+ `sphx_glr_examples_20_basic_simple_flows_and_runs_tutorial.py`{.interpreted-text 00:03.799 0.0
+ role="ref"} (`simple_flows_and_runs_tutorial.py`) MB
+ ---------------------------------------------------------------------------------- ------------ -----
diff --git a/docs/examples/20_basic/simple_datasets_tutorial.rst.md b/docs/examples/20_basic/simple_datasets_tutorial.rst.md
new file mode 100644
index 000000000..326ca1f2d
--- /dev/null
+++ b/docs/examples/20_basic/simple_datasets_tutorial.rst.md
@@ -0,0 +1,173 @@
+::::: only
+html
+
+:::: {.note .sphx-glr-download-link-note}
+::: title
+Note
+:::
+
+`Go to the end <sphx_glr_download_examples_20_basic_simple_datasets_tutorial.py>`{.interpreted-text
+role="ref"} to download the full example code
+::::
+:::::
+
+::: rst-class
+sphx-glr-example-title
+:::
+
+# Datasets {#sphx_glr_examples_20_basic_simple_datasets_tutorial.py}
+
+A basic tutorial on how to list, load and visualize datasets.
+
+In general, we recommend working with tasks, so that the results can be
+easily reproduced. Furthermore, the results can be compared to existing
+results at OpenML. However, for the purposes of this tutorial, we are
+going to work with the datasets directly.
+
+``` default
+# License: BSD 3-Clause
+
+import openml
+```
+
+## List datasets
+
+``` default
+datasets_df = openml.datasets.list_datasets(output_format="dataframe")
+print(datasets_df.head(n=10))
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+did name ... NumberOfNumericFeatures NumberOfSymbolicFeatures
+2 2 anneal ... 6.0 33.0
+3 3 kr-vs-kp ... 0.0 37.0
+4 4 labor ... 8.0 9.0
+5 5 arrhythmia ... 206.0 74.0
+6 6 letter ... 16.0 1.0
+7 7 audiology ... 0.0 70.0
+8 8 liver-disorders ... 6.0 0.0
+9 9 autos ... 15.0 11.0
+10 10 lymph ... 3.0 16.0
+11 11 balance-scale ... 4.0 1.0
+
+[10 rows x 16 columns]
+```
+:::
+
+## Download a dataset
+
+``` default
+# Iris dataset https://www.openml.org/d/61
+dataset = openml.datasets.get_dataset(61)
+
+# Print a summary
+print(
+ f"This is dataset '{dataset.name}', the target feature is "
+ f"'{dataset.default_target_attribute}'"
+)
+print(f"URL: {dataset.url}")
+print(dataset.description[:500])
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+This is dataset 'iris', the target feature is 'class'
+URL: https://api.openml.org/data/v1/download/61/iris.arff
+**Author**: R.A. Fisher
+**Source**: [UCI](https://archive.ics.uci.edu/ml/datasets/Iris) - 1936 - Donated by Michael Marshall
+**Please cite**:
+
+**Iris Plants Database**
+This is perhaps the best known database to be found in the pattern recognition literature. Fisher's paper is a classic in the field and is referenced frequently to this day. (See Duda & Hart, for example.) The data set contains 3 classes of 50 instances each, where each class refers to a type of iris plant. One class
+```
+:::
+
+## Load a dataset
+
+``` default
+# X - An array/dataframe where each row represents one example with
+# the corresponding feature values.
+# y - the classes for each example
+# categorical_indicator - an array that indicates which feature is categorical
+# attribute_names - the names of the features for the examples (X) and
+# target feature (y)
+X, y, categorical_indicator, attribute_names = dataset.get_data(
+ dataset_format="dataframe", target=dataset.default_target_attribute
+)
+```
+
+## Visualize the dataset
+
+``` default
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+
+sns.set_style("darkgrid")
+
+
+def hide_current_axis(*args, **kwds):
+ plt.gca().set_visible(False)
+
+
+# We combine all the data so that we can map the different
+# examples to different colors according to the classes.
+combined_data = pd.concat([X, y], axis=1)
+iris_plot = sns.pairplot(combined_data, hue="class")
+iris_plot.map_upper(hide_current_axis)
+plt.show()
+```
+
+::: {.image-sg .sphx-glr-single-img alt="simple datasets tutorial" srcset="/examples/20_basic/images/sphx_glr_simple_datasets_tutorial_001.png"}
+/examples/20_basic/images/sphx_glr_simple_datasets_tutorial_001.png
+:::
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/openml/venv/lib/python3.10/site-packages/seaborn/axisgrid.py:118: UserWarning: The figure layout has changed to tight
+ self._figure.tight_layout(*args, **kwargs)
+```
+:::
+
+::: rst-class
+sphx-glr-timing
+
+**Total running time of the script:** ( 0 minutes 6.402 seconds)
+:::
+
+::::::: {#sphx_glr_download_examples_20_basic_simple_datasets_tutorial.py}
+:::::: only
+html
+
+::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
+::: {.container .sphx-glr-download .sphx-glr-download-python}
+`Download Python source code: simple_datasets_tutorial.py <simple_datasets_tutorial.py>`{.interpreted-text
+role="download"}
+:::
+
+::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
+`Download Jupyter notebook: simple_datasets_tutorial.ipynb <simple_datasets_tutorial.ipynb>`{.interpreted-text
+role="download"}
+:::
+:::::
+::::::
+:::::::
+
+:::: only
+html
+
+::: rst-class
+sphx-glr-signature
+
+[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
+:::
+::::
diff --git a/docs/examples/20_basic/simple_flows_and_runs_tutorial.rst.md b/docs/examples/20_basic/simple_flows_and_runs_tutorial.rst.md
new file mode 100644
index 000000000..960d65e37
--- /dev/null
+++ b/docs/examples/20_basic/simple_flows_and_runs_tutorial.rst.md
@@ -0,0 +1,174 @@
+::::: only
+html
+
+:::: {.note .sphx-glr-download-link-note}
+::: title
+Note
+:::
+
+`Go to the end <sphx_glr_download_examples_20_basic_simple_flows_and_runs_tutorial.py>`{.interpreted-text
+role="ref"} to download the full example code
+::::
+:::::
+
+::: rst-class
+sphx-glr-example-title
+:::
+
+# Flows and Runs {#sphx_glr_examples_20_basic_simple_flows_and_runs_tutorial.py}
+
+A simple tutorial on how to train/run a model and how to upload the
+results.
+
+``` default
+# License: BSD 3-Clause
+
+import openml
+from sklearn import ensemble, neighbors
+```
+
+:::: warning
+::: title
+Warning
+:::
+
+.. include:: ../../test_server_usage_warning.txt
+::::
+
+``` default
+openml.config.start_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!
+ warnings.warn(
+```
+:::
+
+# Train a machine learning model
+
+``` default
+# NOTE: We are using dataset 20 from the test server: https://test.openml.org/d/20
+dataset = openml.datasets.get_dataset(20)
+X, y, categorical_indicator, attribute_names = dataset.get_data(
+ target=dataset.default_target_attribute
+)
+clf = neighbors.KNeighborsClassifier(n_neighbors=3)
+clf.fit(X, y)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+```
+:::
+
+<div class="output_subarea output_html rendered_html output_result">
+<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: "▸";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: "▾";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: "";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: "";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: "";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id="sk-container-id-1" class="sk-top-container"><div class="sk-text-repr-fallback"><pre>KNeighborsClassifier(n_neighbors=3)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class="sk-container" hidden><div class="sk-item"><div class="sk-estimator sk-toggleable"><input class="sk-toggleable__control sk-hidden--visually" id="sk-estimator-id-1" type="checkbox" checked><label for="sk-estimator-id-1" class="sk-toggleable__label sk-toggleable__label-arrow">KNeighborsClassifier</label><div class="sk-toggleable__content"><pre>KNeighborsClassifier(n_neighbors=3)</pre></div></div></div></div></div>
+</div>
+<br />
+<br />
+
+# Running a model on a task
+
+``` default
+task = openml.tasks.get_task(119)
+clf = ensemble.RandomForestClassifier()
+run = openml.runs.run_model_on_task(clf, task)
+print(run)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+OpenML Run
+==========
+Uploader Name...................: None
+Metric..........................: None
+Local Result - Accuracy (+- STD): 0.7510 +- 0.0000
+Local Runtime - ms (+- STD).....: 95.9373 +- 0.0000
+Run ID..........................: None
+Task ID.........................: 119
+Task Type.......................: None
+Task URL........................: https://test.openml.org/t/119
+Flow ID.........................: 4417
+Flow Name.......................: sklearn.ensemble._forest.RandomForestClassifier
+Flow URL........................: https://test.openml.org/f/4417
+Setup ID........................: None
+Setup String....................: Python_3.10.12. Sklearn_1.3.0. NumPy_1.25.1. SciPy_1.11.1.
+Dataset ID......................: 20
+Dataset URL.....................: https://test.openml.org/d/20
+```
+:::
+
+# Publishing the run
+
+``` default
+myrun = run.publish()
+print(f"Run was uploaded to {myrun.openml_url}")
+print(f"The flow can be found at {myrun.flow.openml_url}")
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+Run was uploaded to https://test.openml.org/r/7325
+The flow can be found at https://test.openml.org/f/4417
+```
+:::
+
+``` default
+openml.config.stop_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-timing
+
+**Total running time of the script:** ( 0 minutes 3.799 seconds)
+:::
+
+::::::: {#sphx_glr_download_examples_20_basic_simple_flows_and_runs_tutorial.py}
+:::::: only
+html
+
+::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
+::: {.container .sphx-glr-download .sphx-glr-download-python}
+`Download Python source code: simple_flows_and_runs_tutorial.py <simple_flows_and_runs_tutorial.py>`{.interpreted-text
+role="download"}
+:::
+
+::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
+`Download Jupyter notebook: simple_flows_and_runs_tutorial.ipynb <simple_flows_and_runs_tutorial.ipynb>`{.interpreted-text
+role="download"}
+:::
+:::::
+::::::
+:::::::
+
+:::: only
+html
+
+::: rst-class
+sphx-glr-signature
+
+[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
+:::
+::::
diff --git a/docs/examples/20_basic/simple_suites_tutorial.rst.md b/docs/examples/20_basic/simple_suites_tutorial.rst.md
new file mode 100644
index 000000000..e636a3a38
--- /dev/null
+++ b/docs/examples/20_basic/simple_suites_tutorial.rst.md
@@ -0,0 +1,188 @@
+::::: only
+html
+
+:::: {.note .sphx-glr-download-link-note}
+::: title
+Note
+:::
+
+`Go to the end <sphx_glr_download_examples_20_basic_simple_suites_tutorial.py>`{.interpreted-text
+role="ref"} to download the full example code
+::::
+:::::
+
+::: rst-class
+sphx-glr-example-title
+:::
+
+# Benchmark suites {#sphx_glr_examples_20_basic_simple_suites_tutorial.py}
+
+This is a brief showcase of OpenML benchmark suites, which were
+introduced by [Bischl et al.
+(2019)](https://arxiv.org/abs/1708.03731v2). Benchmark suites
+standardize the datasets and splits to be used in an experiment or
+paper. They are fully integrated into OpenML and simplify both the
+sharing of the setup and the results.
+
+``` default
+# License: BSD 3-Clause
+
+import openml
+```
+
+## OpenML-CC18
+
+As an example we have a look at the OpenML-CC18, which is a suite of 72
+classification datasets from OpenML which were carefully selected to be
+usable by many algorithms and also represent datasets commonly used in
+machine learning research. These are all datasets from mid-2018 that
+satisfy a large set of clear requirements for thorough yet practical
+benchmarking:
+
+1. the number of observations are between 500 and 100,000 to focus on
+ medium-sized datasets,
+2. the number of features does not exceed 5,000 features to keep the
+ runtime of the algorithms low
+3. the target attribute has at least two classes with no class having
+ less than 20 observations
+4. the ratio of the minority class and the majority class is above 0.05
+ (to eliminate highly imbalanced datasets which require special
+ treatment for both algorithms and evaluation measures).
+
+A full description can be found in the [OpenML benchmarking
+docs](https://docs.openml.org/benchmark/#openml-cc18).
+
+In this example we\'ll focus on how to use benchmark suites in practice.
+
+## Downloading benchmark suites
+
+``` default
+suite = openml.study.get_suite(99)
+print(suite)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+OpenML Benchmark Suite
+======================
+ID..............: 99
+Name............: OpenML-CC18 Curated Classification benchmark
+Status..........: active
+Main Entity Type: task
+Study URL.......: https://www.openml.org/s/99
+# of Data.......: 72
+# of Tasks......: 72
+Creator.........: https://www.openml.org/u/1
+Upload Time.....: 2019-02-21 18:47:13
+```
+:::
+
+The benchmark suite does not download the included tasks and datasets
+itself, but only contains a list of which tasks constitute the study.
+
+Tasks can then be accessed via
+
+``` default
+tasks = suite.tasks
+print(tasks)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+[3, 6, 11, 12, 14, 15, 16, 18, 22, 23, 28, 29, 31, 32, 37, 43, 45, 49, 53, 219, 2074, 2079, 3021, 3022, 3481, 3549, 3560, 3573, 3902, 3903, 3904, 3913, 3917, 3918, 7592, 9910, 9946, 9952, 9957, 9960, 9964, 9971, 9976, 9977, 9978, 9981, 9985, 10093, 10101, 14952, 14954, 14965, 14969, 14970, 125920, 125922, 146195, 146800, 146817, 146819, 146820, 146821, 146822, 146824, 146825, 167119, 167120, 167121, 167124, 167125, 167140, 167141]
+```
+:::
+
+and iterated over for benchmarking. For speed reasons we only iterate
+over the first three tasks:
+
+``` default
+for task_id in tasks[:3]:
+ task = openml.tasks.get_task(task_id)
+ print(task)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+OpenML Classification Task
+==========================
+Task Type Description: https://www.openml.org/tt/TaskType.SUPERVISED_CLASSIFICATION
+Task ID..............: 3
+Task URL.............: https://www.openml.org/t/3
+Estimation Procedure.: crossvalidation
+Target Feature.......: class
+# of Classes.........: 2
+Cost Matrix..........: Available
+OpenML Classification Task
+==========================
+Task Type Description: https://www.openml.org/tt/TaskType.SUPERVISED_CLASSIFICATION
+Task ID..............: 6
+Task URL.............: https://www.openml.org/t/6
+Estimation Procedure.: crossvalidation
+Target Feature.......: class
+# of Classes.........: 26
+Cost Matrix..........: Available
+OpenML Classification Task
+==========================
+Task Type Description: https://www.openml.org/tt/TaskType.SUPERVISED_CLASSIFICATION
+Task ID..............: 11
+Task URL.............: https://www.openml.org/t/11
+Estimation Procedure.: crossvalidation
+Target Feature.......: class
+# of Classes.........: 3
+Cost Matrix..........: Available
+```
+:::
+
+## Further examples
+
+- `sphx_glr_examples_30_extended_suites_tutorial.py`{.interpreted-text
+ role="ref"}
+- `sphx_glr_examples_30_extended_study_tutorial.py`{.interpreted-text
+ role="ref"}
+- `sphx_glr_examples_40_paper_2018_ida_strang_example.py`{.interpreted-text
+ role="ref"}
+
+::: rst-class
+sphx-glr-timing
+
+**Total running time of the script:** ( 0 minutes 9.728 seconds)
+:::
+
+::::::: {#sphx_glr_download_examples_20_basic_simple_suites_tutorial.py}
+:::::: only
+html
+
+::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
+::: {.container .sphx-glr-download .sphx-glr-download-python}
+`Download Python source code: simple_suites_tutorial.py <simple_suites_tutorial.py>`{.interpreted-text
+role="download"}
+:::
+
+::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
+`Download Jupyter notebook: simple_suites_tutorial.ipynb <simple_suites_tutorial.ipynb>`{.interpreted-text
+role="download"}
+:::
+:::::
+::::::
+:::::::
+
+:::: only
+html
+
+::: rst-class
+sphx-glr-signature
+
+[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
+:::
+::::
diff --git a/docs/examples/30_extended/configure_logging.rst.md b/docs/examples/30_extended/configure_logging.rst.md
new file mode 100644
index 000000000..3ba0d4ab5
--- /dev/null
+++ b/docs/examples/30_extended/configure_logging.rst.md
@@ -0,0 +1,107 @@
+::::: only
+html
+
+:::: {.note .sphx-glr-download-link-note}
+::: title
+Note
+:::
+
+`Go to the end <sphx_glr_download_examples_30_extended_configure_logging.py>`{.interpreted-text
+role="ref"} to download the full example code
+::::
+:::::
+
+::: rst-class
+sphx-glr-example-title
+:::
+
+# Logging {#sphx_glr_examples_30_extended_configure_logging.py}
+
+Explains openml-python logging, and shows how to configure it.
+
+Openml-python uses the [Python logging
+module](https://docs.python.org/3/library/logging.html) to provide users
+with log messages. Each log message is assigned a level of importance,
+see the table in Python\'s logging tutorial
+[here](https://docs.python.org/3/howto/logging.html#when-to-use-logging).
+
+By default, openml-python will print log messages of level
+[WARNING]{.title-ref} and above to console. All log messages (including
+[DEBUG]{.title-ref} and [INFO]{.title-ref}) are also saved in a file,
+which can be found in your cache directory (see also the
+`sphx_glr_examples_20_basic_introduction_tutorial.py`{.interpreted-text
+role="ref"}). These file logs are automatically deleted if needed, and
+use at most 2MB of space.
+
+It is possible to configure what log levels to send to console and file.
+When downloading a dataset from OpenML, a [DEBUG]{.title-ref}-level
+message is written:
+
+``` default
+# License: BSD 3-Clause
+
+import openml
+
+openml.datasets.get_dataset("iris")
+
+# With default configuration, the above example will show no output to console.
+# However, in your cache directory you should find a file named 'openml_python.log',
+# which has a DEBUG message written to it. It should be either like
+# "[DEBUG] [10:46:19:openml.datasets.dataset] Saved dataset 61: iris to file ..."
+# or like
+# "[DEBUG] [10:49:38:openml.datasets.dataset] Data pickle file already exists and is up to date."
+# , depending on whether or not you had downloaded iris before.
+# The processed log levels can be configured programmatically:
+
+import logging
+
+openml.config.set_console_log_level(logging.DEBUG)
+openml.config.set_file_log_level(logging.WARNING)
+openml.datasets.get_dataset("iris")
+
+# Now the log level that was previously written to file should also be shown in the console.
+# The message is now no longer written to file as the `file_log` was set to level `WARNING`.
+#
+# It is also possible to specify the desired log levels through the configuration file.
+# This way you will not need to set them on each script separately.
+# Add the line **verbosity = NUMBER** and/or **file_verbosity = NUMBER** to the config file,
+# where 'NUMBER' should be one of:
+#
+# * 0: `logging.WARNING` and up.
+# * 1: `logging.INFO` and up.
+# * 2: `logging.DEBUG` and up (i.e. all messages).
+```
+
+::: rst-class
+sphx-glr-timing
+
+**Total running time of the script:** ( 0 minutes 0.000 seconds)
+:::
+
+::::::: {#sphx_glr_download_examples_30_extended_configure_logging.py}
+:::::: only
+html
+
+::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
+::: {.container .sphx-glr-download .sphx-glr-download-python}
+`Download Python source code: configure_logging.py <configure_logging.py>`{.interpreted-text
+role="download"}
+:::
+
+::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
+`Download Jupyter notebook: configure_logging.ipynb <configure_logging.ipynb>`{.interpreted-text
+role="download"}
+:::
+:::::
+::::::
+:::::::
+
+:::: only
+html
+
+::: rst-class
+sphx-glr-signature
+
+[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
+:::
+::::
diff --git a/docs/examples/30_extended/create_upload_tutorial.rst.md b/docs/examples/30_extended/create_upload_tutorial.rst.md
new file mode 100644
index 000000000..83bfe0301
--- /dev/null
+++ b/docs/examples/30_extended/create_upload_tutorial.rst.md
@@ -0,0 +1,460 @@
+::::: only
+html
+
+:::: {.note .sphx-glr-download-link-note}
+::: title
+Note
+:::
+
+`Go to the end <sphx_glr_download_examples_30_extended_create_upload_tutorial.py>`{.interpreted-text
+role="ref"} to download the full example code
+::::
+:::::
+
+::: rst-class
+sphx-glr-example-title
+:::
+
+# Dataset upload tutorial {#sphx_glr_examples_30_extended_create_upload_tutorial.py}
+
+A tutorial on how to create and upload a dataset to OpenML.
+
+``` default
+# License: BSD 3-Clause
+
+import numpy as np
+import pandas as pd
+import sklearn.datasets
+from scipy.sparse import coo_matrix
+
+import openml
+from openml.datasets.functions import create_dataset
+```
+
+:::: warning
+::: title
+Warning
+:::
+
+.. include:: ../../test_server_usage_warning.txt
+::::
+
+``` default
+openml.config.start_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!
+ warnings.warn(
+```
+:::
+
+Below we will cover the following cases of the dataset object:
+
+- A numpy array
+- A list
+- A pandas dataframe
+- A sparse matrix
+- A pandas sparse dataframe
+
+# Dataset is a numpy array
+
+A numpy array can contain lists in the case of dense data or it can
+contain OrderedDicts in the case of sparse data.
+
+## Prepare dataset
+
+Load an example dataset from scikit-learn which we will upload to
+OpenML.org via the API.
+
+``` default
+diabetes = sklearn.datasets.load_diabetes()
+name = "Diabetes(scikit-learn)"
+X = diabetes.data
+y = diabetes.target
+attribute_names = diabetes.feature_names
+description = diabetes.DESCR
+```
+
+OpenML does not distinguish between the attributes and targets on the
+data level and stores all data in a single matrix.
+
+The target feature is indicated as meta-data of the dataset (and tasks
+on that data).
+
+``` default
+data = np.concatenate((X, y.reshape((-1, 1))), axis=1)
+attribute_names = list(attribute_names)
+attributes = [(attribute_name, "REAL") for attribute_name in attribute_names] + [
+ ("class", "INTEGER")
+]
+citation = (
+ "Bradley Efron, Trevor Hastie, Iain Johnstone and "
+ "Robert Tibshirani (2004) (Least Angle Regression) "
+ "Annals of Statistics (with discussion), 407-499"
+)
+paper_url = "https://web.stanford.edu/~hastie/Papers/LARS/LeastAngle_2002.pdf"
+```
+
+## Create the dataset object
+
+The definition of all fields can be found in the XSD files describing
+the expected format:
+
+<https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/openml.data.upload.xsd>
+
+``` default
+diabetes_dataset = create_dataset(
+ # The name of the dataset (needs to be unique).
+ # Must not be longer than 128 characters and only contain
+ # a-z, A-Z, 0-9 and the following special characters: _\-\.(),
+ name=name,
+ # Textual description of the dataset.
+ description=description,
+ # The person who created the dataset.
+ creator="Bradley Efron, Trevor Hastie, Iain Johnstone and Robert Tibshirani",
+ # People who contributed to the current version of the dataset.
+ contributor=None,
+ # The date the data was originally collected, given by the uploader.
+ collection_date="09-01-2012",
+ # Language in which the data is represented.
+ # Starts with 1 upper case letter, rest lower case, e.g. 'English'.
+ language="English",
+ # License under which the data is/will be distributed.
+ licence="BSD (from scikit-learn)",
+ # Name of the target. Can also have multiple values (comma-separated).
+ default_target_attribute="class",
+ # The attribute that represents the row-id column, if present in the
+ # dataset.
+ row_id_attribute=None,
+ # Attribute or list of attributes that should be excluded in modelling, such as
+ # identifiers and indexes. E.g. "feat1" or ["feat1","feat2"]
+ ignore_attribute=None,
+ # How to cite the paper.
+ citation=citation,
+ # Attributes of the data
+ attributes=attributes,
+ data=data,
+ # A version label which is provided by the user.
+ version_label="test",
+ original_data_url="https://www4.stat.ncsu.edu/~boos/var.select/diabetes.html",
+ paper_url=paper_url,
+)
+```
+
+``` default
+diabetes_dataset.publish()
+print(f"URL for dataset: {diabetes_dataset.openml_url}")
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+URL for dataset: https://test.openml.org/d/1322
+```
+:::
+
+# Dataset is a list
+
+A list can contain lists in the case of dense data or it can contain
+OrderedDicts in the case of sparse data.
+
+Weather dataset:
+<https://storm.cis.fordham.edu/~gweiss/data-mining/datasets.html>
+
+``` default
+data = [
+ ["sunny", 85, 85, "FALSE", "no"],
+ ["sunny", 80, 90, "TRUE", "no"],
+ ["overcast", 83, 86, "FALSE", "yes"],
+ ["rainy", 70, 96, "FALSE", "yes"],
+ ["rainy", 68, 80, "FALSE", "yes"],
+ ["rainy", 65, 70, "TRUE", "no"],
+ ["overcast", 64, 65, "TRUE", "yes"],
+ ["sunny", 72, 95, "FALSE", "no"],
+ ["sunny", 69, 70, "FALSE", "yes"],
+ ["rainy", 75, 80, "FALSE", "yes"],
+ ["sunny", 75, 70, "TRUE", "yes"],
+ ["overcast", 72, 90, "TRUE", "yes"],
+ ["overcast", 81, 75, "FALSE", "yes"],
+ ["rainy", 71, 91, "TRUE", "no"],
+]
+
+attribute_names = [
+ ("outlook", ["sunny", "overcast", "rainy"]),
+ ("temperature", "REAL"),
+ ("humidity", "REAL"),
+ ("windy", ["TRUE", "FALSE"]),
+ ("play", ["yes", "no"]),
+]
+
+description = (
+ "The weather problem is a tiny dataset that we will use repeatedly"
+ " to illustrate machine learning methods. Entirely fictitious, it "
+ "supposedly concerns the conditions that are suitable for playing "
+ "some unspecified game. In general, instances in a dataset are "
+ "characterized by the values of features, or attributes, that measure "
+ "different aspects of the instance. In this case there are four "
+ "attributes: outlook, temperature, humidity, and windy. "
+ "The outcome is whether to play or not."
+)
+
+citation = (
+ "I. H. Witten, E. Frank, M. A. Hall, and ITPro,"
+ "Data mining practical machine learning tools and techniques, "
+ "third edition. Burlington, Mass.: Morgan Kaufmann Publishers, 2011"
+)
+
+weather_dataset = create_dataset(
+ name="Weather",
+ description=description,
+ creator="I. H. Witten, E. Frank, M. A. Hall, and ITPro",
+ contributor=None,
+ collection_date="01-01-2011",
+ language="English",
+ licence=None,
+ default_target_attribute="play",
+ row_id_attribute=None,
+ ignore_attribute=None,
+ citation=citation,
+ attributes=attribute_names,
+ data=data,
+ version_label="example",
+)
+```
+
+``` default
+weather_dataset.publish()
+print(f"URL for dataset: {weather_dataset.openml_url}")
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+URL for dataset: https://test.openml.org/d/1323
+```
+:::
+
+# Dataset is a pandas DataFrame
+
+It might happen that your dataset is made of heterogeneous data which
+can usually be stored as a Pandas DataFrame. DataFrames offer the
+advantage of storing the type of data for each column as well as the
+attribute names. Therefore, when providing a Pandas DataFrame, OpenML
+can infer this information without needing to explicitly provide it when
+calling the function `openml.datasets.create_dataset`{.interpreted-text
+role="func"}. In this regard, you only need to pass `'auto'` to the
+`attributes` parameter.
+
+``` default
+df = pd.DataFrame(data, columns=[col_name for col_name, _ in attribute_names])
+# enforce the categorical column to have a categorical dtype
+df["outlook"] = df["outlook"].astype("category")
+df["windy"] = df["windy"].astype("bool")
+df["play"] = df["play"].astype("category")
+print(df.info())
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+<class 'pandas.core.frame.DataFrame'>
+RangeIndex: 14 entries, 0 to 13
+Data columns (total 5 columns):
+ # Column Non-Null Count Dtype
+--- ------ -------------- -----
+ 0 outlook 14 non-null category
+ 1 temperature 14 non-null int64
+ 2 humidity 14 non-null int64
+ 3 windy 14 non-null bool
+ 4 play 14 non-null category
+dtypes: bool(1), category(2), int64(2)
+memory usage: 650.0 bytes
+None
+```
+:::
+
+We enforce the column \'outlook\' and \'play\' to be a categorical dtype
+while the column \'windy\' is kept as a boolean column. \'temperature\'
+and \'humidity\' are kept as numeric columns. Then, we can call
+`openml.datasets.create_dataset`{.interpreted-text role="func"} by
+passing the dataframe and fixing the parameter `attributes` to `'auto'`.
+
+``` default
+weather_dataset = create_dataset(
+ name="Weather",
+ description=description,
+ creator="I. H. Witten, E. Frank, M. A. Hall, and ITPro",
+ contributor=None,
+ collection_date="01-01-2011",
+ language="English",
+ licence=None,
+ default_target_attribute="play",
+ row_id_attribute=None,
+ ignore_attribute=None,
+ citation=citation,
+ attributes="auto",
+ data=df,
+ version_label="example",
+)
+```
+
+``` default
+weather_dataset.publish()
+print(f"URL for dataset: {weather_dataset.openml_url}")
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+URL for dataset: https://test.openml.org/d/1324
+```
+:::
+
+# Dataset is a sparse matrix
+
+``` default
+sparse_data = coo_matrix(
+ ([0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], ([0, 1, 1, 2, 2, 3, 3], [0, 1, 2, 0, 2, 0, 1]))
+)
+
+column_names = [
+ ("input1", "REAL"),
+ ("input2", "REAL"),
+ ("y", "REAL"),
+]
+
+xor_dataset = create_dataset(
+ name="XOR",
+ description="Dataset representing the XOR operation",
+ creator=None,
+ contributor=None,
+ collection_date=None,
+ language="English",
+ licence=None,
+ default_target_attribute="y",
+ row_id_attribute=None,
+ ignore_attribute=None,
+ citation=None,
+ attributes=column_names,
+ data=sparse_data,
+ version_label="example",
+)
+```
+
+``` default
+xor_dataset.publish()
+print(f"URL for dataset: {xor_dataset.openml_url}")
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+URL for dataset: https://test.openml.org/d/1325
+```
+:::
+
+# Dataset is a pandas dataframe with sparse columns
+
+``` default
+sparse_data = coo_matrix(
+ ([1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0], ([0, 1, 1, 2, 2, 3, 3], [0, 1, 2, 0, 2, 0, 1]))
+)
+column_names = ["input1", "input2", "y"]
+df = pd.DataFrame.sparse.from_spmatrix(sparse_data, columns=column_names)
+print(df.info())
+
+xor_dataset = create_dataset(
+ name="XOR",
+ description="Dataset representing the XOR operation",
+ creator=None,
+ contributor=None,
+ collection_date=None,
+ language="English",
+ licence=None,
+ default_target_attribute="y",
+ row_id_attribute=None,
+ ignore_attribute=None,
+ citation=None,
+ attributes="auto",
+ data=df,
+ version_label="example",
+)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+<class 'pandas.core.frame.DataFrame'>
+RangeIndex: 4 entries, 0 to 3
+Data columns (total 3 columns):
+ # Column Non-Null Count Dtype
+--- ------ -------------- -----
+ 0 input1 4 non-null Sparse[float64, 0]
+ 1 input2 4 non-null Sparse[float64, 0]
+ 2 y 4 non-null Sparse[float64, 0]
+dtypes: Sparse[float64, 0](3)
+memory usage: 212.0 bytes
+None
+```
+:::
+
+``` default
+xor_dataset.publish()
+print(f"URL for dataset: {xor_dataset.openml_url}")
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+URL for dataset: https://test.openml.org/d/1326
+```
+:::
+
+``` default
+openml.config.stop_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-timing
+
+**Total running time of the script:** ( 0 minutes 2.833 seconds)
+:::
+
+::::::: {#sphx_glr_download_examples_30_extended_create_upload_tutorial.py}
+:::::: only
+html
+
+::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
+::: {.container .sphx-glr-download .sphx-glr-download-python}
+`Download Python source code: create_upload_tutorial.py <create_upload_tutorial.py>`{.interpreted-text
+role="download"}
+:::
+
+::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
+`Download Jupyter notebook: create_upload_tutorial.ipynb <create_upload_tutorial.ipynb>`{.interpreted-text
+role="download"}
+:::
+:::::
+::::::
+:::::::
+
+:::: only
+html
+
+::: rst-class
+sphx-glr-signature
+
+[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
+:::
+::::
diff --git a/docs/examples/30_extended/custom_flow_.rst.md b/docs/examples/30_extended/custom_flow_.rst.md
new file mode 100644
index 000000000..0bf45015a
--- /dev/null
+++ b/docs/examples/30_extended/custom_flow_.rst.md
@@ -0,0 +1,294 @@
+::::: only
+html
+
+:::: {.note .sphx-glr-download-link-note}
+::: title
+Note
+:::
+
+`Go to the end <sphx_glr_download_examples_30_extended_custom_flow_.py>`{.interpreted-text
+role="ref"} to download the full example code
+::::
+:::::
+
+::: rst-class
+sphx-glr-example-title
+:::
+
+# Creating and Using a Custom Flow {#sphx_glr_examples_30_extended_custom_flow_.py}
+
+The most convenient way to create a flow for your machine learning
+workflow is to generate it automatically as described in the
+`sphx_glr_examples_30_extended_flow_id_tutorial.py`{.interpreted-text
+role="ref"} tutorial. However, there are scenarios where this is not
+possible, such as when the flow uses a framework without an extension or
+when the flow is described by a script.
+
+In those cases you can still create a custom flow by following the steps
+of this tutorial. As an example we will use the flows generated for the
+[AutoML Benchmark](https://openml.github.io/automlbenchmark/), and also
+show how to link runs to the custom flow.
+
+``` default
+# License: BSD 3-Clause
+
+from collections import OrderedDict
+import numpy as np
+
+import openml
+from openml import OpenMLClassificationTask
+from openml.runs.functions import format_prediction
+```
+
+:::: warning
+::: title
+Warning
+:::
+
+.. include:: ../../test_server_usage_warning.txt
+::::
+
+``` default
+openml.config.start_using_configuration_for_example()
+```
+
+## 1. Defining the flow
+
+The first step is to define all the hyperparameters of your flow. The
+API pages feature a descriptions of each variable of the
+`openml.flows.OpenMLFlow`{.interpreted-text role="class"}. Note that
+[external version]{.title-ref} and [name]{.title-ref} together uniquely
+identify a flow.
+
+The AutoML Benchmark runs AutoML systems across a range of tasks. OpenML
+stores Flows for each AutoML system. However, the AutoML benchmark adds
+preprocessing to the flow, so should be described in a new flow.
+
+We will break down the flow arguments into several groups, for the
+tutorial. First we will define the name and version information. Make
+sure to leave enough information so others can determine exactly which
+version of the package/script is used. Use tags so users can find your
+flow easily.
+
+``` default
+general = dict(
+ name="automlbenchmark_autosklearn",
+ description=(
+ "Auto-sklearn as set up by the AutoML Benchmark"
+ "Source: https://github.com/openml/automlbenchmark/releases/tag/v0.9"
+ ),
+ external_version="amlb==0.9",
+ language="English",
+ tags=["amlb", "benchmark", "study_218"],
+ dependencies="amlb==0.9",
+)
+```
+
+Next we define the flow hyperparameters. We define their name and
+default value in [parameters]{.title-ref}, and provide meta-data for
+each hyperparameter through [parameters_meta_info]{.title-ref}. Note
+that even though the argument name is [parameters]{.title-ref} they
+describe the hyperparameters. The use of ordered dicts is required.
+
+``` default
+flow_hyperparameters = dict(
+ parameters=OrderedDict(time="240", memory="32", cores="8"),
+ parameters_meta_info=OrderedDict(
+ cores=OrderedDict(description="number of available cores", data_type="int"),
+ memory=OrderedDict(description="memory in gigabytes", data_type="int"),
+ time=OrderedDict(description="time in minutes", data_type="int"),
+ ),
+)
+```
+
+It is possible to build a flow which uses other flows. For example, the
+Random Forest Classifier is a flow, but you could also construct a flow
+which uses a Random Forest Classifier in a ML pipeline. When
+constructing the pipeline flow, you can use the Random Forest Classifier
+flow as a *subflow*. It allows for all hyperparameters of the Random
+Classifier Flow to also be specified in your pipeline flow.
+
+Note: you can currently only specific one subflow as part of the
+components.
+
+In this example, the auto-sklearn flow is a subflow: the auto-sklearn
+flow is entirely executed as part of this flow. This allows people to
+specify auto-sklearn hyperparameters used in this flow. In general,
+using a subflow is not required.
+
+Note: flow 9313 is not actually the right flow on the test server, but
+that does not matter for this demonstration.
+
+``` default
+autosklearn_flow = openml.flows.get_flow(9313) # auto-sklearn 0.5.1
+subflow = dict(
+ components=OrderedDict(automl_tool=autosklearn_flow),
+ # If you do not want to reference a subflow, you can use the following:
+ # components=OrderedDict(),
+)
+```
+
+With all parameters of the flow defined, we can now initialize the
+OpenMLFlow and publish. Because we provided all the details already, we
+do not need to provide a [model]{.title-ref} to the flow.
+
+In our case, we don\'t even have a model. It is possible to have a model
+but still require to follow these steps when the model (python object)
+does not have an extensions from which to automatically extract the
+hyperparameters. So whether you have a model with no extension or no
+model at all, explicitly set the model of the flow to
+[None]{.title-ref}.
+
+``` default
+autosklearn_amlb_flow = openml.flows.OpenMLFlow(
+ **general,
+ **flow_hyperparameters,
+ **subflow,
+ model=None,
+)
+autosklearn_amlb_flow.publish()
+print(f"autosklearn flow created: {autosklearn_amlb_flow.flow_id}")
+```
+
+## 2. Using the flow
+
+This Section will show how to upload run data for your custom flow. Take
+care to change the values of parameters as well as the task id, to
+reflect the actual run. Task and parameter values in the example are
+fictional.
+
+``` default
+flow_id = autosklearn_amlb_flow.flow_id
+
+parameters = [
+ OrderedDict([("oml:name", "cores"), ("oml:value", 4), ("oml:component", flow_id)]),
+ OrderedDict([("oml:name", "memory"), ("oml:value", 16), ("oml:component", flow_id)]),
+ OrderedDict([("oml:name", "time"), ("oml:value", 120), ("oml:component", flow_id)]),
+]
+
+task_id = 1200 # Iris Task
+task = openml.tasks.get_task(task_id)
+dataset_id = task.get_dataset().dataset_id
+```
+
+The last bit of information for the run we need are the predicted
+values. The exact format of the predictions will depend on the task.
+
+The predictions should always be a list of lists, each list should
+contain:
+
+- the repeat number: for repeated evaluation strategies. (e.g.
+ repeated cross-validation)
+- the fold number: for cross-validation. (what should this be for
+ holdout?)
+- 0: this field is for backward compatibility.
+- index: the row (of the original dataset) for which the prediction
+ was made.
+- p_1, \..., p_c: for each class the predicted probability of the
+ sample belonging to that class. (no elements for regression tasks)
+ Make sure the order of these elements follows the order of
+ [task.class_labels]{.title-ref}.
+- the predicted class/value for the sample
+- the true class/value for the sample
+
+When using openml-python extensions (such as through
+[run_model_on_task]{.title-ref}), all of this formatting is automatic.
+Unfortunately we can not automate this procedure for custom flows, which
+means a little additional effort is required.
+
+Here we generated some random predictions in place. You can ignore this
+code, or use it to better understand the formatting of the predictions.
+
+Find the repeats/folds for this task:
+
+``` default
+n_repeats, n_folds, _ = task.get_split_dimensions()
+all_test_indices = [
+ (repeat, fold, index)
+ for repeat in range(n_repeats)
+ for fold in range(n_folds)
+ for index in task.get_train_test_split_indices(fold, repeat)[1]
+]
+
+# random class probabilities (Iris has 150 samples and 3 classes):
+r = np.random.rand(150 * n_repeats, 3)
+# scale the random values so that the probabilities of each sample sum to 1:
+y_proba = r / r.sum(axis=1).reshape(-1, 1)
+y_pred = y_proba.argmax(axis=1)
+
+class_map = dict(zip(range(3), task.class_labels))
+_, y_true = task.get_X_and_y()
+y_true = [class_map[y] for y in y_true]
+
+# We format the predictions with the utility function `format_prediction`.
+# It will organize the relevant data in the expected format/order.
+predictions = []
+for where, y, yp, proba in zip(all_test_indices, y_true, y_pred, y_proba):
+ repeat, fold, index = where
+
+ prediction = format_prediction(
+ task=task,
+ repeat=repeat,
+ fold=fold,
+ index=index,
+ prediction=class_map[yp],
+ truth=y,
+ proba={c: pb for (c, pb) in zip(task.class_labels, proba)},
+ )
+ predictions.append(prediction)
+```
+
+Finally we can create the OpenMLRun object and upload. We use the
+argument setup_string because the used flow was a script.
+
+``` default
+benchmark_command = f"python3 runbenchmark.py auto-sklearn medium -m aws -t 119"
+my_run = openml.runs.OpenMLRun(
+ task_id=task_id,
+ flow_id=flow_id,
+ dataset_id=dataset_id,
+ parameter_settings=parameters,
+ setup_string=benchmark_command,
+ data_content=predictions,
+ tags=["study_218"],
+ description_text="Run generated by the Custom Flow tutorial.",
+)
+my_run.publish()
+print("run created:", my_run.run_id)
+
+openml.config.stop_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-timing
+
+**Total running time of the script:** ( 0 minutes 0.000 seconds)
+:::
+
+::::::: {#sphx_glr_download_examples_30_extended_custom_flow_.py}
+:::::: only
+html
+
+::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
+::: {.container .sphx-glr-download .sphx-glr-download-python}
+`Download Python source code: custom_flow_.py <custom_flow_.py>`{.interpreted-text
+role="download"}
+:::
+
+::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
+`Download Jupyter notebook: custom_flow_.ipynb <custom_flow_.ipynb>`{.interpreted-text
+role="download"}
+:::
+:::::
+::::::
+:::::::
+
+:::: only
+html
+
+::: rst-class
+sphx-glr-signature
+
+[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
+:::
+::::
diff --git a/docs/examples/30_extended/datasets_tutorial.rst.md b/docs/examples/30_extended/datasets_tutorial.rst.md
new file mode 100644
index 000000000..ae02ecb31
--- /dev/null
+++ b/docs/examples/30_extended/datasets_tutorial.rst.md
@@ -0,0 +1,1040 @@
+::::: only
+html
+
+:::: {.note .sphx-glr-download-link-note}
+::: title
+Note
+:::
+
+`Go to the end <sphx_glr_download_examples_30_extended_datasets_tutorial.py>`{.interpreted-text
+role="ref"} to download the full example code
+::::
+:::::
+
+::: rst-class
+sphx-glr-example-title
+:::
+
+# Datasets {#sphx_glr_examples_30_extended_datasets_tutorial.py}
+
+How to list and download datasets.
+
+``` default
+# License: BSD 3-Clauses
+
+import openml
+import pandas as pd
+from openml.datasets import edit_dataset, fork_dataset, get_dataset
+```
+
+## Exercise 0
+
+- List datasets
+ - Use the output_format parameter to select output type
+ - Default gives \'dict\' (other option: \'dataframe\', see below)
+
+Note: list_datasets will return a pandas dataframe by default from 0.15.
+When using openml-python 0.14, [list_datasets]{.title-ref} will warn you
+to use output_format=\'dataframe\'.
+
+``` default
+datalist = openml.datasets.list_datasets(output_format="dataframe")
+datalist = datalist[["did", "name", "NumberOfInstances", "NumberOfFeatures", "NumberOfClasses"]]
+
+print(f"First 10 of {len(datalist)} datasets...")
+datalist.head(n=10)
+
+# The same can be done with lesser lines of code
+openml_df = openml.datasets.list_datasets(output_format="dataframe")
+openml_df.head(n=10)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+First 10 of 5308 datasets...
+```
+:::
+
+<div class="output_subarea output_html rendered_html output_result">
+<div>
+<style scoped>
+ .dataframe tbody tr th:only-of-type {
+ vertical-align: middle;
+ }
+
+ .dataframe tbody tr th {
+ vertical-align: top;
+ }
+
+ .dataframe thead th {
+ text-align: right;
+ }
+</style>
+<table border="1" class="dataframe">
+ <thead>
+ <tr style="text-align: right;">
+ <th></th>
+ <th>did</th>
+ <th>name</th>
+ <th>version</th>
+ <th>uploader</th>
+ <th>status</th>
+ <th>format</th>
+ <th>MajorityClassSize</th>
+ <th>MaxNominalAttDistinctValues</th>
+ <th>MinorityClassSize</th>
+ <th>NumberOfClasses</th>
+ <th>NumberOfFeatures</th>
+ <th>NumberOfInstances</th>
+ <th>NumberOfInstancesWithMissingValues</th>
+ <th>NumberOfMissingValues</th>
+ <th>NumberOfNumericFeatures</th>
+ <th>NumberOfSymbolicFeatures</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <th>2</th>
+ <td>2</td>
+ <td>anneal</td>
+ <td>1</td>
+ <td>1</td>
+ <td>active</td>
+ <td>ARFF</td>
+ <td>684.0</td>
+ <td>7.0</td>
+ <td>8.0</td>
+ <td>5.0</td>
+ <td>39.0</td>
+ <td>898.0</td>
+ <td>898.0</td>
+ <td>22175.0</td>
+ <td>6.0</td>
+ <td>33.0</td>
+ </tr>
+ <tr>
+ <th>3</th>
+ <td>3</td>
+ <td>kr-vs-kp</td>
+ <td>1</td>
+ <td>1</td>
+ <td>active</td>
+ <td>ARFF</td>
+ <td>1669.0</td>
+ <td>3.0</td>
+ <td>1527.0</td>
+ <td>2.0</td>
+ <td>37.0</td>
+ <td>3196.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>37.0</td>
+ </tr>
+ <tr>
+ <th>4</th>
+ <td>4</td>
+ <td>labor</td>
+ <td>1</td>
+ <td>1</td>
+ <td>active</td>
+ <td>ARFF</td>
+ <td>37.0</td>
+ <td>3.0</td>
+ <td>20.0</td>
+ <td>2.0</td>
+ <td>17.0</td>
+ <td>57.0</td>
+ <td>56.0</td>
+ <td>326.0</td>
+ <td>8.0</td>
+ <td>9.0</td>
+ </tr>
+ <tr>
+ <th>5</th>
+ <td>5</td>
+ <td>arrhythmia</td>
+ <td>1</td>
+ <td>1</td>
+ <td>active</td>
+ <td>ARFF</td>
+ <td>245.0</td>
+ <td>13.0</td>
+ <td>2.0</td>
+ <td>13.0</td>
+ <td>280.0</td>
+ <td>452.0</td>
+ <td>384.0</td>
+ <td>408.0</td>
+ <td>206.0</td>
+ <td>74.0</td>
+ </tr>
+ <tr>
+ <th>6</th>
+ <td>6</td>
+ <td>letter</td>
+ <td>1</td>
+ <td>1</td>
+ <td>active</td>
+ <td>ARFF</td>
+ <td>813.0</td>
+ <td>26.0</td>
+ <td>734.0</td>
+ <td>26.0</td>
+ <td>17.0</td>
+ <td>20000.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>16.0</td>
+ <td>1.0</td>
+ </tr>
+ <tr>
+ <th>7</th>
+ <td>7</td>
+ <td>audiology</td>
+ <td>1</td>
+ <td>1</td>
+ <td>active</td>
+ <td>ARFF</td>
+ <td>57.0</td>
+ <td>24.0</td>
+ <td>1.0</td>
+ <td>24.0</td>
+ <td>70.0</td>
+ <td>226.0</td>
+ <td>222.0</td>
+ <td>317.0</td>
+ <td>0.0</td>
+ <td>70.0</td>
+ </tr>
+ <tr>
+ <th>8</th>
+ <td>8</td>
+ <td>liver-disorders</td>
+ <td>1</td>
+ <td>1</td>
+ <td>active</td>
+ <td>ARFF</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>0.0</td>
+ <td>6.0</td>
+ <td>345.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>6.0</td>
+ <td>0.0</td>
+ </tr>
+ <tr>
+ <th>9</th>
+ <td>9</td>
+ <td>autos</td>
+ <td>1</td>
+ <td>1</td>
+ <td>active</td>
+ <td>ARFF</td>
+ <td>67.0</td>
+ <td>22.0</td>
+ <td>3.0</td>
+ <td>6.0</td>
+ <td>26.0</td>
+ <td>205.0</td>
+ <td>46.0</td>
+ <td>59.0</td>
+ <td>15.0</td>
+ <td>11.0</td>
+ </tr>
+ <tr>
+ <th>10</th>
+ <td>10</td>
+ <td>lymph</td>
+ <td>1</td>
+ <td>1</td>
+ <td>active</td>
+ <td>ARFF</td>
+ <td>81.0</td>
+ <td>8.0</td>
+ <td>2.0</td>
+ <td>4.0</td>
+ <td>19.0</td>
+ <td>148.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>3.0</td>
+ <td>16.0</td>
+ </tr>
+ <tr>
+ <th>11</th>
+ <td>11</td>
+ <td>balance-scale</td>
+ <td>1</td>
+ <td>1</td>
+ <td>active</td>
+ <td>ARFF</td>
+ <td>288.0</td>
+ <td>3.0</td>
+ <td>49.0</td>
+ <td>3.0</td>
+ <td>5.0</td>
+ <td>625.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>4.0</td>
+ <td>1.0</td>
+ </tr>
+ </tbody>
+</table>
+</div>
+</div>
+<br />
+<br />
+
+## Exercise 1
+
+- Find datasets with more than 10000 examples.
+- Find a dataset called \'eeg_eye_state\'.
+- Find all datasets with more than 50 classes.
+
+``` default
+datalist[datalist.NumberOfInstances > 10000].sort_values(["NumberOfInstances"]).head(n=20)
+""
+datalist.query('name == "eeg-eye-state"')
+""
+datalist.query("NumberOfClasses > 50")
+```
+
+<div class="output_subarea output_html rendered_html output_result">
+<div>
+<style scoped>
+ .dataframe tbody tr th:only-of-type {
+ vertical-align: middle;
+ }
+
+ .dataframe tbody tr th {
+ vertical-align: top;
+ }
+
+ .dataframe thead th {
+ text-align: right;
+ }
+</style>
+<table border="1" class="dataframe">
+ <thead>
+ <tr style="text-align: right;">
+ <th></th>
+ <th>did</th>
+ <th>name</th>
+ <th>NumberOfInstances</th>
+ <th>NumberOfFeatures</th>
+ <th>NumberOfClasses</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <th>1491</th>
+ <td>1491</td>
+ <td>one-hundred-plants-margin</td>
+ <td>1600.0</td>
+ <td>65.0</td>
+ <td>100.0</td>
+ </tr>
+ <tr>
+ <th>1492</th>
+ <td>1492</td>
+ <td>one-hundred-plants-shape</td>
+ <td>1600.0</td>
+ <td>65.0</td>
+ <td>100.0</td>
+ </tr>
+ <tr>
+ <th>1493</th>
+ <td>1493</td>
+ <td>one-hundred-plants-texture</td>
+ <td>1599.0</td>
+ <td>65.0</td>
+ <td>100.0</td>
+ </tr>
+ <tr>
+ <th>4552</th>
+ <td>4552</td>
+ <td>BachChoralHarmony</td>
+ <td>5665.0</td>
+ <td>17.0</td>
+ <td>102.0</td>
+ </tr>
+ <tr>
+ <th>41167</th>
+ <td>41167</td>
+ <td>dionis</td>
+ <td>416188.0</td>
+ <td>61.0</td>
+ <td>355.0</td>
+ </tr>
+ <tr>
+ <th>41169</th>
+ <td>41169</td>
+ <td>helena</td>
+ <td>65196.0</td>
+ <td>28.0</td>
+ <td>100.0</td>
+ </tr>
+ <tr>
+ <th>41960</th>
+ <td>41960</td>
+ <td>seattlecrime6</td>
+ <td>523590.0</td>
+ <td>8.0</td>
+ <td>144.0</td>
+ </tr>
+ <tr>
+ <th>41983</th>
+ <td>41983</td>
+ <td>CIFAR-100</td>
+ <td>60000.0</td>
+ <td>3073.0</td>
+ <td>100.0</td>
+ </tr>
+ <tr>
+ <th>42078</th>
+ <td>42078</td>
+ <td>beer_reviews</td>
+ <td>1586614.0</td>
+ <td>13.0</td>
+ <td>104.0</td>
+ </tr>
+ <tr>
+ <th>42087</th>
+ <td>42087</td>
+ <td>beer_reviews</td>
+ <td>1586614.0</td>
+ <td>13.0</td>
+ <td>104.0</td>
+ </tr>
+ <tr>
+ <th>42088</th>
+ <td>42088</td>
+ <td>beer_reviews</td>
+ <td>1586614.0</td>
+ <td>13.0</td>
+ <td>104.0</td>
+ </tr>
+ <tr>
+ <th>42089</th>
+ <td>42089</td>
+ <td>vancouver_employee</td>
+ <td>1586614.0</td>
+ <td>13.0</td>
+ <td>104.0</td>
+ </tr>
+ <tr>
+ <th>42123</th>
+ <td>42123</td>
+ <td>article_influence</td>
+ <td>3615.0</td>
+ <td>7.0</td>
+ <td>3169.0</td>
+ </tr>
+ <tr>
+ <th>42223</th>
+ <td>42223</td>
+ <td>dataset-autoHorse_fixed</td>
+ <td>201.0</td>
+ <td>69.0</td>
+ <td>186.0</td>
+ </tr>
+ <tr>
+ <th>42396</th>
+ <td>42396</td>
+ <td>aloi</td>
+ <td>108000.0</td>
+ <td>129.0</td>
+ <td>1000.0</td>
+ </tr>
+ <tr>
+ <th>43723</th>
+ <td>43723</td>
+ <td>Toronto-Apartment-Rental-Price</td>
+ <td>1124.0</td>
+ <td>7.0</td>
+ <td>188.0</td>
+ </tr>
+ <tr>
+ <th>44282</th>
+ <td>44282</td>
+ <td>Meta_Album_PLK_Mini</td>
+ <td>3440.0</td>
+ <td>3.0</td>
+ <td>86.0</td>
+ </tr>
+ <tr>
+ <th>44283</th>
+ <td>44283</td>
+ <td>Meta_Album_FLW_Mini</td>
+ <td>4080.0</td>
+ <td>3.0</td>
+ <td>102.0</td>
+ </tr>
+ <tr>
+ <th>44284</th>
+ <td>44284</td>
+ <td>Meta_Album_SPT_Mini</td>
+ <td>2920.0</td>
+ <td>3.0</td>
+ <td>73.0</td>
+ </tr>
+ <tr>
+ <th>44285</th>
+ <td>44285</td>
+ <td>Meta_Album_BRD_Mini</td>
+ <td>12600.0</td>
+ <td>3.0</td>
+ <td>315.0</td>
+ </tr>
+ <tr>
+ <th>44288</th>
+ <td>44288</td>
+ <td>Meta_Album_TEX_Mini</td>
+ <td>2560.0</td>
+ <td>3.0</td>
+ <td>64.0</td>
+ </tr>
+ <tr>
+ <th>44289</th>
+ <td>44289</td>
+ <td>Meta_Album_CRS_Mini</td>
+ <td>7840.0</td>
+ <td>3.0</td>
+ <td>196.0</td>
+ </tr>
+ <tr>
+ <th>44292</th>
+ <td>44292</td>
+ <td>Meta_Album_INS_2_Mini</td>
+ <td>4080.0</td>
+ <td>3.0</td>
+ <td>102.0</td>
+ </tr>
+ <tr>
+ <th>44298</th>
+ <td>44298</td>
+ <td>Meta_Album_DOG_Mini</td>
+ <td>4800.0</td>
+ <td>3.0</td>
+ <td>120.0</td>
+ </tr>
+ <tr>
+ <th>44304</th>
+ <td>44304</td>
+ <td>Meta_Album_TEX_ALOT_Mini</td>
+ <td>10000.0</td>
+ <td>3.0</td>
+ <td>250.0</td>
+ </tr>
+ <tr>
+ <th>44306</th>
+ <td>44306</td>
+ <td>Meta_Album_INS_Mini</td>
+ <td>4160.0</td>
+ <td>3.0</td>
+ <td>104.0</td>
+ </tr>
+ <tr>
+ <th>44317</th>
+ <td>44317</td>
+ <td>Meta_Album_PLK_Extended</td>
+ <td>473273.0</td>
+ <td>3.0</td>
+ <td>102.0</td>
+ </tr>
+ <tr>
+ <th>44318</th>
+ <td>44318</td>
+ <td>Meta_Album_FLW_Extended</td>
+ <td>8189.0</td>
+ <td>3.0</td>
+ <td>102.0</td>
+ </tr>
+ <tr>
+ <th>44319</th>
+ <td>44319</td>
+ <td>Meta_Album_SPT_Extended</td>
+ <td>10416.0</td>
+ <td>3.0</td>
+ <td>73.0</td>
+ </tr>
+ <tr>
+ <th>44320</th>
+ <td>44320</td>
+ <td>Meta_Album_BRD_Extended</td>
+ <td>49054.0</td>
+ <td>3.0</td>
+ <td>315.0</td>
+ </tr>
+ <tr>
+ <th>44322</th>
+ <td>44322</td>
+ <td>Meta_Album_TEX_Extended</td>
+ <td>8675.0</td>
+ <td>3.0</td>
+ <td>64.0</td>
+ </tr>
+ <tr>
+ <th>44323</th>
+ <td>44323</td>
+ <td>Meta_Album_CRS_Extended</td>
+ <td>16185.0</td>
+ <td>3.0</td>
+ <td>196.0</td>
+ </tr>
+ <tr>
+ <th>44326</th>
+ <td>44326</td>
+ <td>Meta_Album_INS_2_Extended</td>
+ <td>75222.0</td>
+ <td>3.0</td>
+ <td>102.0</td>
+ </tr>
+ <tr>
+ <th>44331</th>
+ <td>44331</td>
+ <td>Meta_Album_DOG_Extended</td>
+ <td>20480.0</td>
+ <td>3.0</td>
+ <td>120.0</td>
+ </tr>
+ <tr>
+ <th>44337</th>
+ <td>44337</td>
+ <td>Meta_Album_TEX_ALOT_Extended</td>
+ <td>25000.0</td>
+ <td>3.0</td>
+ <td>250.0</td>
+ </tr>
+ <tr>
+ <th>44340</th>
+ <td>44340</td>
+ <td>Meta_Album_INS_Extended</td>
+ <td>170506.0</td>
+ <td>3.0</td>
+ <td>117.0</td>
+ </tr>
+ <tr>
+ <th>44533</th>
+ <td>44533</td>
+ <td>dionis_seed_0_nrows_2000_nclasses_10_ncols_100...</td>
+ <td>2000.0</td>
+ <td>61.0</td>
+ <td>355.0</td>
+ </tr>
+ <tr>
+ <th>44534</th>
+ <td>44534</td>
+ <td>dionis_seed_1_nrows_2000_nclasses_10_ncols_100...</td>
+ <td>2000.0</td>
+ <td>61.0</td>
+ <td>355.0</td>
+ </tr>
+ <tr>
+ <th>44535</th>
+ <td>44535</td>
+ <td>dionis_seed_2_nrows_2000_nclasses_10_ncols_100...</td>
+ <td>2000.0</td>
+ <td>61.0</td>
+ <td>355.0</td>
+ </tr>
+ <tr>
+ <th>44536</th>
+ <td>44536</td>
+ <td>dionis_seed_3_nrows_2000_nclasses_10_ncols_100...</td>
+ <td>2000.0</td>
+ <td>61.0</td>
+ <td>355.0</td>
+ </tr>
+ <tr>
+ <th>44537</th>
+ <td>44537</td>
+ <td>dionis_seed_4_nrows_2000_nclasses_10_ncols_100...</td>
+ <td>2000.0</td>
+ <td>61.0</td>
+ <td>355.0</td>
+ </tr>
+ <tr>
+ <th>44728</th>
+ <td>44728</td>
+ <td>helena_seed_0_nrows_2000_nclasses_10_ncols_100...</td>
+ <td>2000.0</td>
+ <td>28.0</td>
+ <td>100.0</td>
+ </tr>
+ <tr>
+ <th>44729</th>
+ <td>44729</td>
+ <td>helena_seed_1_nrows_2000_nclasses_10_ncols_100...</td>
+ <td>2000.0</td>
+ <td>28.0</td>
+ <td>100.0</td>
+ </tr>
+ <tr>
+ <th>44730</th>
+ <td>44730</td>
+ <td>helena_seed_2_nrows_2000_nclasses_10_ncols_100...</td>
+ <td>2000.0</td>
+ <td>28.0</td>
+ <td>100.0</td>
+ </tr>
+ <tr>
+ <th>44731</th>
+ <td>44731</td>
+ <td>helena_seed_3_nrows_2000_nclasses_10_ncols_100...</td>
+ <td>2000.0</td>
+ <td>28.0</td>
+ <td>100.0</td>
+ </tr>
+ <tr>
+ <th>44732</th>
+ <td>44732</td>
+ <td>helena_seed_4_nrows_2000_nclasses_10_ncols_100...</td>
+ <td>2000.0</td>
+ <td>28.0</td>
+ <td>100.0</td>
+ </tr>
+ <tr>
+ <th>45049</th>
+ <td>45049</td>
+ <td>MD_MIX_Mini_Copy</td>
+ <td>28240.0</td>
+ <td>69.0</td>
+ <td>706.0</td>
+ </tr>
+ <tr>
+ <th>45102</th>
+ <td>45102</td>
+ <td>dailybike</td>
+ <td>731.0</td>
+ <td>13.0</td>
+ <td>606.0</td>
+ </tr>
+ <tr>
+ <th>45103</th>
+ <td>45103</td>
+ <td>dailybike</td>
+ <td>731.0</td>
+ <td>13.0</td>
+ <td>606.0</td>
+ </tr>
+ <tr>
+ <th>45104</th>
+ <td>45104</td>
+ <td>PLK_Mini_Copy</td>
+ <td>3440.0</td>
+ <td>3.0</td>
+ <td>86.0</td>
+ </tr>
+ <tr>
+ <th>45274</th>
+ <td>45274</td>
+ <td>PASS</td>
+ <td>1439588.0</td>
+ <td>7.0</td>
+ <td>94137.0</td>
+ </tr>
+ <tr>
+ <th>45569</th>
+ <td>45569</td>
+ <td>DBLP-QuAD</td>
+ <td>10000.0</td>
+ <td>10.0</td>
+ <td>9999.0</td>
+ </tr>
+ </tbody>
+</table>
+</div>
+</div>
+<br />
+<br />
+
+### Download datasets
+
+``` default
+# This is done based on the dataset ID.
+dataset = openml.datasets.get_dataset(1471)
+
+# Print a summary
+print(
+ f"This is dataset '{dataset.name}', the target feature is "
+ f"'{dataset.default_target_attribute}'"
+)
+print(f"URL: {dataset.url}")
+print(dataset.description[:500])
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+This is dataset 'eeg-eye-state', the target feature is 'Class'
+URL: https://api.openml.org/data/v1/download/1587924/eeg-eye-state.arff
+**Author**: Oliver Roesler
+**Source**: [UCI](https://archive.ics.uci.edu/ml/datasets/EEG+Eye+State), Baden-Wuerttemberg, Cooperative State University (DHBW), Stuttgart, Germany
+**Please cite**: [UCI](https://archive.ics.uci.edu/ml/citation_policy.html)
+
+All data is from one continuous EEG measurement with the Emotiv EEG Neuroheadset. The duration of the measurement was 117 seconds. The eye state was detected via a camera during the EEG measurement and added later manually to the file after
+```
+:::
+
+Get the actual data.
+
+openml-python returns data as pandas dataframes (stored in the
+[eeg]{.title-ref} variable below), and also some additional metadata
+that we don\'t care about right now.
+
+``` default
+eeg, *_ = dataset.get_data()
+```
+
+You can optionally choose to have openml separate out a column from the
+dataset. In particular, many datasets for supervised problems have a set
+[default_target_attribute]{.title-ref} which may help identify the
+target variable.
+
+``` default
+X, y, categorical_indicator, attribute_names = dataset.get_data(
+ target=dataset.default_target_attribute
+)
+print(X.head())
+print(X.info())
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+V1 V2 V3 V4 ... V11 V12 V13 V14
+0 4329.23 4009.23 4289.23 4148.21 ... 4211.28 4280.51 4635.90 4393.85
+1 4324.62 4004.62 4293.85 4148.72 ... 4207.69 4279.49 4632.82 4384.10
+2 4327.69 4006.67 4295.38 4156.41 ... 4206.67 4282.05 4628.72 4389.23
+3 4328.72 4011.79 4296.41 4155.90 ... 4210.77 4287.69 4632.31 4396.41
+4 4326.15 4011.79 4292.31 4151.28 ... 4212.82 4288.21 4632.82 4398.46
+
+[5 rows x 14 columns]
+<class 'pandas.core.frame.DataFrame'>
+RangeIndex: 14980 entries, 0 to 14979
+Data columns (total 14 columns):
+# Column Non-Null Count Dtype
+--- ------ -------------- -----
+0 V1 14980 non-null float64
+1 V2 14980 non-null float64
+2 V3 14980 non-null float64
+3 V4 14980 non-null float64
+4 V5 14980 non-null float64
+5 V6 14980 non-null float64
+6 V7 14980 non-null float64
+7 V8 14980 non-null float64
+8 V9 14980 non-null float64
+9 V10 14980 non-null float64
+10 V11 14980 non-null float64
+11 V12 14980 non-null float64
+12 V13 14980 non-null float64
+13 V14 14980 non-null float64
+dtypes: float64(14)
+memory usage: 1.6 MB
+None
+```
+:::
+
+Sometimes you only need access to a dataset\'s metadata. In those cases,
+you can download the dataset without downloading the data file. The
+dataset object can be used as normal. Whenever you use any functionality
+that requires the data, such as [get_data]{.title-ref}, the data will be
+downloaded. Starting from 0.15, not downloading data will be the default
+behavior instead. The data will be downloading automatically when you
+try to access it through openml objects, e.g., using
+[dataset.features]{.title-ref}.
+
+``` default
+dataset = openml.datasets.get_dataset(1471, download_data=False)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+```
+:::
+
+## Exercise 2
+
+- Explore the data visually.
+
+``` default
+eegs = eeg.sample(n=1000)
+_ = pd.plotting.scatter_matrix(
+ X.iloc[:100, :4],
+ c=y[:100],
+ figsize=(10, 10),
+ marker="o",
+ hist_kwds={"bins": 20},
+ alpha=0.8,
+ cmap="plasma",
+)
+```
+
+::: {.image-sg .sphx-glr-single-img alt="datasets tutorial" srcset="/examples/30_extended/images/sphx_glr_datasets_tutorial_001.png"}
+/examples/30_extended/images/sphx_glr_datasets_tutorial_001.png
+:::
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/openml/venv/lib/python3.10/site-packages/pandas/plotting/_matplotlib/misc.py:97: UserWarning: No data for colormapping provided via 'c'. Parameters 'cmap' will be ignored
+ ax.scatter(
+```
+:::
+
+### Edit a created dataset
+
+This example uses the test server, to avoid editing a dataset on the
+main server.
+
+:::: warning
+::: title
+Warning
+:::
+
+.. include:: ../../test_server_usage_warning.txt
+::::
+
+``` default
+openml.config.start_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!
+ warnings.warn(
+```
+:::
+
+Edit non-critical fields, allowed for all authorized users: description,
+creator, contributor, collection_date, language, citation,
+original_data_url, paper_url
+
+``` default
+desc = (
+ "This data sets consists of 3 different types of irises' "
+ "(Setosa, Versicolour, and Virginica) petal and sepal length,"
+ " stored in a 150x4 numpy.ndarray"
+)
+did = 128
+data_id = edit_dataset(
+ did,
+ description=desc,
+ creator="R.A.Fisher",
+ collection_date="1937",
+ citation="The use of multiple measurements in taxonomic problems",
+ language="English",
+)
+edited_dataset = get_dataset(data_id)
+print(f"Edited dataset ID: {data_id}")
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+Edited dataset ID: 128
+```
+:::
+
+Editing critical fields (default_target_attribute, row_id_attribute,
+ignore_attribute) is allowed only for the dataset owner. Further,
+critical fields cannot be edited if the dataset has any tasks associated
+with it. To edit critical fields of a dataset (without tasks) owned by
+you, configure the API key: openml.config.apikey =
+\'FILL_IN_OPENML_API_KEY\' This example here only shows a failure when
+trying to work on a dataset not owned by you:
+
+``` default
+try:
+ data_id = edit_dataset(1, default_target_attribute="shape")
+except openml.exceptions.OpenMLServerException as e:
+ print(e)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+https://test.openml.org/api/v1/xml/data/edit returned code 1065: Critical features default_target_attribute, row_id_attribute and ignore_attribute can be edited only by the owner. Fork the dataset if changes are required. - None
+```
+:::
+
+### Fork dataset
+
+Used to create a copy of the dataset with you as the owner. Use this API
+only if you are unable to edit the critical fields
+(default_target_attribute, ignore_attribute, row_id_attribute) of a
+dataset through the edit_dataset API. After the dataset is forked, you
+can edit the new version of the dataset using edit_dataset.
+
+``` default
+data_id = fork_dataset(1)
+print(data_id)
+data_id = edit_dataset(data_id, default_target_attribute="shape")
+print(f"Forked dataset ID: {data_id}")
+
+openml.config.stop_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+1306
+Forked dataset ID: 1306
+```
+:::
+
+::: rst-class
+sphx-glr-timing
+
+**Total running time of the script:** ( 0 minutes 8.138 seconds)
+:::
+
+::::::: {#sphx_glr_download_examples_30_extended_datasets_tutorial.py}
+:::::: only
+html
+
+::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
+::: {.container .sphx-glr-download .sphx-glr-download-python}
+`Download Python source code: datasets_tutorial.py <datasets_tutorial.py>`{.interpreted-text
+role="download"}
+:::
+
+::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
+`Download Jupyter notebook: datasets_tutorial.ipynb <datasets_tutorial.ipynb>`{.interpreted-text
+role="download"}
+:::
+:::::
+::::::
+:::::::
+
+:::: only
+html
+
+::: rst-class
+sphx-glr-signature
+
+[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
+:::
+::::
diff --git a/docs/examples/30_extended/fetch_evaluations_tutorial.rst.md b/docs/examples/30_extended/fetch_evaluations_tutorial.rst.md
new file mode 100644
index 000000000..953363f9a
--- /dev/null
+++ b/docs/examples/30_extended/fetch_evaluations_tutorial.rst.md
@@ -0,0 +1,360 @@
+::::: only
+html
+
+:::: {.note .sphx-glr-download-link-note}
+::: title
+Note
+:::
+
+`Go to the end <sphx_glr_download_examples_30_extended_fetch_evaluations_tutorial.py>`{.interpreted-text
+role="ref"} to download the full example code
+::::
+:::::
+
+::: rst-class
+sphx-glr-example-title
+:::
+
+# Fetching Evaluations {#sphx_glr_examples_30_extended_fetch_evaluations_tutorial.py}
+
+Evaluations contain a concise summary of the results of all runs made.
+Each evaluation provides information on the dataset used, the flow
+applied, the setup used, the metric evaluated, and the result obtained
+on the metric, for each such run made. These collection of results can
+be used for efficient benchmarking of an algorithm and also allow
+transparent reuse of results from previous experiments on similar
+parameters.
+
+In this example, we shall do the following:
+
+- Retrieve evaluations based on different metrics
+- Fetch evaluations pertaining to a specific task
+- Sort the obtained results in descending order of the metric
+- Plot a cumulative distribution function for the evaluations
+- Compare the top 10 performing flows based on the evaluation
+ performance
+- Retrieve evaluations with hyperparameter settings
+
+``` default
+# License: BSD 3-Clause
+
+import openml
+```
+
+## Listing evaluations
+
+Evaluations can be retrieved from the database in the chosen output
+format. Required filters can be applied to retrieve results from runs as
+required.
+
+``` default
+# We shall retrieve a small set (only 10 entries) to test the listing function for evaluations
+openml.evaluations.list_evaluations(
+ function="predictive_accuracy", size=10, output_format="dataframe"
+)
+
+# Using other evaluation metrics, 'precision' in this case
+evals = openml.evaluations.list_evaluations(
+ function="precision", size=10, output_format="dataframe"
+)
+
+# Querying the returned results for precision above 0.98
+print(evals[evals.value > 0.98])
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+run_id task_id ... values array_data
+0 62 1 ... None [0.714286,0.98,0.992658,0,0.985294,0.904762]
+1 237 1 ... None [1,0.942857,0.991215,0,1,0.95]
+3 413 1 ... None [1,0.980198,0.994152,0,1,0.948718]
+4 500 1 ... None [1,0.99,0.997059,0,0.985294,0.863636]
+
+[4 rows x 14 columns]
+```
+:::
+
+### Viewing a sample task
+
+Over here we shall briefly take a look at the details of the task.
+
+``` default
+# We will start by displaying a simple *supervised classification* task:
+task_id = 167140 # https://www.openml.org/t/167140
+task = openml.tasks.get_task(task_id)
+print(task)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+OpenML Classification Task
+==========================
+Task Type Description: https://www.openml.org/tt/TaskType.SUPERVISED_CLASSIFICATION
+Task ID..............: 167140
+Task URL.............: https://www.openml.org/t/167140
+Estimation Procedure.: crossvalidation
+Target Feature.......: class
+# of Classes.........: 3
+Cost Matrix..........: Available
+```
+:::
+
+### Obtaining all the evaluations for the task
+
+We\'ll now obtain all the evaluations that were uploaded for the task we
+displayed previously. Note that we now filter the evaluations based on
+another parameter \'task\'.
+
+``` default
+metric = "predictive_accuracy"
+evals = openml.evaluations.list_evaluations(
+ function=metric, tasks=[task_id], output_format="dataframe"
+)
+# Displaying the first 10 rows
+print(evals.head(n=10))
+# Sorting the evaluations in decreasing order of the metric chosen
+evals = evals.sort_values(by="value", ascending=False)
+print("\nDisplaying head of sorted dataframe: ")
+print(evals.head())
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+run_id task_id setup_id ... value values array_data
+0 9199772 167140 7130140 ... 0.938481 None None
+1 9199845 167140 7130139 ... 0.623352 None None
+2 9202086 167140 7131884 ... 0.918393 None None
+3 9202092 167140 7131890 ... 0.962335 None None
+4 9202096 167140 7131894 ... 0.961707 None None
+5 9202099 167140 7131897 ... 0.519146 None None
+6 9202125 167140 7131923 ... 0.957313 None None
+7 9202139 167140 7131937 ... 0.519146 None None
+8 9202159 167140 7131957 ... 0.628060 None None
+9 9202254 167140 7132052 ... 0.610483 None None
+
+[10 rows x 14 columns]
+
+Displaying head of sorted dataframe:
+ run_id task_id setup_id ... value values array_data
+2788 10279797 167140 8166877 ... 0.968613 None None
+2789 10279798 167140 8166877 ... 0.968613 None None
+2246 10232158 167140 8157180 ... 0.967671 None None
+2263 10232987 167140 8157961 ... 0.967671 None None
+2314 10271558 167140 8154506 ... 0.967043 None None
+
+[5 rows x 14 columns]
+```
+:::
+
+## Obtaining CDF of metric for chosen task
+
+We shall now analyse how the performance of various flows have been on
+this task, by seeing the likelihood of the accuracy obtained across all
+runs. We shall now plot a cumulative distributive function (CDF) for the
+accuracies obtained.
+
+``` default
+from matplotlib import pyplot as plt
+
+
+def plot_cdf(values, metric="predictive_accuracy"):
+ max_val = max(values)
+ n, bins, patches = plt.hist(values, density=True, histtype="step", cumulative=True, linewidth=3)
+ patches[0].set_xy(patches[0].get_xy()[:-1])
+ plt.xlim(max(0, min(values) - 0.1), 1)
+ plt.title("CDF")
+ plt.xlabel(metric)
+ plt.ylabel("Likelihood")
+ plt.grid(visible=True, which="major", linestyle="-")
+ plt.minorticks_on()
+ plt.grid(visible=True, which="minor", linestyle="--")
+ plt.axvline(max_val, linestyle="--", color="gray")
+ plt.text(max_val, 0, "%.3f" % max_val, fontsize=9)
+ plt.show()
+
+
+plot_cdf(evals.value, metric)
+# This CDF plot shows that for the given task, based on the results of the
+# runs uploaded, it is almost certain to achieve an accuracy above 52%, i.e.,
+# with non-zero probability. While the maximum accuracy seen till now is 96.5%.
+```
+
+::: {.image-sg .sphx-glr-single-img alt="CDF" srcset="/examples/30_extended/images/sphx_glr_fetch_evaluations_tutorial_001.png"}
+/examples/30_extended/images/sphx_glr_fetch_evaluations_tutorial_001.png
+:::
+
+## Comparing top 10 performing flows
+
+Let us now try to see which flows generally performed the best for this
+task. For this, we shall compare the top performing flows.
+
+``` default
+import numpy as np
+import pandas as pd
+
+
+def plot_flow_compare(evaluations, top_n=10, metric="predictive_accuracy"):
+ # Collecting the top 10 performing unique flow_id
+ flow_ids = evaluations.flow_id.unique()[:top_n]
+
+ df = pd.DataFrame()
+ # Creating a data frame containing only the metric values of the selected flows
+ # assuming evaluations is sorted in decreasing order of metric
+ for i in range(len(flow_ids)):
+ flow_values = evaluations[evaluations.flow_id == flow_ids[i]].value
+ df = pd.concat([df, flow_values], ignore_index=True, axis=1)
+ fig, axs = plt.subplots()
+ df.boxplot()
+ axs.set_title("Boxplot comparing " + metric + " for different flows")
+ axs.set_ylabel(metric)
+ axs.set_xlabel("Flow ID")
+ axs.set_xticklabels(flow_ids)
+ axs.grid(which="major", linestyle="-", linewidth="0.5", color="gray", axis="y")
+ axs.minorticks_on()
+ axs.grid(which="minor", linestyle="--", linewidth="0.5", color="gray", axis="y")
+ # Counting the number of entries for each flow in the data frame
+ # which gives the number of runs for each flow
+ flow_freq = list(df.count(axis=0, numeric_only=True))
+ for i in range(len(flow_ids)):
+ axs.text(i + 1.05, np.nanmin(df.values), str(flow_freq[i]) + "\nrun(s)", fontsize=7)
+ plt.show()
+
+
+plot_flow_compare(evals, metric=metric, top_n=10)
+# The boxplots below show how the flows perform across multiple runs on the chosen
+# task. The green horizontal lines represent the median accuracy of all the runs for
+# that flow (number of runs denoted at the bottom of the boxplots). The higher the
+# green line, the better the flow is for the task at hand. The ordering of the flows
+# are in the descending order of the higest accuracy value seen under that flow.
+
+# Printing the corresponding flow names for the top 10 performing flow IDs
+top_n = 10
+flow_ids = evals.flow_id.unique()[:top_n]
+flow_names = evals.flow_name.unique()[:top_n]
+for i in range(top_n):
+ print((flow_ids[i], flow_names[i]))
+```
+
+::: {.image-sg .sphx-glr-single-img alt="Boxplot comparing predictive_accuracy for different flows" srcset="/examples/30_extended/images/sphx_glr_fetch_evaluations_tutorial_002.png"}
+/examples/30_extended/images/sphx_glr_fetch_evaluations_tutorial_002.png
+:::
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+(12736, 'sklearn.pipeline.Pipeline(simpleimputer=sklearn.impute._base.SimpleImputer,histgradientboostingclassifier=sklearn.ensemble._hist_gradient_boosting.gradient_boosting.HistGradientBoostingClassifier)(1)')
+(8399, 'sklearn.model_selection._search.RandomizedSearchCV(estimator=sklearn.pipeline.Pipeline(imputation=hyperimp.utils.preprocessing.ConditionalImputer,hotencoding=sklearn.preprocessing.data.OneHotEncoder,scaling=sklearn.preprocessing.data.StandardScaler,variencethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,clf=sklearn.svm.classes.SVC))(1)')
+(8353, 'sklearn.pipeline.Pipeline(imputation=hyperimp.utils.preprocessing.ConditionalImputer2,hotencoding=sklearn.preprocessing.data.OneHotEncoder,scaling=sklearn.preprocessing.data.StandardScaler,variencethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,clf=sklearn.svm.classes.SVC)(1)')
+(12127, 'sklearn.ensemble._hist_gradient_boosting.gradient_boosting.HistGradientBoostingClassifier(1)')
+(16347, 'sklearn.pipeline.Pipeline(simpleimputer=sklearn.impute._base.SimpleImputer,onehotencoder=sklearn.preprocessing._encoders.OneHotEncoder,svc=sklearn.svm.classes.SVC)(1)')
+(17373, 'sklearn.model_selection._search_successive_halving.HalvingRandomSearchCV(estimator=sklearn.ensemble._hist_gradient_boosting.gradient_boosting.HistGradientBoostingClassifier)(4)')
+(8317, 'sklearn.pipeline.Pipeline(imputation=hyperimp.utils.preprocessing.ConditionalImputer,hotencoding=sklearn.preprocessing.data.OneHotEncoder,scaling=sklearn.preprocessing.data.StandardScaler,variencethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,clf=sklearn.svm.classes.SVC)(1)')
+(17369, 'sklearn.model_selection._search_successive_halving.HalvingRandomSearchCV(estimator=sklearn.ensemble._hist_gradient_boosting.gradient_boosting.HistGradientBoostingClassifier)(3)')
+(17374, 'sklearn.model_selection._search.RandomizedSearchCV(estimator=sklearn.ensemble._hist_gradient_boosting.gradient_boosting.HistGradientBoostingClassifier)(4)')
+(17371, 'sklearn.model_selection._search.RandomizedSearchCV(estimator=sklearn.ensemble._hist_gradient_boosting.gradient_boosting.HistGradientBoostingClassifier)(3)')
+```
+:::
+
+### Obtaining evaluations with hyperparameter settings
+
+We\'ll now obtain the evaluations of a task and a flow with the
+hyperparameters
+
+``` default
+# List evaluations in descending order based on predictive_accuracy with
+# hyperparameters
+evals_setups = openml.evaluations.list_evaluations_setups(
+ function="predictive_accuracy", tasks=[31], size=100, sort_order="desc"
+)
+
+""
+print(evals_setups.head())
+
+""
+# Return evaluations for flow_id in descending order based on predictive_accuracy
+# with hyperparameters. parameters_in_separate_columns returns parameters in
+# separate columns
+evals_setups = openml.evaluations.list_evaluations_setups(
+ function="predictive_accuracy", flows=[6767], size=100, parameters_in_separate_columns=True
+)
+
+""
+print(evals_setups.head(10))
+
+""
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+run_id ... parameters
+0 6725054 ... {'mlr.classif.ranger(13)_num.trees': '48', 'ml...
+1 2083190 ... {'sklearn.ensemble.forest.RandomForestClassifi...
+2 3962979 ... {'mlr.classif.ranger(13)_num.trees': '42', 'ml...
+3 2083596 ... {'sklearn.ensemble.forest.RandomForestClassifi...
+4 6162649 ... {'mlr.classif.ranger(13)_num.trees': '715', 'm...
+
+[5 rows x 15 columns]
+run_id ... mlr.classif.xgboost(9)_nthread
+0 2420531 ... NaN
+1 2420532 ... NaN
+2 2420533 ... NaN
+3 2420534 ... NaN
+4 2420535 ... NaN
+5 2420536 ... NaN
+6 2420537 ... NaN
+7 2431903 ... NaN
+8 2432142 ... NaN
+9 2432377 ... NaN
+
+[10 rows x 29 columns]
+
+''
+```
+:::
+
+::: rst-class
+sphx-glr-timing
+
+**Total running time of the script:** ( 0 minutes 13.992 seconds)
+:::
+
+::::::: {#sphx_glr_download_examples_30_extended_fetch_evaluations_tutorial.py}
+:::::: only
+html
+
+::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
+::: {.container .sphx-glr-download .sphx-glr-download-python}
+`Download Python source code: fetch_evaluations_tutorial.py <fetch_evaluations_tutorial.py>`{.interpreted-text
+role="download"}
+:::
+
+::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
+`Download Jupyter notebook: fetch_evaluations_tutorial.ipynb <fetch_evaluations_tutorial.ipynb>`{.interpreted-text
+role="download"}
+:::
+:::::
+::::::
+:::::::
+
+:::: only
+html
+
+::: rst-class
+sphx-glr-signature
+
+[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
+:::
+::::
diff --git a/docs/examples/30_extended/fetch_runtimes_tutorial.rst.md b/docs/examples/30_extended/fetch_runtimes_tutorial.rst.md
new file mode 100644
index 000000000..a5235fa32
--- /dev/null
+++ b/docs/examples/30_extended/fetch_runtimes_tutorial.rst.md
@@ -0,0 +1,1135 @@
+::::: only
+html
+
+:::: {.note .sphx-glr-download-link-note}
+::: title
+Note
+:::
+
+`Go to the end <sphx_glr_download_examples_30_extended_fetch_runtimes_tutorial.py>`{.interpreted-text
+role="ref"} to download the full example code
+::::
+:::::
+
+::: rst-class
+sphx-glr-example-title
+:::
+
+# Measuring runtimes for Scikit-learn models {#sphx_glr_examples_30_extended_fetch_runtimes_tutorial.py}
+
+The runtime of machine learning models on specific datasets can be a
+deciding factor on the choice of algorithms, especially for benchmarking
+and comparison purposes. OpenML\'s scikit-learn extension provides
+runtime data from runs of model fit and prediction on tasks or datasets,
+for both the CPU-clock as well as the actual wallclock-time incurred.
+The objective of this example is to illustrate how to retrieve such
+timing measures, and also offer some potential means of usage and
+interpretation of the same.
+
+It should be noted that there are multiple levels at which parallelism
+can occur.
+
+- At the outermost level, OpenML tasks contain fixed data splits, on
+ which the defined model/flow is executed. Thus, a model can be fit
+ on each OpenML dataset fold in parallel using the
+ [n_jobs]{.title-ref} parameter to [run_model_on_task]{.title-ref} or
+ [run_flow_on_task]{.title-ref} (illustrated under Case 2 & 3 below).
+- The model/flow specified can also include scikit-learn models that
+ perform their own parallelization. For instance, by specifying
+ [n_jobs]{.title-ref} in a Random Forest model definition (covered
+ under Case 2 below).
+- The sklearn model can further be an HPO estimator and contain it\'s
+ own parallelization. If the base estimator used also supports
+ [parallelization]{.title-ref}, then there\'s at least a 2-level
+ nested definition for parallelization possible (covered under Case 3
+ below).
+
+We shall cover these 5 representative scenarios for:
+
+- (Case 1) Retrieving runtimes for Random Forest training and
+ prediction on each of the cross-validation folds
+- (Case 2) Testing the above setting in a parallel setup and monitor
+ the difference using runtimes retrieved
+- (Case 3) Comparing RandomSearchCV and GridSearchCV on the above task
+ based on runtimes
+- (Case 4) Running models that don\'t run in parallel or models which
+ scikit-learn doesn\'t parallelize
+- (Case 5) Running models that do not release the Python Global
+ Interpreter Lock (GIL)
+
+``` default
+# License: BSD 3-Clause
+
+import openml
+import numpy as np
+from matplotlib import pyplot as plt
+from joblib.parallel import parallel_backend
+
+from sklearn.naive_bayes import GaussianNB
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.neural_network import MLPClassifier
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
+```
+
+## Preparing tasks and scikit-learn models
+
+``` default
+task_id = 167119
+
+task = openml.tasks.get_task(task_id)
+print(task)
+
+# Viewing associated data
+n_repeats, n_folds, n_samples = task.get_split_dimensions()
+print(
+ "Task {}: number of repeats: {}, number of folds: {}, number of samples {}.".format(
+ task_id,
+ n_repeats,
+ n_folds,
+ n_samples,
+ )
+)
+
+
+# Creating utility function
+def print_compare_runtimes(measures):
+ for repeat, val1 in measures["usercpu_time_millis_training"].items():
+ for fold, val2 in val1.items():
+ print(
+ "Repeat #{}-Fold #{}: CPU-{:.3f} vs Wall-{:.3f}".format(
+ repeat, fold, val2, measures["wall_clock_time_millis_training"][repeat][fold]
+ )
+ )
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+OpenML Classification Task
+==========================
+Task Type Description: https://www.openml.org/tt/TaskType.SUPERVISED_CLASSIFICATION
+Task ID..............: 167119
+Task URL.............: https://www.openml.org/t/167119
+Estimation Procedure.: crossvalidation
+Target Feature.......: class
+# of Classes.........: 3
+Cost Matrix..........: Available
+Task 167119: number of repeats: 1, number of folds: 10, number of samples 1.
+```
+:::
+
+## Case 1: Running a Random Forest model on an OpenML task
+
+We\'ll run a Random Forest model and obtain an OpenML run object. We can
+see the evaluations recorded per fold for the dataset and the
+information available for this run.
+
+``` default
+clf = RandomForestClassifier(n_estimators=10)
+
+run1 = openml.runs.run_model_on_task(
+ model=clf,
+ task=task,
+ upload_flow=False,
+ avoid_duplicate_runs=False,
+)
+measures = run1.fold_evaluations
+
+print("The timing and performance metrics available: ")
+for key in measures.keys():
+ print(key)
+print()
+
+print(
+ "The performance metric is recorded under `predictive_accuracy` per "
+ "fold and can be retrieved as: "
+)
+for repeat, val1 in measures["predictive_accuracy"].items():
+ for fold, val2 in val1.items():
+ print("Repeat #{}-Fold #{}: {:.4f}".format(repeat, fold, val2))
+ print()
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+The timing and performance metrics available:
+usercpu_time_millis_training
+wall_clock_time_millis_training
+usercpu_time_millis_testing
+usercpu_time_millis
+wall_clock_time_millis_testing
+wall_clock_time_millis
+predictive_accuracy
+
+The performance metric is recorded under `predictive_accuracy` per fold and can be retrieved as:
+Repeat #0-Fold #0: 0.7834
+Repeat #0-Fold #1: 0.7773
+Repeat #0-Fold #2: 0.7813
+Repeat #0-Fold #3: 0.7811
+Repeat #0-Fold #4: 0.7863
+Repeat #0-Fold #5: 0.7829
+Repeat #0-Fold #6: 0.7718
+Repeat #0-Fold #7: 0.7798
+Repeat #0-Fold #8: 0.7825
+Repeat #0-Fold #9: 0.7735
+```
+:::
+
+The remaining entries recorded in [measures]{.title-ref} are the runtime
+records related as:
+
+usercpu_time_millis = usercpu_time_millis_training +
+usercpu_time_millis_testing
+
+wall_clock_time_millis = wall_clock_time_millis_training +
+wall_clock_time_millis_testing
+
+The timing measures recorded as [\*\_millis_training]{.title-ref}
+contain the per repeat-per fold timing incurred for the execution of the
+[.fit()]{.title-ref} procedure of the model. For
+[usercpu_time\_\*]{.title-ref} the time recorded using
+[time.process_time()]{.title-ref} is converted to
+[milliseconds]{.title-ref} and stored. Similarly,
+[time.time()]{.title-ref} is used to record the time entry for
+[wall_clock_time\_\*]{.title-ref}. The [\*\_millis_testing]{.title-ref}
+entry follows the same procedure but for time taken for the
+[.predict()]{.title-ref} procedure.
+
+``` default
+# Comparing the CPU and wall-clock training times of the Random Forest model
+print_compare_runtimes(measures)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+Repeat #0-Fold #0: CPU-235.799 vs Wall-235.806
+Repeat #0-Fold #1: CPU-234.915 vs Wall-234.942
+Repeat #0-Fold #2: CPU-215.000 vs Wall-215.677
+Repeat #0-Fold #3: CPU-218.780 vs Wall-218.210
+Repeat #0-Fold #4: CPU-210.419 vs Wall-210.423
+Repeat #0-Fold #5: CPU-210.813 vs Wall-211.084
+Repeat #0-Fold #6: CPU-210.696 vs Wall-218.624
+Repeat #0-Fold #7: CPU-206.779 vs Wall-206.782
+Repeat #0-Fold #8: CPU-219.276 vs Wall-218.185
+Repeat #0-Fold #9: CPU-211.948 vs Wall-211.660
+```
+:::
+
+## Case 2: Running Scikit-learn model on an OpenML task in parallel
+
+Redefining the model to allow parallelism with [n_jobs=2]{.title-ref} (2
+cores)
+
+``` default
+clf = RandomForestClassifier(n_estimators=10, n_jobs=2)
+
+run2 = openml.runs.run_model_on_task(
+ model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False
+)
+measures = run2.fold_evaluations
+# The wall-clock time recorded per fold should be lesser than Case 1 above
+print_compare_runtimes(measures)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+Repeat #0-Fold #0: CPU-291.231 vs Wall-177.857
+Repeat #0-Fold #1: CPU-247.121 vs Wall-160.623
+Repeat #0-Fold #2: CPU-299.050 vs Wall-171.314
+Repeat #0-Fold #3: CPU-219.565 vs Wall-130.540
+Repeat #0-Fold #4: CPU-214.352 vs Wall-128.408
+Repeat #0-Fold #5: CPU-214.530 vs Wall-129.069
+Repeat #0-Fold #6: CPU-216.249 vs Wall-131.271
+Repeat #0-Fold #7: CPU-214.707 vs Wall-129.280
+Repeat #0-Fold #8: CPU-215.784 vs Wall-130.975
+Repeat #0-Fold #9: CPU-224.041 vs Wall-127.277
+```
+:::
+
+Running a Random Forest model on an OpenML task in parallel (all cores
+available):
+
+``` default
+# Redefining the model to use all available cores with `n_jobs=-1`
+clf = RandomForestClassifier(n_estimators=10, n_jobs=-1)
+
+run3 = openml.runs.run_model_on_task(
+ model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False
+)
+measures = run3.fold_evaluations
+# The wall-clock time recorded per fold should be lesser than the case above,
+# if more than 2 CPU cores are available. The speed-up is more pronounced for
+# larger datasets.
+print_compare_runtimes(measures)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+Repeat #0-Fold #0: CPU-223.135 vs Wall-141.702
+Repeat #0-Fold #1: CPU-252.852 vs Wall-157.798
+Repeat #0-Fold #2: CPU-225.624 vs Wall-144.946
+Repeat #0-Fold #3: CPU-215.932 vs Wall-137.153
+Repeat #0-Fold #4: CPU-218.327 vs Wall-132.138
+Repeat #0-Fold #5: CPU-216.836 vs Wall-128.753
+Repeat #0-Fold #6: CPU-217.526 vs Wall-141.789
+Repeat #0-Fold #7: CPU-218.256 vs Wall-135.525
+Repeat #0-Fold #8: CPU-218.904 vs Wall-134.427
+Repeat #0-Fold #9: CPU-221.687 vs Wall-130.505
+```
+:::
+
+We can now observe that the ratio of CPU time to wallclock time is lower
+than in case 1. This happens because joblib by default spawns
+subprocesses for the workloads for which CPU time cannot be tracked.
+Therefore, interpreting the reported CPU and wallclock time requires
+knowledge of the parallelization applied at runtime.
+
+Running the same task with a different parallel backend. Joblib provides
+multiple backends: {[loky]{.title-ref} (default),
+[multiprocessing]{.title-ref}, [dask]{.title-ref},
+[threading]{.title-ref}, [sequential]{.title-ref}}. The backend can be
+explicitly set using a joblib context manager. The behaviour of the job
+distribution can change and therefore the scale of runtimes recorded
+too.
+
+``` default
+with parallel_backend(backend="multiprocessing", n_jobs=-1):
+ run3_ = openml.runs.run_model_on_task(
+ model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False
+ )
+measures = run3_.fold_evaluations
+print_compare_runtimes(measures)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+Repeat #0-Fold #0: CPU-324.893 vs Wall-545.866
+Repeat #0-Fold #1: CPU-248.720 vs Wall-412.322
+Repeat #0-Fold #2: CPU-269.108 vs Wall-424.729
+Repeat #0-Fold #3: CPU-264.310 vs Wall-401.721
+Repeat #0-Fold #4: CPU-262.941 vs Wall-379.026
+Repeat #0-Fold #5: CPU-242.700 vs Wall-376.538
+Repeat #0-Fold #6: CPU-257.596 vs Wall-418.498
+Repeat #0-Fold #7: CPU-233.435 vs Wall-437.289
+Repeat #0-Fold #8: CPU-257.681 vs Wall-409.761
+Repeat #0-Fold #9: CPU-282.320 vs Wall-428.106
+```
+:::
+
+The CPU time interpretation becomes ambiguous when jobs are distributed
+over an unknown number of cores or when subprocesses are spawned for
+which the CPU time cannot be tracked, as in the examples above. It is
+impossible for OpenML-Python to capture the availability of the number
+of cores/threads, their eventual utilisation and whether workloads are
+executed in subprocesses, for various cases that can arise as
+demonstrated in the rest of the example. Therefore, the final
+interpretation of the runtimes is left to the [user]{.title-ref}.
+
+## Case 3: Running and benchmarking HPO algorithms with their runtimes
+
+We shall now optimize a similar RandomForest model for the same task
+using scikit-learn\'s HPO support by using GridSearchCV to optimize our
+earlier RandomForest model\'s hyperparameter [n_estimators]{.title-ref}.
+Scikit-learn also provides a [refit_time\_]{.title-ref} for such HPO
+models, i.e., the time incurred by training and evaluating the model on
+the best found parameter setting. This is included in the
+[wall_clock_time_millis_training]{.title-ref} measure recorded.
+
+``` default
+from sklearn.model_selection import GridSearchCV
+
+
+clf = RandomForestClassifier(n_estimators=10, n_jobs=2)
+
+# GridSearchCV model
+n_iter = 5
+grid_pipe = GridSearchCV(
+ estimator=clf,
+ param_grid={"n_estimators": np.linspace(start=1, stop=50, num=n_iter).astype(int).tolist()},
+ cv=2,
+ n_jobs=2,
+)
+
+run4 = openml.runs.run_model_on_task(
+ model=grid_pipe, task=task, upload_flow=False, avoid_duplicate_runs=False, n_jobs=2
+)
+measures = run4.fold_evaluations
+print_compare_runtimes(measures)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+Repeat #0-Fold #0: CPU-4216.657 vs Wall-3927.152
+Repeat #0-Fold #1: CPU-4267.256 vs Wall-3532.315
+Repeat #0-Fold #2: CPU-4153.419 vs Wall-3756.332
+Repeat #0-Fold #3: CPU-4165.637 vs Wall-3484.633
+Repeat #0-Fold #4: CPU-4148.764 vs Wall-3386.673
+Repeat #0-Fold #5: CPU-4409.518 vs Wall-3800.539
+Repeat #0-Fold #6: CPU-4381.566 vs Wall-3520.530
+Repeat #0-Fold #7: CPU-4377.360 vs Wall-3966.456
+Repeat #0-Fold #8: CPU-4427.580 vs Wall-3939.647
+Repeat #0-Fold #9: CPU-4459.380 vs Wall-3545.670
+```
+:::
+
+Like any optimisation problem, scikit-learn\'s HPO estimators also
+generate a sequence of configurations which are evaluated, using which
+the best found configuration is tracked throughout the trace. The OpenML
+run object stores these traces as OpenMLRunTrace objects accessible
+using keys of the pattern (repeat, fold, iterations). Here
+[fold]{.title-ref} implies the outer-cross validation fold as obtained
+from the task data splits in OpenML. GridSearchCV here performs grid
+search over the inner-cross validation folds as parameterized by the
+[cv]{.title-ref} parameter. Since [GridSearchCV]{.title-ref} in this
+example performs a [2-fold]{.title-ref} cross validation, the runtime
+recorded per repeat-per fold in the run object is for the entire
+[fit()]{.title-ref} procedure of GridSearchCV thus subsuming the
+runtimes of the 2-fold (inner) CV search performed.
+
+``` default
+# We earlier extracted the number of repeats and folds for this task:
+print("# repeats: {}\n# folds: {}".format(n_repeats, n_folds))
+
+# To extract the training runtime of the first repeat, first fold:
+print(run4.fold_evaluations["wall_clock_time_millis_training"][0][0])
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+# repeats: 1
+# folds: 10
+3927.151918411255
+```
+:::
+
+To extract the training runtime of the 1-st repeat, 4-th (outer) fold
+and also to fetch the parameters and performance of the evaluations made
+during the 1-st repeat, 4-th fold evaluation by the Grid Search model.
+
+``` default
+_repeat = 0
+_fold = 3
+print(
+ "Total runtime for repeat {}'s fold {}: {:4f} ms".format(
+ _repeat, _fold, run4.fold_evaluations["wall_clock_time_millis_training"][_repeat][_fold]
+ )
+)
+for i in range(n_iter):
+ key = (_repeat, _fold, i)
+ r = run4.trace.trace_iterations[key]
+ print(
+ "n_estimators: {:>2} - score: {:.3f}".format(
+ r.parameters["parameter_n_estimators"], r.evaluation
+ )
+ )
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+Total runtime for repeat 0's fold 3: 3484.633446 ms
+n_estimators: 1 - score: 0.759
+n_estimators: 13 - score: 0.799
+n_estimators: 25 - score: 0.803
+n_estimators: 37 - score: 0.804
+n_estimators: 50 - score: 0.803
+```
+:::
+
+Scikit-learn\'s HPO estimators also come with an argument
+[refit=True]{.title-ref} as a default. In our previous model definition
+it was set to True by default, which meant that the best found
+hyperparameter configuration was used to refit or retrain the model
+without any inner cross validation. This extra refit time measure is
+provided by the scikit-learn model as the attribute
+[refit_time\_]{.title-ref}. This time is included in the
+[wall_clock_time_millis_training]{.title-ref} measure.
+
+For non-HPO estimators, [wall_clock_time_millis =
+wall_clock_time_millis_training +
+wall_clock_time_millis_testing]{.title-ref}.
+
+For HPO estimators, [wall_clock_time_millis =
+wall_clock_time_millis_training + wall_clock_time_millis_testing +
+refit_time]{.title-ref}.
+
+This refit time can therefore be explicitly extracted in this manner:
+
+``` default
+def extract_refit_time(run, repeat, fold):
+ refit_time = (
+ run.fold_evaluations["wall_clock_time_millis"][repeat][fold]
+ - run.fold_evaluations["wall_clock_time_millis_training"][repeat][fold]
+ - run.fold_evaluations["wall_clock_time_millis_testing"][repeat][fold]
+ )
+ return refit_time
+
+
+for repeat in range(n_repeats):
+ for fold in range(n_folds):
+ print(
+ "Repeat #{}-Fold #{}: {:.4f}".format(
+ repeat, fold, extract_refit_time(run4, repeat, fold)
+ )
+ )
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+Repeat #0-Fold #0: 665.5693
+Repeat #0-Fold #1: 539.9771
+Repeat #0-Fold #2: 706.4621
+Repeat #0-Fold #3: 634.3455
+Repeat #0-Fold #4: 598.8989
+Repeat #0-Fold #5: 755.9273
+Repeat #0-Fold #6: 806.9417
+Repeat #0-Fold #7: 764.6010
+Repeat #0-Fold #8: 825.0468
+Repeat #0-Fold #9: 550.1802
+```
+:::
+
+Along with the GridSearchCV already used above, we demonstrate how such
+optimisation traces can be retrieved by showing an application of these
+traces - comparing the speed of finding the best configuration using
+RandomizedSearchCV and GridSearchCV available with scikit-learn.
+
+``` default
+# RandomizedSearchCV model
+rs_pipe = RandomizedSearchCV(
+ estimator=clf,
+ param_distributions={
+ "n_estimators": np.linspace(start=1, stop=50, num=15).astype(int).tolist()
+ },
+ cv=2,
+ n_iter=n_iter,
+ n_jobs=2,
+)
+run5 = openml.runs.run_model_on_task(
+ model=rs_pipe, task=task, upload_flow=False, avoid_duplicate_runs=False, n_jobs=2
+)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+```
+:::
+
+Since for the call to `openml.runs.run_model_on_task` the parameter
+`n_jobs` is set to its default `None`, the evaluations across the OpenML
+folds are not parallelized. Hence, the time recorded is agnostic to the
+`n_jobs` being set at both the HPO estimator `GridSearchCV` as well as
+the base estimator `RandomForestClassifier` in this case. The OpenML
+extension only records the time taken for the completion of the complete
+`fit()` call, per-repeat per-fold.
+
+This notion can be used to extract and plot the best found performance
+per fold by the HPO model and the corresponding time taken for search
+across that fold. Moreover, since `n_jobs=None` for
+`openml.runs.run_model_on_task` the runtimes per fold can be
+cumulatively added to plot the trace against time.
+
+``` default
+def extract_trace_data(run, n_repeats, n_folds, n_iter, key=None):
+ key = "wall_clock_time_millis_training" if key is None else key
+ data = {"score": [], "runtime": []}
+ for i_r in range(n_repeats):
+ for i_f in range(n_folds):
+ data["runtime"].append(run.fold_evaluations[key][i_r][i_f])
+ for i_i in range(n_iter):
+ r = run.trace.trace_iterations[(i_r, i_f, i_i)]
+ if r.selected:
+ data["score"].append(r.evaluation)
+ break
+ return data
+
+
+def get_incumbent_trace(trace):
+ best_score = 1
+ inc_trace = []
+ for i, r in enumerate(trace):
+ if i == 0 or (1 - r) < best_score:
+ best_score = 1 - r
+ inc_trace.append(best_score)
+ return inc_trace
+
+
+grid_data = extract_trace_data(run4, n_repeats, n_folds, n_iter)
+rs_data = extract_trace_data(run5, n_repeats, n_folds, n_iter)
+
+plt.clf()
+plt.plot(
+ np.cumsum(grid_data["runtime"]), get_incumbent_trace(grid_data["score"]), label="Grid Search"
+)
+plt.plot(
+ np.cumsum(rs_data["runtime"]), get_incumbent_trace(rs_data["score"]), label="Random Search"
+)
+plt.xscale("log")
+plt.yscale("log")
+plt.xlabel("Wallclock time (in milliseconds)")
+plt.ylabel("1 - Accuracy")
+plt.title("Optimisation Trace Comparison")
+plt.legend()
+plt.show()
+```
+
+::: {.image-sg .sphx-glr-single-img alt="Optimisation Trace Comparison" srcset="/examples/30_extended/images/sphx_glr_fetch_runtimes_tutorial_001.png"}
+/examples/30_extended/images/sphx_glr_fetch_runtimes_tutorial_001.png
+:::
+
+## Case 4: Running models that scikit-learn doesn\'t parallelize
+
+Both scikit-learn and OpenML depend on parallelism implemented through
+[joblib]{.title-ref}. However, there can be cases where either models
+cannot be parallelized or don\'t depend on joblib for its parallelism. 2
+such cases are illustrated below.
+
+Running a Decision Tree model that doesn\'t support parallelism
+implicitly, but using OpenML to parallelize evaluations for the
+outer-cross validation folds.
+
+``` default
+dt = DecisionTreeClassifier()
+
+run6 = openml.runs.run_model_on_task(
+ model=dt, task=task, upload_flow=False, avoid_duplicate_runs=False, n_jobs=2
+)
+measures = run6.fold_evaluations
+print_compare_runtimes(measures)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+Repeat #0-Fold #0: CPU-73.760 vs Wall-73.762
+Repeat #0-Fold #1: CPU-72.873 vs Wall-72.874
+Repeat #0-Fold #2: CPU-72.735 vs Wall-72.745
+Repeat #0-Fold #3: CPU-74.092 vs Wall-74.092
+Repeat #0-Fold #4: CPU-73.657 vs Wall-73.658
+Repeat #0-Fold #5: CPU-72.254 vs Wall-72.257
+Repeat #0-Fold #6: CPU-73.470 vs Wall-73.470
+Repeat #0-Fold #7: CPU-72.530 vs Wall-72.534
+Repeat #0-Fold #8: CPU-74.001 vs Wall-74.018
+Repeat #0-Fold #9: CPU-73.600 vs Wall-73.602
+```
+:::
+
+Although the decision tree does not run in parallel, it can release the
+[Python
+GIL](https://docs.python.org/dev/glossary.html#term-global-interpreter-lock).
+This can result in surprising runtime measures as demonstrated below:
+
+``` default
+with parallel_backend("threading", n_jobs=-1):
+ run7 = openml.runs.run_model_on_task(
+ model=dt, task=task, upload_flow=False, avoid_duplicate_runs=False
+ )
+measures = run7.fold_evaluations
+print_compare_runtimes(measures)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+Repeat #0-Fold #0: CPU-80.470 vs Wall-80.463
+Repeat #0-Fold #1: CPU-74.982 vs Wall-74.985
+Repeat #0-Fold #2: CPU-83.721 vs Wall-83.721
+Repeat #0-Fold #3: CPU-88.097 vs Wall-88.118
+Repeat #0-Fold #4: CPU-77.615 vs Wall-77.624
+Repeat #0-Fold #5: CPU-95.236 vs Wall-95.238
+Repeat #0-Fold #6: CPU-98.978 vs Wall-100.188
+Repeat #0-Fold #7: CPU-83.243 vs Wall-83.185
+Repeat #0-Fold #8: CPU-89.183 vs Wall-89.200
+Repeat #0-Fold #9: CPU-97.797 vs Wall-95.633
+```
+:::
+
+Running a Neural Network from scikit-learn that uses scikit-learn
+independent parallelism using libraries such as [MKL, OpenBLAS or
+BLIS](https://scikit-learn.org/stable/computing/parallelism.html#parallel-numpy-and-scipy-routines-from-numerical-libraries).
+
+``` default
+mlp = MLPClassifier(max_iter=10)
+
+run8 = openml.runs.run_model_on_task(
+ model=mlp, task=task, upload_flow=False, avoid_duplicate_runs=False
+)
+measures = run8.fold_evaluations
+print_compare_runtimes(measures)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/openml/venv/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/openml/venv/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/openml/venv/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/openml/venv/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/openml/venv/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/openml/venv/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/openml/venv/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/openml/venv/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/openml/venv/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/openml/venv/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.
+ warnings.warn(
+Repeat #0-Fold #0: CPU-866.453 vs Wall-866.533
+Repeat #0-Fold #1: CPU-918.903 vs Wall-756.131
+Repeat #0-Fold #2: CPU-997.260 vs Wall-727.315
+Repeat #0-Fold #3: CPU-859.797 vs Wall-657.584
+Repeat #0-Fold #4: CPU-922.494 vs Wall-701.211
+Repeat #0-Fold #5: CPU-1112.121 vs Wall-817.650
+Repeat #0-Fold #6: CPU-908.489 vs Wall-682.244
+Repeat #0-Fold #7: CPU-889.331 vs Wall-624.544
+Repeat #0-Fold #8: CPU-908.252 vs Wall-630.604
+Repeat #0-Fold #9: CPU-888.762 vs Wall-654.652
+```
+:::
+
+## Case 5: Running Scikit-learn models that don\'t release GIL
+
+Certain Scikit-learn models do not release the [Python
+GIL](https://docs.python.org/dev/glossary.html#term-global-interpreter-lock)
+and are also not executed in parallel via a BLAS library. In such cases,
+the CPU times and wallclock times are most likely trustworthy. Note
+however that only very few models such as naive Bayes models are of this
+kind.
+
+``` default
+clf = GaussianNB()
+
+with parallel_backend("multiprocessing", n_jobs=-1):
+ run9 = openml.runs.run_model_on_task(
+ model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False
+ )
+measures = run9.fold_evaluations
+print_compare_runtimes(measures)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+Repeat #0-Fold #0: CPU-26.803 vs Wall-26.800
+Repeat #0-Fold #1: CPU-26.351 vs Wall-26.349
+Repeat #0-Fold #2: CPU-26.371 vs Wall-26.370
+Repeat #0-Fold #3: CPU-26.402 vs Wall-26.400
+Repeat #0-Fold #4: CPU-26.782 vs Wall-26.781
+Repeat #0-Fold #5: CPU-26.479 vs Wall-26.479
+Repeat #0-Fold #6: CPU-26.393 vs Wall-26.390
+Repeat #0-Fold #7: CPU-26.430 vs Wall-26.427
+Repeat #0-Fold #8: CPU-26.654 vs Wall-26.657
+Repeat #0-Fold #9: CPU-26.409 vs Wall-26.406
+```
+:::
+
+## Summmary
+
+The scikit-learn extension for OpenML-Python records model runtimes for
+the CPU-clock and the wall-clock times. The above examples illustrated
+how these recorded runtimes can be extracted when using a scikit-learn
+model and under parallel setups too. To summarize, the scikit-learn
+extension measures the:
+
+- [CPU-time]{.title-ref} & [wallclock-time]{.title-ref} for the whole
+ run
+ - A run here corresponds to a call to
+ [run_model_on_task]{.title-ref} or
+ [run_flow_on_task]{.title-ref}
+ - The recorded time is for the model fit for each of the
+ outer-cross validations folds, i.e., the OpenML data splits
+- Python\'s [time]{.title-ref} module is used to compute the runtimes
+ - [CPU-time]{.title-ref} is recorded using the responses of
+ [time.process_time()]{.title-ref}
+ - [wallclock-time]{.title-ref} is recorded using the responses of
+ [time.time()]{.title-ref}
+- The timings recorded by OpenML per outer-cross validation fold is
+ agnostic to model parallelisation
+ - The wallclock times reported in Case 2 above highlights the
+ speed-up on using [n_jobs=-1]{.title-ref} in comparison to
+ [n_jobs=2]{.title-ref}, since the timing recorded by OpenML is
+ for the entire [fit()]{.title-ref} procedure, whereas the
+ parallelisation is performed inside [fit()]{.title-ref} by
+ scikit-learn
+ - The CPU-time for models that are run in parallel can be
+ difficult to interpret
+- [CPU-time]{.title-ref} & [wallclock-time]{.title-ref} for each
+ search per outer fold in an HPO run
+ - Reports the total time for performing search on each of the
+ OpenML data split, subsuming any sort of parallelism that
+ happened as part of the HPO estimator or the underlying base
+ estimator
+ - Also allows extraction of the [refit_time]{.title-ref} that
+ scikit-learn measures using [time.time()]{.title-ref} for
+ retraining the model per outer fold, for the best found
+ configuration
+- [CPU-time]{.title-ref} & [wallclock-time]{.title-ref} for models
+ that scikit-learn doesn\'t parallelize
+ - Models like Decision Trees or naive Bayes don\'t parallelize and
+ thus both the wallclock and CPU times are similar in runtime for
+ the OpenML call
+ - However, models implemented in Cython, such as the Decision
+ Trees can release the GIL and still run in parallel if a
+ [threading]{.title-ref} backend is used by joblib.
+ - Scikit-learn Neural Networks can undergo parallelization
+ implicitly owing to thread-level parallelism involved in the
+ linear algebraic operations and thus the wallclock-time and
+ CPU-time can differ.
+
+Because of all the cases mentioned above it is crucial to understand
+which case is triggered when reporting runtimes for scikit-learn models
+measured with OpenML-Python!
+
+::: rst-class
+sphx-glr-timing
+
+**Total running time of the script:** ( 1 minutes 13.261 seconds)
+:::
+
+::::::: {#sphx_glr_download_examples_30_extended_fetch_runtimes_tutorial.py}
+:::::: only
+html
+
+::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
+::: {.container .sphx-glr-download .sphx-glr-download-python}
+`Download Python source code: fetch_runtimes_tutorial.py <fetch_runtimes_tutorial.py>`{.interpreted-text
+role="download"}
+:::
+
+::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
+`Download Jupyter notebook: fetch_runtimes_tutorial.ipynb <fetch_runtimes_tutorial.ipynb>`{.interpreted-text
+role="download"}
+:::
+:::::
+::::::
+:::::::
+
+:::: only
+html
+
+::: rst-class
+sphx-glr-signature
+
+[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
+:::
+::::
diff --git a/docs/examples/30_extended/flow_id_tutorial.rst.md b/docs/examples/30_extended/flow_id_tutorial.rst.md
new file mode 100644
index 000000000..62e3049e8
--- /dev/null
+++ b/docs/examples/30_extended/flow_id_tutorial.rst.md
@@ -0,0 +1,208 @@
+::::: only
+html
+
+:::: {.note .sphx-glr-download-link-note}
+::: title
+Note
+:::
+
+`Go to the end <sphx_glr_download_examples_30_extended_flow_id_tutorial.py>`{.interpreted-text
+role="ref"} to download the full example code
+::::
+:::::
+
+::: rst-class
+sphx-glr-example-title
+:::
+
+# Obtaining Flow IDs {#sphx_glr_examples_30_extended_flow_id_tutorial.py}
+
+This tutorial discusses different ways to obtain the ID of a flow in
+order to perform further analysis.
+
+``` default
+# License: BSD 3-Clause
+
+import sklearn.tree
+
+import openml
+```
+
+:::: warning
+::: title
+Warning
+:::
+
+.. include:: ../../test_server_usage_warning.txt
+::::
+
+``` default
+openml.config.start_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!
+ warnings.warn(
+```
+:::
+
+Defining a classifier
+
+``` default
+clf = sklearn.tree.DecisionTreeClassifier()
+```
+
+## 1. Obtaining a flow given a classifier
+
+``` default
+flow = openml.extensions.get_extension_by_model(clf).model_to_flow(clf).publish()
+flow_id = flow.flow_id
+print(flow_id)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+4595
+```
+:::
+
+This piece of code is rather involved. First, it retrieves a
+`~openml.extensions.Extension`{.interpreted-text role="class"} which is
+registered and can handle the given model, in our case it is
+`openml.extensions.sklearn.SklearnExtension`{.interpreted-text
+role="class"}. Second, the extension converts the classifier into an
+instance of `openml.OpenMLFlow`{.interpreted-text role="class"}. Third
+and finally, the publish method checks whether the current flow is
+already present on OpenML. If not, it uploads the flow, otherwise, it
+updates the current instance with all information computed by the server
+(which is obviously also done when uploading/publishing a flow).
+
+To simplify the usage we have created a helper function which automates
+all these steps:
+
+``` default
+flow_id = openml.flows.get_flow_id(model=clf)
+print(flow_id)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+4595
+```
+:::
+
+## 2. Obtaining a flow given its name
+
+The schema of a flow is given in XSD
+([here](https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/openml.implementation.upload.xsd)).
+\# noqa E501 Only two fields are required, a unique name, and an
+external version. While it should be pretty obvious why we need a name,
+the need for the additional external version information might not be
+immediately clear. However, this information is very important as it
+allows to have multiple flows with the same name for different versions
+of a software. This might be necessary if an algorithm or implementation
+introduces, renames or drop hyperparameters over time.
+
+``` default
+print(flow.name, flow.external_version)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+sklearn.tree._classes.DecisionTreeClassifier openml==0.14.1,sklearn==1.3.0
+```
+:::
+
+The name and external version are automatically added to a flow when
+constructing it from a model. We can then use them to retrieve the flow
+id as follows:
+
+``` default
+flow_id = openml.flows.flow_exists(name=flow.name, external_version=flow.external_version)
+print(flow_id)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+4595
+```
+:::
+
+We can also retrieve all flows for a given name:
+
+``` default
+flow_ids = openml.flows.get_flow_id(name=flow.name)
+print(flow_ids)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+[22, 26, 30, 267, 1178, 4102, 4112, 4127, 4595, 6211]
+```
+:::
+
+This also works with the actual model (generalizing the first part of
+this example):
+
+``` default
+flow_ids = openml.flows.get_flow_id(model=clf, exact_version=False)
+print(flow_ids)
+
+# Deactivating test server
+openml.config.stop_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+[22, 26, 30, 267, 1178, 4102, 4112, 4127, 4595, 6211]
+```
+:::
+
+::: rst-class
+sphx-glr-timing
+
+**Total running time of the script:** ( 0 minutes 4.274 seconds)
+:::
+
+::::::: {#sphx_glr_download_examples_30_extended_flow_id_tutorial.py}
+:::::: only
+html
+
+::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
+::: {.container .sphx-glr-download .sphx-glr-download-python}
+`Download Python source code: flow_id_tutorial.py <flow_id_tutorial.py>`{.interpreted-text
+role="download"}
+:::
+
+::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
+`Download Jupyter notebook: flow_id_tutorial.ipynb <flow_id_tutorial.ipynb>`{.interpreted-text
+role="download"}
+:::
+:::::
+::::::
+:::::::
+
+:::: only
+html
+
+::: rst-class
+sphx-glr-signature
+
+[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
+:::
+::::
diff --git a/docs/examples/30_extended/flows_and_runs_tutorial.rst.md b/docs/examples/30_extended/flows_and_runs_tutorial.rst.md
new file mode 100644
index 000000000..7b5066735
--- /dev/null
+++ b/docs/examples/30_extended/flows_and_runs_tutorial.rst.md
@@ -0,0 +1,604 @@
+::::: only
+html
+
+:::: {.note .sphx-glr-download-link-note}
+::: title
+Note
+:::
+
+`Go to the end <sphx_glr_download_examples_30_extended_flows_and_runs_tutorial.py>`{.interpreted-text
+role="ref"} to download the full example code
+::::
+:::::
+
+::: rst-class
+sphx-glr-example-title
+:::
+
+# Flows and Runs {#sphx_glr_examples_30_extended_flows_and_runs_tutorial.py}
+
+How to train/run a model and how to upload the results.
+
+``` default
+# License: BSD 3-Clause
+
+import openml
+from sklearn import compose, ensemble, impute, neighbors, preprocessing, pipeline, tree
+```
+
+We\'ll use the test server for the rest of this tutorial.
+
+:::: warning
+::: title
+Warning
+:::
+
+.. include:: ../../test_server_usage_warning.txt
+::::
+
+``` default
+openml.config.start_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!
+ warnings.warn(
+```
+:::
+
+## Train machine learning models
+
+Train a scikit-learn model on the data manually.
+
+``` default
+# NOTE: We are using dataset 68 from the test server: https://test.openml.org/d/68
+dataset = openml.datasets.get_dataset(68)
+X, y, categorical_indicator, attribute_names = dataset.get_data(
+ target=dataset.default_target_attribute
+)
+clf = neighbors.KNeighborsClassifier(n_neighbors=1)
+clf.fit(X, y)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+```
+:::
+
+<div class="output_subarea output_html rendered_html output_result">
+<style>#sk-container-id-2 {color: black;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: "▸";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: "▾";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: "";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: "";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: "";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id="sk-container-id-2" class="sk-top-container"><div class="sk-text-repr-fallback"><pre>KNeighborsClassifier(n_neighbors=1)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class="sk-container" hidden><div class="sk-item"><div class="sk-estimator sk-toggleable"><input class="sk-toggleable__control sk-hidden--visually" id="sk-estimator-id-2" type="checkbox" checked><label for="sk-estimator-id-2" class="sk-toggleable__label sk-toggleable__label-arrow">KNeighborsClassifier</label><div class="sk-toggleable__content"><pre>KNeighborsClassifier(n_neighbors=1)</pre></div></div></div></div></div>
+</div>
+<br />
+<br />
+
+You can also ask for meta-data to automatically preprocess the data.
+
+- e.g. categorical features -\> do feature encoding
+
+``` default
+dataset = openml.datasets.get_dataset(17)
+X, y, categorical_indicator, attribute_names = dataset.get_data(
+ target=dataset.default_target_attribute
+)
+print(f"Categorical features: {categorical_indicator}")
+transformer = compose.ColumnTransformer(
+ [("one_hot_encoder", preprocessing.OneHotEncoder(categories="auto"), categorical_indicator)]
+)
+X = transformer.fit_transform(X)
+clf.fit(X, y)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+Categorical features: [True, False, True, True, False, True, True, False, True, True, False, True, False, True, True, False, True, False, True, True]
+```
+:::
+
+<div class="output_subarea output_html rendered_html output_result">
+<style>#sk-container-id-3 {color: black;}#sk-container-id-3 pre{padding: 0;}#sk-container-id-3 div.sk-toggleable {background-color: white;}#sk-container-id-3 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-3 label.sk-toggleable__label-arrow:before {content: "▸";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-3 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-3 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-3 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: "▾";}#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-3 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-3 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-3 div.sk-parallel-item::after {content: "";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-serial::before {content: "";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-3 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-3 div.sk-item {position: relative;z-index: 1;}#sk-container-id-3 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-3 div.sk-item::before, #sk-container-id-3 div.sk-parallel-item::before {content: "";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-3 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-3 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-3 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-3 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-3 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-3 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-3 div.sk-label-container {text-align: center;}#sk-container-id-3 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-3 div.sk-text-repr-fallback {display: none;}</style><div id="sk-container-id-3" class="sk-top-container"><div class="sk-text-repr-fallback"><pre>KNeighborsClassifier(n_neighbors=1)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class="sk-container" hidden><div class="sk-item"><div class="sk-estimator sk-toggleable"><input class="sk-toggleable__control sk-hidden--visually" id="sk-estimator-id-3" type="checkbox" checked><label for="sk-estimator-id-3" class="sk-toggleable__label sk-toggleable__label-arrow">KNeighborsClassifier</label><div class="sk-toggleable__content"><pre>KNeighborsClassifier(n_neighbors=1)</pre></div></div></div></div></div>
+</div>
+<br />
+<br />
+
+## Runs: Easily explore models
+
+We can run (many) scikit-learn algorithms on (many) OpenML tasks.
+
+``` default
+# Get a task
+task = openml.tasks.get_task(403)
+
+# Build any classifier or pipeline
+clf = tree.DecisionTreeClassifier()
+
+# Run the flow
+run = openml.runs.run_model_on_task(clf, task)
+
+print(run)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+OpenML Run
+==========
+Uploader Name...................: None
+Metric..........................: None
+Local Result - Accuracy (+- STD): 0.8469 +- 0.0120
+Local Runtime - ms (+- STD).....: 143.4586 +- 6.3831
+Run ID..........................: None
+Task ID.........................: 403
+Task Type.......................: None
+Task URL........................: https://test.openml.org/t/403
+Flow ID.........................: 4595
+Flow Name.......................: sklearn.tree._classes.DecisionTreeClassifier
+Flow URL........................: https://test.openml.org/f/4595
+Setup ID........................: None
+Setup String....................: Python_3.10.12. Sklearn_1.3.0. NumPy_1.25.1. SciPy_1.11.1.
+Dataset ID......................: 68
+Dataset URL.....................: https://test.openml.org/d/68
+```
+:::
+
+Share the run on the OpenML server
+
+So far the run is only available locally. By calling the publish
+function, the run is sent to the OpenML server:
+
+``` default
+myrun = run.publish()
+# For this tutorial, our configuration publishes to the test server
+# as to not pollute the main server.
+print(f"Uploaded to {myrun.openml_url}")
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+Uploaded to https://test.openml.org/r/7358
+```
+:::
+
+We can now also inspect the flow object which was automatically created:
+
+``` default
+flow = openml.flows.get_flow(run.flow_id)
+print(flow)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+OpenML Flow
+===========
+Flow ID.........: 4595 (version 9)
+Flow URL........: https://test.openml.org/f/4595
+Flow Name.......: sklearn.tree._classes.DecisionTreeClassifier
+Flow Description: A decision tree classifier.
+Upload Date.....: 2023-07-20 19:37:57
+Dependencies....: sklearn==1.3.0
+numpy>=1.17.3
+scipy>=1.5.0
+joblib>=1.1.1
+threadpoolctl>=2.0.0
+```
+:::
+
+### It also works with pipelines
+
+When you need to handle \'dirty\' data, build pipelines to model then
+automatically. To demonstrate this using the dataset
+[credit-a](https://test.openml.org/d/16) via
+[task](https://test.openml.org/t/96) as it contains both numerical and
+categorical variables and missing values in both.
+
+``` default
+task = openml.tasks.get_task(96)
+
+# OpenML helper functions for sklearn can be plugged in directly for complicated pipelines
+from openml.extensions.sklearn import cat, cont
+
+pipe = pipeline.Pipeline(
+ steps=[
+ (
+ "Preprocessing",
+ compose.ColumnTransformer(
+ [
+ (
+ "categorical",
+ preprocessing.OneHotEncoder(sparse=False, handle_unknown="ignore"),
+ cat, # returns the categorical feature indices
+ ),
+ (
+ "continuous",
+ impute.SimpleImputer(strategy="median"),
+ cont,
+ ), # returns the numeric feature indices
+ ]
+ ),
+ ),
+ ("Classifier", ensemble.RandomForestClassifier(n_estimators=10)),
+ ]
+)
+
+run = openml.runs.run_model_on_task(pipe, task, avoid_duplicate_runs=False)
+myrun = run.publish()
+print(f"Uploaded to {myrun.openml_url}")
+
+
+# The above pipeline works with the helper functions that internally deal with pandas DataFrame.
+# In the case, pandas is not available, or a NumPy based data processing is the requirement, the
+# above pipeline is presented below to work with NumPy.
+
+# Extracting the indices of the categorical columns
+features = task.get_dataset().features
+categorical_feature_indices = []
+numeric_feature_indices = []
+for i in range(len(features)):
+ if features[i].name == task.target_name:
+ continue
+ if features[i].data_type == "nominal":
+ categorical_feature_indices.append(i)
+ else:
+ numeric_feature_indices.append(i)
+
+pipe = pipeline.Pipeline(
+ steps=[
+ (
+ "Preprocessing",
+ compose.ColumnTransformer(
+ [
+ (
+ "categorical",
+ preprocessing.OneHotEncoder(sparse=False, handle_unknown="ignore"),
+ categorical_feature_indices,
+ ),
+ (
+ "continuous",
+ impute.SimpleImputer(strategy="median"),
+ numeric_feature_indices,
+ ),
+ ]
+ ),
+ ),
+ ("Classifier", ensemble.RandomForestClassifier(n_estimators=10)),
+ ]
+)
+
+run = openml.runs.run_model_on_task(pipe, task, avoid_duplicate_runs=False)
+myrun = run.publish()
+print(f"Uploaded to {myrun.openml_url}")
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/openml/venv/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:972: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.
+ warnings.warn(
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+Uploaded to https://test.openml.org/r/7359
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/openml/venv/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:972: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.
+ warnings.warn(
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+Uploaded to https://test.openml.org/r/7360
+```
+:::
+
+## Running flows on tasks offline for later upload
+
+For those scenarios where there is no access to internet, it is possible
+to run a model on a task without uploading results or flows to the
+server immediately.
+
+``` default
+# To perform the following line offline, it is required to have been called before
+# such that the task is cached on the local openml cache directory:
+task = openml.tasks.get_task(96)
+
+# The following lines can then be executed offline:
+run = openml.runs.run_model_on_task(
+ pipe,
+ task,
+ avoid_duplicate_runs=False,
+ upload_flow=False,
+)
+
+# The run may be stored offline, and the flow will be stored along with it:
+run.to_filesystem(directory="myrun")
+
+# They may be loaded and uploaded at a later time
+run = openml.runs.OpenMLRun.from_filesystem(directory="myrun")
+run.publish()
+
+# Publishing the run will automatically upload the related flow if
+# it does not yet exist on the server.
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/openml/venv/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:972: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.
+ warnings.warn(
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+
+OpenML Run
+==========
+Uploader Name...................: None
+Metric..........................: None
+Local Result - Accuracy (+- STD): 0.8678 +- 0.0000
+Local Runtime - ms (+- STD).....: 27.5688 +- 0.0000
+Run ID..........................: 7361
+Run URL.........................: https://test.openml.org/r/7361
+Task ID.........................: 96
+Task Type.......................: None
+Task URL........................: https://test.openml.org/t/96
+Flow ID.........................: 5042
+Flow Name.......................: sklearn.pipeline.Pipeline(Preprocessing=sklearn.compose._column_transformer.ColumnTransformer(categorical=sklearn.preprocessing._encoders.OneHotEncoder,continuous=sklearn.impute._base.SimpleImputer),Classifier=sklearn.ensemble._forest.RandomForestClassifier)
+Flow URL........................: https://test.openml.org/f/5042
+Setup ID........................: None
+Setup String....................: Python_3.10.12. Sklearn_1.3.0. NumPy_1.25.1. SciPy_1.11.1.
+Dataset ID......................: None
+Dataset URL.....................: https://test.openml.org/d/None
+```
+:::
+
+Alternatively, one can also directly run flows.
+
+``` default
+# Get a task
+task = openml.tasks.get_task(403)
+
+# Build any classifier or pipeline
+clf = tree.ExtraTreeClassifier()
+
+# Obtain the scikit-learn extension interface to convert the classifier
+# into a flow object.
+extension = openml.extensions.get_extension_by_model(clf)
+flow = extension.model_to_flow(clf)
+
+run = openml.runs.run_flow_on_task(flow, task)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+```
+:::
+
+## Challenge
+
+Try to build the best possible models on several OpenML tasks, compare
+your results with the rest of the class and learn from them. Some tasks
+you could try (or browse openml.org):
+
+- EEG eye state: data_id:[1471](https://www.openml.org/d/1471),
+ task_id:[14951](https://www.openml.org/t/14951)
+- Volcanoes on Venus: data_id:[1527](https://www.openml.org/d/1527),
+ task_id:[10103](https://www.openml.org/t/10103)
+- Walking activity: data_id:[1509](https://www.openml.org/d/1509),
+ task_id:[9945](https://www.openml.org/t/9945), 150k instances.
+- Covertype (Satellite): data_id:[150](https://www.openml.org/d/150),
+ task_id:[218](https://www.openml.org/t/218), 500k instances.
+- Higgs (Physics): data_id:[23512](https://www.openml.org/d/23512),
+ task_id:[52950](https://www.openml.org/t/52950), 100k instances,
+ missing values.
+
+``` default
+# Easy benchmarking:
+for task_id in [115]: # Add further tasks. Disclaimer: they might take some time
+ task = openml.tasks.get_task(task_id)
+ data = openml.datasets.get_dataset(task.dataset_id)
+ clf = neighbors.KNeighborsClassifier(n_neighbors=5)
+
+ run = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False)
+ myrun = run.publish()
+ print(f"kNN on {data.name}: {myrun.openml_url}")
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+kNN on diabetes: https://test.openml.org/r/7362
+```
+:::
+
+``` default
+openml.config.stop_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-timing
+
+**Total running time of the script:** ( 0 minutes 12.993 seconds)
+:::
+
+::::::: {#sphx_glr_download_examples_30_extended_flows_and_runs_tutorial.py}
+:::::: only
+html
+
+::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
+::: {.container .sphx-glr-download .sphx-glr-download-python}
+`Download Python source code: flows_and_runs_tutorial.py <flows_and_runs_tutorial.py>`{.interpreted-text
+role="download"}
+:::
+
+::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
+`Download Jupyter notebook: flows_and_runs_tutorial.ipynb <flows_and_runs_tutorial.ipynb>`{.interpreted-text
+role="download"}
+:::
+:::::
+::::::
+:::::::
+
+:::: only
+html
+
+::: rst-class
+sphx-glr-signature
+
+[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
+:::
+::::
diff --git a/docs/examples/30_extended/index.rst.md b/docs/examples/30_extended/index.rst.md
new file mode 100644
index 000000000..7311ca69e
--- /dev/null
+++ b/docs/examples/30_extended/index.rst.md
@@ -0,0 +1,205 @@
+# In-Depth Examples {#sphx_glr_examples_30_extended}
+
+Extended examples for the usage of the OpenML python connector.
+
+<div class="sphx-glr-thumbnails">
+<div class="sphx-glr-thumbcontainer" tooltip="This tutorial discusses different ways to obtain the ID of a flow in order to perform further a...">
+
+::: only
+html
+
+
+
+`sphx_glr_examples_30_extended_flow_id_tutorial.py`{.interpreted-text
+role="ref"}
+:::
+
+<div class="sphx-glr-thumbnail-title">Obtaining Flow IDs</div>
+</div>
+<div class="sphx-glr-thumbcontainer" tooltip="How to list, download and upload benchmark suites.">
+
+::: only
+html
+
+
+
+`sphx_glr_examples_30_extended_suites_tutorial.py`{.interpreted-text
+role="ref"}
+:::
+
+<div class="sphx-glr-thumbnail-title">Benchmark suites</div>
+</div>
+<div class="sphx-glr-thumbcontainer" tooltip="Explains openml-python logging, and shows how to configure it.">
+
+::: only
+html
+
+
+
+`sphx_glr_examples_30_extended_configure_logging.py`{.interpreted-text
+role="ref"}
+:::
+
+<div class="sphx-glr-thumbnail-title">Logging</div>
+</div>
+<div class="sphx-glr-thumbcontainer" tooltip="Benchmark studies">
+
+::: only
+html
+
+
+
+`sphx_glr_examples_30_extended_study_tutorial.py`{.interpreted-text
+role="ref"}
+:::
+
+<div class="sphx-glr-thumbnail-title">Benchmark studies</div>
+</div>
+<div class="sphx-glr-thumbcontainer" tooltip="Plotting hyperparameter surfaces">
+
+::: only
+html
+
+
+
+`sphx_glr_examples_30_extended_plot_svm_hyperparameters_tutorial.py`{.interpreted-text
+role="ref"}
+:::
+
+<div class="sphx-glr-thumbnail-title">Plotting hyperparameter surfaces</div>
+</div>
+<div class="sphx-glr-thumbcontainer" tooltip="A tutorial on how to list and download tasks.">
+
+::: only
+html
+
+
+
+`sphx_glr_examples_30_extended_tasks_tutorial.py`{.interpreted-text
+role="ref"}
+:::
+
+<div class="sphx-glr-thumbnail-title">Tasks</div>
+</div>
+<div class="sphx-glr-thumbcontainer" tooltip="How to list and download datasets.">
+
+::: only
+html
+
+
+
+`sphx_glr_examples_30_extended_datasets_tutorial.py`{.interpreted-text
+role="ref"}
+:::
+
+<div class="sphx-glr-thumbnail-title">Datasets</div>
+</div>
+<div class="sphx-glr-thumbcontainer" tooltip="By: Jan N. van Rijn">
+
+::: only
+html
+
+
+
+`sphx_glr_examples_30_extended_run_setup_tutorial.py`{.interpreted-text
+role="ref"}
+:::
+
+<div class="sphx-glr-thumbnail-title">Run Setup</div>
+</div>
+<div class="sphx-glr-thumbcontainer" tooltip="The most convenient way to create a flow for your machine learning workflow is to generate it a...">
+
+::: only
+html
+
+
+
+`sphx_glr_examples_30_extended_custom_flow_.py`{.interpreted-text
+role="ref"}
+:::
+
+<div class="sphx-glr-thumbnail-title">Creating and Using a Custom Flow</div>
+</div>
+<div class="sphx-glr-thumbcontainer" tooltip="Evaluations contain a concise summary of the results of all runs made. Each evaluation provides...">
+
+::: only
+html
+
+
+
+`sphx_glr_examples_30_extended_fetch_evaluations_tutorial.py`{.interpreted-text
+role="ref"}
+:::
+
+<div class="sphx-glr-thumbnail-title">Fetching Evaluations</div>
+</div>
+<div class="sphx-glr-thumbcontainer" tooltip="Tasks define a target and a train/test split. Normally, they are the input to the function open...">
+
+::: only
+html
+
+
+
+`sphx_glr_examples_30_extended_task_manual_iteration_tutorial.py`{.interpreted-text
+role="ref"}
+:::
+
+<div class="sphx-glr-thumbnail-title">Tasks: retrieving splits</div>
+</div>
+<div class="sphx-glr-thumbcontainer" tooltip="How to train/run a model and how to upload the results.">
+
+::: only
+html
+
+
+
+`sphx_glr_examples_30_extended_flows_and_runs_tutorial.py`{.interpreted-text
+role="ref"}
+:::
+
+<div class="sphx-glr-thumbnail-title">Flows and Runs</div>
+</div>
+<div class="sphx-glr-thumbcontainer" tooltip="A tutorial on how to create and upload a dataset to OpenML.">
+
+::: only
+html
+
+
+
+`sphx_glr_examples_30_extended_create_upload_tutorial.py`{.interpreted-text
+role="ref"}
+:::
+
+<div class="sphx-glr-thumbnail-title">Dataset upload tutorial</div>
+</div>
+<div class="sphx-glr-thumbcontainer" tooltip="The runtime of machine learning models on specific datasets can be a deciding factor on the cho...">
+
+::: only
+html
+
+
+
+`sphx_glr_examples_30_extended_fetch_runtimes_tutorial.py`{.interpreted-text
+role="ref"}
+:::
+
+<div class="sphx-glr-thumbnail-title">Measuring runtimes for Scikit-learn models</div>
+</div>
+</div>
+
+::: {.toctree hidden=""}
+/examples/30_extended/flow_id_tutorial
+/examples/30_extended/suites_tutorial
+/examples/30_extended/configure_logging
+/examples/30_extended/study_tutorial
+/examples/30_extended/plot_svm_hyperparameters_tutorial
+/examples/30_extended/tasks_tutorial
+/examples/30_extended/datasets_tutorial
+/examples/30_extended/run_setup_tutorial
+/examples/30_extended/[custom_flow]()
+/examples/30_extended/fetch_evaluations_tutorial
+/examples/30_extended/task_manual_iteration_tutorial
+/examples/30_extended/flows_and_runs_tutorial
+/examples/30_extended/create_upload_tutorial
+/examples/30_extended/fetch_runtimes_tutorial
+:::
diff --git a/docs/examples/30_extended/plot_svm_hyperparameters_tutorial.rst.md b/docs/examples/30_extended/plot_svm_hyperparameters_tutorial.rst.md
new file mode 100644
index 000000000..f272bbe33
--- /dev/null
+++ b/docs/examples/30_extended/plot_svm_hyperparameters_tutorial.rst.md
@@ -0,0 +1,235 @@
+::::: only
+html
+
+:::: {.note .sphx-glr-download-link-note}
+::: title
+Note
+:::
+
+`Go to the end <sphx_glr_download_examples_30_extended_plot_svm_hyperparameters_tutorial.py>`{.interpreted-text
+role="ref"} to download the full example code
+::::
+:::::
+
+::: rst-class
+sphx-glr-example-title
+:::
+
+# Plotting hyperparameter surfaces {#sphx_glr_examples_30_extended_plot_svm_hyperparameters_tutorial.py}
+
+``` default
+# License: BSD 3-Clause
+
+import openml
+import numpy as np
+```
+
+## First step - obtaining the data
+
+First, we need to choose an SVM flow, for example 8353, and a task.
+Finding the IDs of them are not part of this tutorial, this could for
+example be done via the website.
+
+For this we use the function `list_evaluations_setup` which can
+automatically join evaluations conducted by the server with the
+hyperparameter settings extracted from the uploaded runs (called
+*setup*).
+
+``` default
+df = openml.evaluations.list_evaluations_setups(
+ function="predictive_accuracy",
+ flows=[8353],
+ tasks=[6],
+ output_format="dataframe",
+ # Using this flag incorporates the hyperparameters into the returned dataframe. Otherwise,
+ # the dataframe would contain a field ``paramaters`` containing an unparsed dictionary.
+ parameters_in_separate_columns=True,
+)
+print(df.head(n=10))
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+run_id ... sklearn.pipeline.Pipeline(imputation=hyperimp.utils.preprocessing.ConditionalImputer2,hotencoding=sklearn.preprocessing.data.OneHotEncoder,scaling=sklearn.preprocessing.data.StandardScaler,variencethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,clf=sklearn.svm.classes.SVC)(1)_memory
+0 9154533 ... null
+1 9154536 ... null
+2 9154539 ... null
+3 9154544 ... null
+4 9154546 ... null
+5 9154558 ... null
+6 9154563 ... null
+7 9154569 ... null
+8 9154572 ... null
+9 9154576 ... null
+
+[10 rows x 46 columns]
+```
+:::
+
+We can see all the hyperparameter names in the columns of the dataframe:
+
+``` default
+for name in df.columns:
+ print(name)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+run_id
+task_id
+setup_id
+flow_id
+flow_name
+data_id
+data_name
+function
+upload_time
+uploader
+uploader_name
+value
+values
+array_data
+sklearn.preprocessing.data.OneHotEncoder(17)_categorical_features
+sklearn.preprocessing.data.OneHotEncoder(17)_dtype
+sklearn.preprocessing.data.OneHotEncoder(17)_handle_unknown
+sklearn.preprocessing.data.OneHotEncoder(17)_n_values
+sklearn.preprocessing.data.OneHotEncoder(17)_sparse
+sklearn.feature_selection.variance_threshold.VarianceThreshold(11)_threshold
+sklearn.preprocessing.data.StandardScaler(5)_copy
+sklearn.preprocessing.data.StandardScaler(5)_with_mean
+sklearn.preprocessing.data.StandardScaler(5)_with_std
+sklearn.svm.classes.SVC(16)_C
+sklearn.svm.classes.SVC(16)_cache_size
+sklearn.svm.classes.SVC(16)_class_weight
+sklearn.svm.classes.SVC(16)_coef0
+sklearn.svm.classes.SVC(16)_decision_function_shape
+sklearn.svm.classes.SVC(16)_degree
+sklearn.svm.classes.SVC(16)_gamma
+sklearn.svm.classes.SVC(16)_kernel
+sklearn.svm.classes.SVC(16)_max_iter
+sklearn.svm.classes.SVC(16)_probability
+sklearn.svm.classes.SVC(16)_random_state
+sklearn.svm.classes.SVC(16)_shrinking
+sklearn.svm.classes.SVC(16)_tol
+sklearn.svm.classes.SVC(16)_verbose
+hyperimp.utils.preprocessing.ConditionalImputer2(1)_axis
+hyperimp.utils.preprocessing.ConditionalImputer2(1)_categorical_features
+hyperimp.utils.preprocessing.ConditionalImputer2(1)_copy
+hyperimp.utils.preprocessing.ConditionalImputer2(1)_fill_empty
+hyperimp.utils.preprocessing.ConditionalImputer2(1)_missing_values
+hyperimp.utils.preprocessing.ConditionalImputer2(1)_strategy
+hyperimp.utils.preprocessing.ConditionalImputer2(1)_strategy_nominal
+hyperimp.utils.preprocessing.ConditionalImputer2(1)_verbose
+sklearn.pipeline.Pipeline(imputation=hyperimp.utils.preprocessing.ConditionalImputer2,hotencoding=sklearn.preprocessing.data.OneHotEncoder,scaling=sklearn.preprocessing.data.StandardScaler,variencethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,clf=sklearn.svm.classes.SVC)(1)_memory
+```
+:::
+
+Next, we cast and transform the hyperparameters of interest (`C` and
+`gamma`) so that we can nicely plot them.
+
+``` default
+hyperparameters = ["sklearn.svm.classes.SVC(16)_C", "sklearn.svm.classes.SVC(16)_gamma"]
+df[hyperparameters] = df[hyperparameters].astype(float).apply(np.log10)
+```
+
+## Option 1 - plotting via the pandas helper functions
+
+``` default
+df.plot.hexbin(
+ x="sklearn.svm.classes.SVC(16)_C",
+ y="sklearn.svm.classes.SVC(16)_gamma",
+ C="value",
+ reduce_C_function=np.mean,
+ gridsize=25,
+ title="SVM performance landscape",
+)
+```
+
+::: {.image-sg .sphx-glr-single-img alt="SVM performance landscape" srcset="/examples/30_extended/images/sphx_glr_plot_svm_hyperparameters_tutorial_001.png"}
+/examples/30_extended/images/sphx_glr_plot_svm_hyperparameters_tutorial_001.png
+:::
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+<Axes: title={'center': 'SVM performance landscape'}, xlabel='sklearn.svm.classes.SVC(16)_C', ylabel='sklearn.svm.classes.SVC(16)_gamma'>
+```
+:::
+
+## Option 2 - plotting via matplotlib
+
+``` default
+import matplotlib.pyplot as plt
+
+fig, ax = plt.subplots()
+
+C = df["sklearn.svm.classes.SVC(16)_C"]
+gamma = df["sklearn.svm.classes.SVC(16)_gamma"]
+score = df["value"]
+
+# Plotting all evaluations:
+ax.plot(C, gamma, "ko", ms=1)
+# Create a contour plot
+cntr = ax.tricontourf(C, gamma, score, levels=12, cmap="RdBu_r")
+# Adjusting the colorbar
+fig.colorbar(cntr, ax=ax, label="accuracy")
+# Adjusting the axis limits
+ax.set(
+ xlim=(min(C), max(C)),
+ ylim=(min(gamma), max(gamma)),
+ xlabel="C (log10)",
+ ylabel="gamma (log10)",
+)
+ax.set_title("SVM performance landscape")
+```
+
+::: {.image-sg .sphx-glr-single-img alt="SVM performance landscape" srcset="/examples/30_extended/images/sphx_glr_plot_svm_hyperparameters_tutorial_002.png"}
+/examples/30_extended/images/sphx_glr_plot_svm_hyperparameters_tutorial_002.png
+:::
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+Text(0.5, 1.0, 'SVM performance landscape')
+```
+:::
+
+::: rst-class
+sphx-glr-timing
+
+**Total running time of the script:** ( 0 minutes 6.466 seconds)
+:::
+
+::::::: {#sphx_glr_download_examples_30_extended_plot_svm_hyperparameters_tutorial.py}
+:::::: only
+html
+
+::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
+::: {.container .sphx-glr-download .sphx-glr-download-python}
+`Download Python source code: plot_svm_hyperparameters_tutorial.py <plot_svm_hyperparameters_tutorial.py>`{.interpreted-text
+role="download"}
+:::
+
+::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
+`Download Jupyter notebook: plot_svm_hyperparameters_tutorial.ipynb <plot_svm_hyperparameters_tutorial.ipynb>`{.interpreted-text
+role="download"}
+:::
+:::::
+::::::
+:::::::
+
+:::: only
+html
+
+::: rst-class
+sphx-glr-signature
+
+[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
+:::
+::::
diff --git a/docs/examples/30_extended/run_setup_tutorial.rst.md b/docs/examples/30_extended/run_setup_tutorial.rst.md
new file mode 100644
index 000000000..656ce560f
--- /dev/null
+++ b/docs/examples/30_extended/run_setup_tutorial.rst.md
@@ -0,0 +1,215 @@
+::::: only
+html
+
+:::: {.note .sphx-glr-download-link-note}
+::: title
+Note
+:::
+
+`Go to the end <sphx_glr_download_examples_30_extended_run_setup_tutorial.py>`{.interpreted-text
+role="ref"} to download the full example code
+::::
+:::::
+
+::: rst-class
+sphx-glr-example-title
+:::
+
+# Run Setup {#sphx_glr_examples_30_extended_run_setup_tutorial.py}
+
+By: Jan N. van Rijn
+
+One of the key features of the openml-python library is that is allows
+to reinstantiate flows with hyperparameter settings that were uploaded
+before. This tutorial uses the concept of setups. Although setups are
+not extensively described in the OpenML documentation (because most
+users will not directly use them), they form a important concept within
+OpenML distinguishing between hyperparameter configurations. A setup is
+the combination of a flow with all its hyperparameters set.
+
+A key requirement for reinstantiating a flow is to have the same
+scikit-learn version as the flow that was uploaded. However, this
+tutorial will upload the flow (that will later be reinstantiated)
+itself, so it can be ran with any scikit-learn version that is supported
+by this library. In this case, the requirement of the corresponding
+scikit-learn versions is automatically met.
+
+In this tutorial we will
+
+: 1) Create a flow and use it to solve a task;
+ 2) Download the flow, reinstantiate the model with same
+ hyperparameters, and solve the same task again;
+ 3) We will verify that the obtained results are exactly the same.
+
+``` default
+# License: BSD 3-Clause
+
+import numpy as np
+import openml
+from openml.extensions.sklearn import cat, cont
+
+from sklearn.pipeline import make_pipeline, Pipeline
+from sklearn.compose import ColumnTransformer
+from sklearn.impute import SimpleImputer
+from sklearn.preprocessing import OneHotEncoder, FunctionTransformer
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.decomposition import TruncatedSVD
+```
+
+:::: warning
+::: title
+Warning
+:::
+
+.. include:: ../../test_server_usage_warning.txt
+::::
+
+``` default
+openml.config.start_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!
+ warnings.warn(
+```
+:::
+
+## 1) Create a flow and use it to solve a task
+
+``` default
+# first, let's download the task that we are interested in
+task = openml.tasks.get_task(6)
+
+
+# we will create a fairly complex model, with many preprocessing components and
+# many potential hyperparameters. Of course, the model can be as complex and as
+# easy as you want it to be
+
+
+cat_imp = make_pipeline(
+ OneHotEncoder(handle_unknown="ignore", sparse=False),
+ TruncatedSVD(),
+)
+cont_imp = SimpleImputer(strategy="median")
+ct = ColumnTransformer([("cat", cat_imp, cat), ("cont", cont_imp, cont)])
+model_original = Pipeline(
+ steps=[
+ ("transform", ct),
+ ("estimator", RandomForestClassifier()),
+ ]
+)
+
+# Let's change some hyperparameters. Of course, in any good application we
+# would tune them using, e.g., Random Search or Bayesian Optimization, but for
+# the purpose of this tutorial we set them to some specific values that might
+# or might not be optimal
+hyperparameters_original = {
+ "estimator__criterion": "gini",
+ "estimator__n_estimators": 50,
+ "estimator__max_depth": 10,
+ "estimator__min_samples_leaf": 1,
+}
+model_original.set_params(**hyperparameters_original)
+
+# solve the task and upload the result (this implicitly creates the flow)
+run = openml.runs.run_model_on_task(model_original, task, avoid_duplicate_runs=False)
+run_original = run.publish() # this implicitly uploads the flow
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/openml/venv/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:972: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.
+ warnings.warn(
+/code/openml/extensions/sklearn/extension.py:1785: UserWarning: Estimator only predicted for 5/6 classes!
+ warnings.warn(message)
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+```
+:::
+
+## 2) Download the flow and solve the same task again.
+
+``` default
+# obtain setup id (note that the setup id is assigned by the OpenML server -
+# therefore it was not yet available in our local copy of the run)
+run_downloaded = openml.runs.get_run(run_original.run_id)
+setup_id = run_downloaded.setup_id
+
+# after this, we can easily reinstantiate the model
+model_duplicate = openml.setups.initialize_model(setup_id)
+# it will automatically have all the hyperparameters set
+
+# and run the task again
+run_duplicate = openml.runs.run_model_on_task(model_duplicate, task, avoid_duplicate_runs=False)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/openml/venv/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:972: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.
+ warnings.warn(
+/code/openml/extensions/sklearn/extension.py:1785: UserWarning: Estimator only predicted for 5/6 classes!
+ warnings.warn(message)
+```
+:::
+
+## 3) We will verify that the obtained results are exactly the same.
+
+``` default
+# the run has stored all predictions in the field data content
+np.testing.assert_array_equal(run_original.data_content, run_duplicate.data_content)
+```
+
+``` default
+openml.config.stop_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-timing
+
+**Total running time of the script:** ( 0 minutes 6.595 seconds)
+:::
+
+::::::: {#sphx_glr_download_examples_30_extended_run_setup_tutorial.py}
+:::::: only
+html
+
+::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
+::: {.container .sphx-glr-download .sphx-glr-download-python}
+`Download Python source code: run_setup_tutorial.py <run_setup_tutorial.py>`{.interpreted-text
+role="download"}
+:::
+
+::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
+`Download Jupyter notebook: run_setup_tutorial.ipynb <run_setup_tutorial.ipynb>`{.interpreted-text
+role="download"}
+:::
+:::::
+::::::
+:::::::
+
+:::: only
+html
+
+::: rst-class
+sphx-glr-signature
+
+[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
+:::
+::::
diff --git a/docs/examples/30_extended/sg_execution_times.rst.md b/docs/examples/30_extended/sg_execution_times.rst.md
new file mode 100644
index 000000000..f9800d22b
--- /dev/null
+++ b/docs/examples/30_extended/sg_execution_times.rst.md
@@ -0,0 +1,51 @@
+orphan
+
+:
+
+# Computation times {#sphx_glr_examples_30_extended_sg_execution_times}
+
+**03:46.366** total execution time for **examples_30_extended** files:
+
+ ---------------------------------------------------------------------------------------- ----------- -----
+ `sphx_glr_examples_30_extended_fetch_runtimes_tutorial.py`{.interpreted-text role="ref"} 01:13.261 0.0
+ (`fetch_runtimes_tutorial.py`) MB
+
+ `sphx_glr_examples_30_extended_tasks_tutorial.py`{.interpreted-text role="ref"} 00:47.382 0.0
+ (`tasks_tutorial.py`) MB
+
+ `sphx_glr_examples_30_extended_suites_tutorial.py`{.interpreted-text role="ref"} 00:30.398 0.0
+ (`suites_tutorial.py`) MB
+
+ `sphx_glr_examples_30_extended_study_tutorial.py`{.interpreted-text role="ref"} 00:16.432 0.0
+ (`study_tutorial.py`) MB
+
+ `sphx_glr_examples_30_extended_fetch_evaluations_tutorial.py`{.interpreted-text 00:13.992 0.0
+ role="ref"} (`fetch_evaluations_tutorial.py`) MB
+
+ `sphx_glr_examples_30_extended_flows_and_runs_tutorial.py`{.interpreted-text role="ref"} 00:12.993 0.0
+ (`flows_and_runs_tutorial.py`) MB
+
+ `sphx_glr_examples_30_extended_datasets_tutorial.py`{.interpreted-text role="ref"} 00:08.138 0.0
+ (`datasets_tutorial.py`) MB
+
+ `sphx_glr_examples_30_extended_run_setup_tutorial.py`{.interpreted-text role="ref"} 00:06.595 0.0
+ (`run_setup_tutorial.py`) MB
+
+ `sphx_glr_examples_30_extended_plot_svm_hyperparameters_tutorial.py`{.interpreted-text 00:06.466 0.0
+ role="ref"} (`plot_svm_hyperparameters_tutorial.py`) MB
+
+ `sphx_glr_examples_30_extended_flow_id_tutorial.py`{.interpreted-text role="ref"} 00:04.274 0.0
+ (`flow_id_tutorial.py`) MB
+
+ `sphx_glr_examples_30_extended_task_manual_iteration_tutorial.py`{.interpreted-text 00:03.602 0.0
+ role="ref"} (`task_manual_iteration_tutorial.py`) MB
+
+ `sphx_glr_examples_30_extended_create_upload_tutorial.py`{.interpreted-text role="ref"} 00:02.833 0.0
+ (`create_upload_tutorial.py`) MB
+
+ `sphx_glr_examples_30_extended_configure_logging.py`{.interpreted-text role="ref"} 00:00.000 0.0
+ (`configure_logging.py`) MB
+
+ `sphx_glr_examples_30_extended_custom_flow_.py`{.interpreted-text role="ref"} 00:00.000 0.0
+ (`custom_flow_.py`) MB
+ ---------------------------------------------------------------------------------------- ----------- -----
diff --git a/docs/examples/30_extended/study_tutorial.rst.md b/docs/examples/30_extended/study_tutorial.rst.md
new file mode 100644
index 000000000..fcf4f1569
--- /dev/null
+++ b/docs/examples/30_extended/study_tutorial.rst.md
@@ -0,0 +1,402 @@
+::::: only
+html
+
+:::: {.note .sphx-glr-download-link-note}
+::: title
+Note
+:::
+
+`Go to the end <sphx_glr_download_examples_30_extended_study_tutorial.py>`{.interpreted-text
+role="ref"} to download the full example code
+::::
+:::::
+
+::: rst-class
+sphx-glr-example-title
+:::
+
+# Benchmark studies {#sphx_glr_examples_30_extended_study_tutorial.py}
+
+How to list, download and upload benchmark studies. In contrast to
+[benchmark
+suites](https://docs.openml.org/benchmark/#benchmarking-suites) which
+hold a list of tasks, studies hold a list of runs. As runs contain all
+information on flows and tasks, all required information about a study
+can be retrieved.
+
+``` default
+# License: BSD 3-Clause
+
+import uuid
+
+from sklearn.ensemble import RandomForestClassifier
+
+import openml
+```
+
+## Listing studies
+
+- Use the output_format parameter to select output type
+- Default gives `dict`, but we\'ll use `dataframe` to obtain an
+ easier-to-work-with data structure
+
+``` default
+studies = openml.study.list_studies(output_format="dataframe", status="all")
+print(studies.head(n=10))
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+id alias main_entity_type ... status creation_date creator
+1 1 Study_1 run ... in_preparation 2015-10-20 15:27:26 2
+2 2 Study_2 run ... in_preparation 2015-10-20 15:28:44 2
+3 3 Study_3 run ... in_preparation 2015-10-20 15:34:39 2
+5 5 Study_5 run ... in_preparation 2015-11-19 11:20:33 749
+7 7 Study_7 run ... in_preparation 2016-01-06 17:49:36 64
+8 8 Study_8 run ... in_preparation 2016-03-13 13:38:31 1135
+10 10 Study_10 run ... in_preparation 2016-03-16 12:16:08 507
+11 11 Study_11 run ... in_preparation 2016-03-22 16:48:06 1
+12 12 Study_12 run ... in_preparation 2016-03-31 15:05:45 1195
+13 13 Study_13 run ... in_preparation 2016-04-05 13:57:41 62
+
+[10 rows x 7 columns]
+```
+:::
+
+### Downloading studies
+
+This is done based on the study ID.
+
+``` default
+study = openml.study.get_study(123)
+print(study)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+OpenML Study
+============
+ID..............: 123
+Name............: Linear vs. Non Linear
+Status..........: active
+Main Entity Type: run
+Study URL.......: https://www.openml.org/s/123
+# of Data.......: 299
+# of Tasks......: 299
+# of Flows......: 5
+# of Runs.......: 1693
+Creator.........: https://www.openml.org/u/1
+Upload Time.....: 2019-02-21 19:55:30
+```
+:::
+
+Studies also features a description:
+
+``` default
+print(study.description)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+Comparison of linear and non-linear models.
+
+[Jupyter Notebook](https://github.com/janvanrijn/linear-vs-non-linear/blob/master/notebook/Linear-vs-Non-Linear.ipynb)
+```
+:::
+
+Studies are a container for runs:
+
+``` default
+print(study.runs)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+[9199877, 9199878, 9199879, 9199880, 9199881, 9199882, 9199883, 9199884, 9199885, 9199886, 9199887, 9199888, 9199889, 9199890, 9199891, 9199892, 9199893, 9199894, 9199895, 9199897, 9199898, 9199899, 9199900, 9199901, 9199902, 9199903, 9199904, 9199905, 9199906, 9199907, 9199908, 9199909, 9199910, 9199911, 9199912, 9199913, 9199914, 9199915, 9199916, 9199917, 9199918, 9199919, 9199920, 9199921, 9199922, 9199923, 9199924, 9199925, 9199926, 9199927, 9199928, 9199929, 9199930, 9199931, 9199932, 9199933, 9199934, 9199935, 9199936, 9199937, 9199938, 9199939, 9199940, 9199941, 9199942, 9199943, 9199944, 9199945, 9199946, 9199947, 9199948, 9199950, 9199951, 9199952, 9199953, 9199954, 9199955, 9199956, 9199957, 9199958, 9199959, 9199960, 9199961, 9199963, 9199964, 9199965, 9199966, 9199967, 9199968, 9199969, 9199970, 9199971, 9199972, 9199973, 9199974, 9199975, 9199976, 9199977, 9199978, 9199979, 9199981, 9199982, 9199983, 9199984, 9199985, 9199986, 9199987, 9199988, 9199989, 9199990, 9199991, 9199992, 9199993, 9199994, 9199995, 9199996, 9199997, 9199998, 9199999, 9200000, 9200001, 9200002, 9200003, 9200004, 9200006, 9200007, 9200008, 9200009, 9200010, 9200011, 9200012, 9200013, 9200014, 9200015, 9200016, 9200017, 9200018, 9200019, 9200020, 9200021, 9200022, 9200023, 9200024, 9200025, 9200026, 9200027, 9200028, 9200029, 9200030, 9200031, 9200032, 9200033, 9200034, 9200035, 9200036, 9200037, 9200038, 9200039, 9200040, 9200041, 9200042, 9200043, 9200044, 9200045, 9200046, 9200047, 9200048, 9200049, 9200050, 9200051, 9200052, 9200053, 9200054, 9200055, 9200056, 9200057, 9200058, 9200059, 9200060, 9200061, 9200062, 9200063, 9200064, 9200065, 9200066, 9200067, 9200068, 9200069, 9200070, 9200071, 9200072, 9200073, 9200074, 9200075, 9200076, 9200077, 9200078, 9200079, 9200080, 9200081, 9200082, 9200083, 9200084, 9200085, 9200086, 9200087, 9200088, 9200089, 9200090, 9200091, 9200092, 9200093, 9200094, 9200095, 9200096, 9200097, 9200098, 9200099, 9200100, 9200101, 9200102, 9200103, 9200104, 9200105, 9200106, 9200107, 9200108, 9200109, 9200110, 9200111, 9200112, 9200113, 9200114, 9200115, 9200116, 9200117, 9200118, 9200119, 9200120, 9200121, 9200122, 9200123, 9200124, 9200125, 9200126, 9200127, 9200128, 9200129, 9200130, 9200131, 9200132, 9200133, 9200134, 9200135, 9200136, 9200137, 9200138, 9200139, 9200140, 9200141, 9200142, 9200143, 9200144, 9200145, 9200146, 9200147, 9200148, 9200149, 9200150, 9200151, 9200152, 9200153, 9200154, 9200155, 9200156, 9200157, 9200158, 9200159, 9200160, 9200161, 9200162, 9200163, 9200164, 9200165, 9200166, 9200167, 9200168, 9200169, 9200171, 9200173, 9200174, 9200175, 9200176, 9200177, 9200178, 9200180, 9200181, 9200182, 9200183, 9200184, 9200185, 9200186, 9200187, 9200188, 9200189, 9200190, 9200191, 9200192, 9200193, 9200194, 9200195, 9200196, 9200197, 9200198, 9200199, 9200200, 9200201, 9200202, 9200203, 9200204, 9200205, 9200206, 9200207, 9200208, 9200209, 9200210, 9200211, 9200212, 9200213, 9200214, 9200215, 9200216, 9200217, 9200218, 9200219, 9200220, 9200221, 9200222, 9200223, 9200224, 9200225, 9200226, 9200227, 9200228, 9200229, 9200230, 9200231, 9200232, 9200233, 9200234, 9200235, 9200236, 9200237, 9200238, 9200239, 9200240, 9200241, 9200242, 9200243, 9200244, 9200245, 9200246, 9200247, 9200248, 9200249, 9200250, 9200251, 9200252, 9200253, 9200254, 9200255, 9200256, 9200257, 9200258, 9200259, 9200260, 9200261, 9200262, 9200263, 9200264, 9200265, 9200266, 9200267, 9200268, 9200269, 9200270, 9200271, 9200272, 9200273, 9200274, 9200275, 9200276, 9200277, 9200278, 9200279, 9200280, 9200281, 9200282, 9200283, 9200284, 9200285, 9200286, 9200287, 9200288, 9200289, 9200290, 9200291, 9200292, 9200293, 9200294, 9200295, 9200296, 9200297, 9200298, 9200299, 9200300, 9200301, 9200302, 9200303, 9200304, 9200305, 9200306, 9200307, 9200308, 9200310, 9200311, 9200312, 9200313, 9200314, 9200315, 9200316, 9200317, 9200318, 9200319, 9200320, 9200321, 9200322, 9200324, 9200325, 9200326, 9200327, 9200328, 9200329, 9200330, 9200331, 9200332, 9200333, 9200334, 9200335, 9200336, 9200337, 9200338, 9200339, 9200340, 9200341, 9200342, 9200343, 9200344, 9200345, 9200346, 9200347, 9200348, 9200349, 9200350, 9200351, 9200352, 9200353, 9200354, 9200355, 9200356, 9200357, 9200358, 9200359, 9200361, 9200362, 9200364, 9200365, 9200366, 9200367, 9200368, 9200369, 9200370, 9200371, 9200372, 9200373, 9200374, 9200375, 9200376, 9200377, 9200378, 9200379, 9200380, 9200382, 9200383, 9200384, 9200385, 9200386, 9200387, 9200388, 9200389, 9200390, 9200391, 9200392, 9200393, 9200394, 9200395, 9200396, 9200397, 9200398, 9200399, 9200400, 9200401, 9200402, 9200403, 9200404, 9200405, 9200406, 9200407, 9200408, 9200409, 9200410, 9200411, 9200412, 9200413, 9200414, 9200415, 9200416, 9200417, 9200418, 9200419, 9200420, 9200421, 9200422, 9200424, 9200425, 9200426, 9200427, 9200428, 9200429, 9200430, 9200431, 9200432, 9200433, 9200434, 9200435, 9200436, 9200437, 9200438, 9200439, 9200440, 9200441, 9200442, 9200443, 9200444, 9200445, 9200446, 9200447, 9200448, 9200449, 9200450, 9200451, 9200452, 9200453, 9200454, 9200455, 9200456, 9200457, 9200458, 9200459, 9200460, 9200461, 9200462, 9200463, 9200464, 9200465, 9200466, 9200467, 9200468, 9200469, 9200470, 9200471, 9200472, 9200473, 9200474, 9200475, 9200476, 9200477, 9200478, 9200479, 9200480, 9200481, 9200482, 9200483, 9200484, 9200485, 9200486, 9200487, 9200488, 9200489, 9200490, 9200491, 9200492, 9200493, 9200494, 9200495, 9200496, 9200497, 9200498, 9200499, 9200500, 9200501, 9200502, 9200503, 9200504, 9200505, 9200506, 9200507, 9200508, 9200509, 9200510, 9200511, 9200512, 9200513, 9200514, 9200515, 9200516, 9200517, 9200518, 9200519, 9200520, 9200521, 9200522, 9200523, 9200524, 9200525, 9200526, 9200527, 9200528, 9200529, 9200530, 9200531, 9200532, 9200533, 9200534, 9200535, 9200536, 9200537, 9200538, 9200539, 9200540, 9200541, 9200542, 9200543, 9200544, 9200545, 9200546, 9200547, 9200548, 9200549, 9200550, 9200551, 9200552, 9200553, 9200554, 9200555, 9200556, 9200557, 9200558, 9200559, 9200560, 9200561, 9200562, 9200563, 9200564, 9200565, 9200566, 9200567, 9200568, 9200569, 9200570, 9200571, 9200572, 9200573, 9200574, 9200575, 9200576, 9200577, 9200578, 9200579, 9200580, 9200581, 9200582, 9200583, 9200584, 9200585, 9200586, 9200587, 9200588, 9200589, 9200590, 9200591, 9200592, 9200593, 9200594, 9200595, 9200596, 9200597, 9200598, 9200599, 9200600, 9200601, 9200602, 9200603, 9200604, 9200605, 9200606, 9200607, 9200608, 9200609, 9200610, 9200611, 9200612, 9200613, 9200614, 9200615, 9200616, 9200617, 9200618, 9200619, 9200620, 9200621, 9200622, 9200623, 9200624, 9200625, 9200626, 9200627, 9200628, 9200629, 9200630, 9200631, 9200632, 9200633, 9200634, 9200635, 9200636, 9200637, 9200638, 9200639, 9200640, 9200641, 9200642, 9200643, 9200644, 9200645, 9200646, 9200647, 9200648, 9200649, 9200650, 9200651, 9200652, 9200653, 9200654, 9200655, 9200656, 9200657, 9200658, 9200659, 9200660, 9200661, 9200662, 9200663, 9200664, 9200665, 9200666, 9200667, 9200668, 9200669, 9200670, 9200671, 9200672, 9200673, 9200674, 9200675, 9200676, 9200677, 9200678, 9200679, 9200680, 9200681, 9200682, 9200683, 9200684, 9200685, 9200686, 9200687, 9200688, 9200689, 9200690, 9200691, 9200692, 9200693, 9200694, 9200695, 9200696, 9200697, 9200698, 9200699, 9200700, 9200701, 9200702, 9200703, 9200704, 9200705, 9200706, 9200707, 9200708, 9200709, 9200710, 9200711, 9200712, 9200713, 9200714, 9200715, 9200716, 9200717, 9200719, 9200720, 9200721, 9200722, 9200723, 9200724, 9200725, 9200726, 9200727, 9200728, 9200729, 9200730, 9200731, 9200732, 9200733, 9200734, 9200735, 9200736, 9200737, 9200738, 9200739, 9200740, 9200741, 9200742, 9200743, 9200744, 9200745, 9200746, 9200747, 9200748, 9200749, 9200750, 9200751, 9200752, 9200753, 9200754, 9200755, 9200756, 9200757, 9200758, 9200759, 9200760, 9200761, 9200762, 9200763, 9200764, 9200765, 9200766, 9200767, 9200768, 9200769, 9200770, 9200771, 9200772, 9200773, 9200774, 9200775, 9200776, 9200778, 9200779, 9200780, 9200781, 9200782, 9200783, 9200784, 9200785, 9200786, 9200787, 9200788, 9200789, 9200790, 9200791, 9200792, 9200793, 9200794, 9200795, 9200796, 9200797, 9200798, 9200799, 9200800, 9200801, 9200802, 9200803, 9200804, 9200806, 9200807, 9200808, 9200809, 9200810, 9200811, 9200812, 9200813, 9200814, 9200815, 9200816, 9200817, 9200818, 9200819, 9200820, 9200821, 9200822, 9200823, 9200824, 9200825, 9200826, 9200827, 9200828, 9200829, 9200830, 9200831, 9200832, 9200833, 9200834, 9200835, 9200836, 9200837, 9200838, 9200839, 9200840, 9200841, 9200842, 9200843, 9200844, 9200845, 9200846, 9200847, 9200848, 9200849, 9200850, 9200851, 9200852, 9200853, 9200854, 9200855, 9200856, 9200857, 9200858, 9200859, 9200860, 9200861, 9200862, 9200863, 9200864, 9200865, 9200866, 9200867, 9200868, 9200869, 9200870, 9200871, 9200872, 9200873, 9200874, 9200875, 9200876, 9200877, 9200878, 9200879, 9200880, 9200881, 9200882, 9200883, 9200884, 9200885, 9200886, 9200887, 9200888, 9200889, 9200890, 9200891, 9200892, 9200893, 9200894, 9200895, 9200896, 9200897, 9200898, 9200899, 9200900, 9200901, 9200902, 9200903, 9200904, 9200905, 9200906, 9200907, 9200908, 9200909, 9200910, 9200911, 9200912, 9200913, 9200914, 9200915, 9200916, 9200917, 9200918, 9200919, 9200920, 9200921, 9200922, 9200923, 9200924, 9200925, 9200926, 9200927, 9200928, 9200929, 9200930, 9200931, 9200932, 9200933, 9200934, 9200935, 9200936, 9200937, 9200938, 9200939, 9200940, 9200941, 9200942, 9200943, 9200944, 9200945, 9200946, 9200947, 9200948, 9200949, 9200950, 9200951, 9200952, 9200953, 9200954, 9200955, 9200956, 9200957, 9200958, 9200959, 9200960, 9200961, 9200962, 9200963, 9200964, 9200965, 9200966, 9200967, 9200968, 9200969, 9200970, 9200971, 9200972, 9200973, 9200974, 9200975, 9200976, 9200977, 9200978, 9200979, 9200980, 9200981, 9200982, 9200983, 9200984, 9200985, 9200986, 9200987, 9200988, 9200989, 9200990, 9200991, 9200993, 9200994, 9200995, 9200996, 9200997, 9200998, 9200999, 9201000, 9201001, 9201002, 9201003, 9201004, 9201005, 9201006, 9201007, 9201008, 9201009, 9201010, 9201011, 9201012, 9201013, 9201014, 9201015, 9201016, 9201017, 9201018, 9201019, 9201020, 9201021, 9201022, 9201023, 9201024, 9201025, 9201026, 9201027, 9201028, 9201029, 9201030, 9201031, 9201032, 9201033, 9201034, 9201035, 9201036, 9201037, 9201038, 9201039, 9201040, 9201041, 9201042, 9201043, 9201044, 9201045, 9201046, 9201047, 9201048, 9201049, 9201050, 9201051, 9201052, 9201053, 9201054, 9201055, 9201056, 9201057, 9201058, 9201059, 9201060, 9201061, 9201062, 9201063, 9201064, 9201065, 9201066, 9201067, 9201068, 9201069, 9201070, 9201071, 9201072, 9201073, 9201074, 9201075, 9201076, 9201077, 9201078, 9201079, 9201080, 9201081, 9201082, 9201083, 9201084, 9201085, 9201086, 9201087, 9201088, 9201089, 9201090, 9201091, 9201092, 9201093, 9201094, 9201095, 9201096, 9201097, 9201098, 9201099, 9201100, 9201101, 9201102, 9201103, 9201104, 9201105, 9201106, 9201107, 9201108, 9201109, 9201110, 9201111, 9201112, 9201113, 9201114, 9201115, 9201116, 9201117, 9201118, 9201119, 9201120, 9201121, 9201122, 9201124, 9201125, 9201126, 9201127, 9201128, 9201129, 9201130, 9201131, 9201132, 9201133, 9201134, 9201135, 9201136, 9201137, 9201138, 9201139, 9201140, 9201141, 9201142, 9201143, 9201144, 9201145, 9201146, 9201147, 9201148, 9201149, 9201150, 9201151, 9201152, 9201153, 9201154, 9201155, 9201156, 9201157, 9201158, 9201159, 9201160, 9201161, 9201162, 9201163, 9201164, 9201165, 9201166, 9201167, 9201168, 9201169, 9201170, 9201171, 9201172, 9201173, 9201174, 9201175, 9201176, 9201177, 9201178, 9201179, 9201180, 9201181, 9201182, 9201183, 9201184, 9201185, 9201186, 9201187, 9201188, 9201189, 9201190, 9201191, 9201192, 9201193, 9201194, 9201195, 9201196, 9201197, 9201198, 9201199, 9201200, 9201201, 9201202, 9201203, 9201204, 9201205, 9201206, 9201207, 9201208, 9201209, 9201210, 9201211, 9201212, 9201213, 9201214, 9201215, 9201216, 9201217, 9201218, 9201219, 9201220, 9201221, 9201222, 9201223, 9201224, 9201225, 9201226, 9201227, 9201228, 9201229, 9201230, 9201231, 9201232, 9201233, 9201234, 9201235, 9201236, 9201237, 9201238, 9201239, 9201240, 9201241, 9201242, 9201243, 9201244, 9201245, 9201246, 9201247, 9201248, 9201249, 9201250, 9201251, 9201252, 9201253, 9201254, 9201255, 9201256, 9201257, 9201258, 9201259, 9201260, 9201261, 9201262, 9201263, 9201264, 9201265, 9201266, 9201267, 9201268, 9201269, 9201270, 9201271, 9201272, 9201273, 9201274, 9201275, 9201276, 9201277, 9201278, 9201279, 9201280, 9201281, 9201282, 9201283, 9201284, 9201285, 9201286, 9201287, 9201288, 9201289, 9201290, 9201291, 9201292, 9201293, 9201294, 9201295, 9201296, 9201297, 9201298, 9201299, 9201300, 9201301, 9201302, 9201303, 9201304, 9201305, 9201306, 9201307, 9201308, 9201309, 9201310, 9201311, 9201312, 9201313, 9201314, 9201315, 9201316, 9201317, 9201318, 9201319, 9201320, 9201321, 9201322, 9201323, 9201324, 9201325, 9201326, 9201327, 9201328, 9201329, 9201330, 9201331, 9201332, 9201333, 9201334, 9201335, 9201336, 9201337, 9201338, 9201339, 9201340, 9201341, 9201342, 9201343, 9201344, 9201345, 9201346, 9201347, 9201348, 9201349, 9201350, 9201351, 9201352, 9201353, 9201354, 9201355, 9201356, 9201357, 9201358, 9201359, 9201360, 9201361, 9201362, 9201363, 9201364, 9201365, 9201366, 9201367, 9201368, 9201369, 9201370, 9201371, 9201372, 9201373, 9201374, 9201375, 9201376, 9201377, 9201378, 9201379, 9201380, 9201381, 9201382, 9201384, 9201385, 9201386, 9201387, 9201388, 9201389, 9201390, 9201391, 9201392, 9201393, 9201394, 9201395, 9201396, 9201397, 9201398, 9201399, 9201400, 9201401, 9201402, 9201403, 9201404, 9201405, 9201406, 9201407, 9201408, 9201409, 9201410, 9201411, 9201412, 9201413, 9201414, 9201415, 9201416, 9201417, 9201418, 9201419, 9201420, 9201422, 9201423, 9201424, 9201425, 9201426, 9201427, 9201428, 9201429, 9201430, 9201431, 9201432, 9201433, 9201434, 9201435, 9201436, 9201437, 9201438, 9201439, 9201440, 9201441, 9201442, 9201443, 9201444, 9201445, 9201446, 9201447, 9201448, 9201449, 9201450, 9201451, 9201452, 9201453, 9201454, 9201455, 9201456, 9201457, 9201458, 9201459, 9201460, 9201461, 9201462, 9201463, 9201464, 9201465, 9201466, 9201467, 9201468, 9201469, 9201470, 9201471, 9201472, 9201473, 9201474, 9201475, 9201476, 9201477, 9201478, 9201479, 9201480, 9201481, 9201482, 9201483, 9201484, 9201485, 9201486, 9201487, 9201488, 9201489, 9201490, 9201491, 9201492, 9201493, 9201494, 9201495, 9201496, 9201497, 9201498, 9201499, 9201500, 9201501, 9201502, 9201503, 9201504, 9201505, 9201506, 9201507, 9201508, 9201509, 9201510, 9201511, 9201512, 9201514, 9201515, 9201516, 9201517, 9201518, 9201519, 9201520, 9201521, 9201522, 9201523, 9201524, 9201525, 9201526, 9201527, 9201528, 9201529, 9201530, 9201531, 9201532, 9201533, 9201534, 9201535, 9201536, 9201537, 9201538, 9201539, 9201540, 9201541, 9201542, 9201543, 9201544, 9201545, 9201546, 9201547, 9201548, 9201549, 9201550, 9201551, 9201552, 9201553, 9201554, 9201555, 9201556, 9201557, 9201558, 9201559, 9201560, 9201561, 9201562, 9201563, 9201564, 9201565, 9201566, 9201567, 9201568, 9201569, 9201570, 9201571, 9201572, 9201573, 9201574, 9201575, 9201576, 9201577, 9201578, 9201579, 9201580, 9201581, 9201582, 9201583, 9201584, 9201585, 9201586, 9201587, 9201588, 9201589, 9201590, 9201591]
+```
+:::
+
+And we can use the evaluation listing functionality to learn more about
+the evaluations available for the conducted runs:
+
+``` default
+evaluations = openml.evaluations.list_evaluations(
+ function="predictive_accuracy",
+ output_format="dataframe",
+ study=study.study_id,
+)
+print(evaluations.head())
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+run_id task_id setup_id ... value values array_data
+0 9199877 3 7130157 ... 0.974969 None None
+1 9199878 6 7130158 ... 0.716500 None None
+2 9199879 6 7130159 ... 0.967200 None None
+3 9199880 11 7130158 ... 0.886400 None None
+4 9199881 11 7130159 ... 0.976000 None None
+
+[5 rows x 14 columns]
+```
+:::
+
+We\'ll use the test server for the rest of this tutorial.
+
+:::: warning
+::: title
+Warning
+:::
+
+.. include:: ../../test_server_usage_warning.txt
+::::
+
+``` default
+openml.config.start_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!
+ warnings.warn(
+```
+:::
+
+### Uploading studies
+
+Creating a study is as simple as creating any kind of other OpenML
+entity. In this examples we\'ll create a few runs for the OpenML-100
+benchmark suite which is available on the OpenML test server.
+
+``` default
+# Model to be used
+clf = RandomForestClassifier()
+
+# We'll create a study with one run on 3 datasets present in the suite
+tasks = [115, 259, 307]
+
+# To verify
+suite = openml.study.get_suite(1)
+print(all([t_id in suite.tasks for t_id in tasks]))
+
+run_ids = []
+for task_id in tasks:
+ task = openml.tasks.get_task(task_id)
+ run = openml.runs.run_model_on_task(clf, task)
+ run.publish()
+ run_ids.append(run.run_id)
+
+# The study needs a machine-readable and unique alias. To obtain this,
+# we simply generate a random uuid.
+alias = uuid.uuid4().hex
+
+new_study = openml.study.create_study(
+ name="Test-Study",
+ description="Test study for the Python tutorial on studies",
+ run_ids=run_ids,
+ alias=alias,
+ benchmark_suite=suite.study_id,
+)
+new_study.publish()
+print(new_study)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+True
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+OpenML Study
+============
+ID..............: 333
+Name............: Test-Study
+Status..........: None
+Main Entity Type: run
+Study URL.......: https://test.openml.org/s/333
+# of Runs.......: 3
+```
+:::
+
+``` default
+openml.config.stop_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-timing
+
+**Total running time of the script:** ( 0 minutes 16.432 seconds)
+:::
+
+::::::: {#sphx_glr_download_examples_30_extended_study_tutorial.py}
+:::::: only
+html
+
+::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
+::: {.container .sphx-glr-download .sphx-glr-download-python}
+`Download Python source code: study_tutorial.py <study_tutorial.py>`{.interpreted-text
+role="download"}
+:::
+
+::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
+`Download Jupyter notebook: study_tutorial.ipynb <study_tutorial.ipynb>`{.interpreted-text
+role="download"}
+:::
+:::::
+::::::
+:::::::
+
+:::: only
+html
+
+::: rst-class
+sphx-glr-signature
+
+[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
+:::
+::::
diff --git a/docs/examples/30_extended/suites_tutorial.rst.md b/docs/examples/30_extended/suites_tutorial.rst.md
new file mode 100644
index 000000000..c1d74f8e4
--- /dev/null
+++ b/docs/examples/30_extended/suites_tutorial.rst.md
@@ -0,0 +1,291 @@
+::::: only
+html
+
+:::: {.note .sphx-glr-download-link-note}
+::: title
+Note
+:::
+
+`Go to the end <sphx_glr_download_examples_30_extended_suites_tutorial.py>`{.interpreted-text
+role="ref"} to download the full example code
+::::
+:::::
+
+::: rst-class
+sphx-glr-example-title
+:::
+
+# Benchmark suites {#sphx_glr_examples_30_extended_suites_tutorial.py}
+
+How to list, download and upload benchmark suites.
+
+If you want to learn more about benchmark suites, check out our brief
+introductory tutorial
+`sphx_glr_examples_20_basic_simple_suites_tutorial.py`{.interpreted-text
+role="ref"} or the [OpenML benchmark
+docs](https://docs.openml.org/benchmark/#benchmarking-suites).
+
+``` default
+# License: BSD 3-Clause
+
+import uuid
+
+import numpy as np
+
+import openml
+```
+
+## Listing suites
+
+- Use the output_format parameter to select output type
+- Default gives `dict`, but we\'ll use `dataframe` to obtain an
+ easier-to-work-with data structure
+
+``` default
+suites = openml.study.list_suites(output_format="dataframe", status="all")
+print(suites.head(n=10))
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+id alias ... creation_date creator
+14 14 OpenML100 ... 2019-02-21 18:40:13 1
+99 99 OpenML-CC18 ... 2019-02-21 18:47:13 1
+218 218 AutoML-Benchmark ... 2019-05-02 13:35:08 869
+219 219 FOREX ... 2019-06-04 00:45:17 1
+225 225 OpenML-friendly ... 2019-09-16 19:41:46 1
+236 236 a9ee1f0b2a4b48b6b6da1653fe92890e ... 2020-04-06 21:38:55 64
+239 239 c638a5d3d31241179f9b4853951fdb79 ... 2020-04-19 22:15:30 64
+240 240 e5e7f56c8655433eb2418c240ec8b8c0 ... 2020-04-28 02:30:38 2902
+253 253 testecc18 ... 2020-09-01 00:57:54 8598
+258 258 NaN ... 2020-09-30 08:30:00 8598
+
+[10 rows x 7 columns]
+```
+:::
+
+### Downloading suites
+
+This is done based on the dataset ID.
+
+``` default
+suite = openml.study.get_suite(99)
+print(suite)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+OpenML Benchmark Suite
+======================
+ID..............: 99
+Name............: OpenML-CC18 Curated Classification benchmark
+Status..........: active
+Main Entity Type: task
+Study URL.......: https://www.openml.org/s/99
+# of Data.......: 72
+# of Tasks......: 72
+Creator.........: https://www.openml.org/u/1
+Upload Time.....: 2019-02-21 18:47:13
+```
+:::
+
+Suites also feature a description:
+
+``` default
+print(suite.description)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+```` none
+We advocate the use of curated, comprehensive benchmark suites of machine learning datasets, backed by standardized OpenML-based interfaces and complementary software toolkits written in Python, Java and R. We demonstrate how to easily execute comprehensive benchmarking studies using standardized OpenML-based benchmarking suites and complementary software toolkits written in Python, Java and R. Major distinguishing features of OpenML benchmark suites are (i) ease of use through standardized data formats, APIs, and existing client libraries; (ii) machine-readable meta-information regarding the contents of the suite; and (iii) online sharing of results, enabling large scale comparisons. As a first such suite, we propose the OpenML-CC18, a machine learning benchmark suite of 72 classification datasets carefully curated from the thousands of datasets on OpenML.
+
+The inclusion criteria are:
+* classification tasks on dense data set
+independent observations
+* number of classes larger or equal to 2, each class with at least 20 observations and ratio of minority to majority class must exceed 5%
+* number of observations between 500 and 100000
+* number of features after one-hot-encoding less than 5000
+* no artificial data sets
+* no subsets of larger data sets nor binarizations of other data sets
+* no data sets which are perfectly predictable by using a single feature or by using a simple decision tree
+* source or reference available
+
+
+If you use this benchmarking suite, please cite:
+
+Bernd Bischl, Giuseppe Casalicchio, Matthias Feurer, Frank Hutter, Michel Lang, Rafael G. Mantovani, Jan N. van Rijn and Joaquin Vanschoren. “OpenML Benchmarking Suites” arXiv:1708.03731v2 [stats.ML] (2019).
+
+```
+@article{oml-benchmarking-suites,
+ title={OpenML Benchmarking Suites},
+ author={Bernd Bischl and Giuseppe Casalicchio and Matthias Feurer and Frank Hutter and Michel Lang and Rafael G. Mantovani and Jan N. van Rijn and Joaquin Vanschoren},
+ year={2019},
+ journal={arXiv:1708.03731v2 [stat.ML]}
+}
+```
+````
+:::
+
+Suites are a container for tasks:
+
+``` default
+print(suite.tasks)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+[3, 6, 11, 12, 14, 15, 16, 18, 22, 23, 28, 29, 31, 32, 37, 43, 45, 49, 53, 219, 2074, 2079, 3021, 3022, 3481, 3549, 3560, 3573, 3902, 3903, 3904, 3913, 3917, 3918, 7592, 9910, 9946, 9952, 9957, 9960, 9964, 9971, 9976, 9977, 9978, 9981, 9985, 10093, 10101, 14952, 14954, 14965, 14969, 14970, 125920, 125922, 146195, 146800, 146817, 146819, 146820, 146821, 146822, 146824, 146825, 167119, 167120, 167121, 167124, 167125, 167140, 167141]
+```
+:::
+
+And we can use the task listing functionality to learn more about them:
+
+``` default
+tasks = openml.tasks.list_tasks(output_format="dataframe")
+
+# Using ``@`` in `pd.DataFrame.query <
+# https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.query.html>`_
+# accesses variables outside of the current dataframe.
+tasks = tasks.query("tid in @suite.tasks")
+print(tasks.describe().transpose())
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+count mean ... 75% max
+tid 72.0 41849.305556 ... 42707.50 167141.0
+did 72.0 10472.263889 ... 10594.25 41027.0
+MajorityClassSize 72.0 4820.166667 ... 2796.00 48658.0
+MaxNominalAttDistinctValues 71.0 8.718310 ... 10.00 71.0
+MinorityClassSize 72.0 2199.777778 ... 1326.50 47662.0
+NumberOfClasses 72.0 5.680556 ... 8.25 46.0
+NumberOfFeatures 72.0 196.597222 ... 71.50 3073.0
+NumberOfInstances 72.0 12148.972222 ... 6771.75 96320.0
+NumberOfInstancesWithMissingValues 72.0 121.347222 ... 0.00 3772.0
+NumberOfMissingValues 72.0 226.597222 ... 0.00 6465.0
+NumberOfNumericFeatures 72.0 166.875000 ... 64.00 3072.0
+NumberOfSymbolicFeatures 72.0 29.722222 ... 5.25 1556.0
+target_feature_left 0.0 NaN ... NaN NaN
+
+[13 rows x 8 columns]
+```
+:::
+
+We\'ll use the test server for the rest of this tutorial.
+
+:::: warning
+::: title
+Warning
+:::
+
+.. include:: ../../test_server_usage_warning.txt
+::::
+
+``` default
+openml.config.start_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!
+ warnings.warn(
+```
+:::
+
+### Uploading suites
+
+Uploading suites is as simple as uploading any kind of other OpenML
+entity - the only reason why we need so much code in this example is
+because we upload some random data.
+
+``` default
+# We'll take a random subset of at least ten tasks of all available tasks on
+# the test server:
+all_tasks = list(openml.tasks.list_tasks(output_format="dataframe")["tid"])
+task_ids_for_suite = sorted(np.random.choice(all_tasks, replace=False, size=20))
+
+# The study needs a machine-readable and unique alias. To obtain this,
+# we simply generate a random uuid.
+
+alias = uuid.uuid4().hex
+
+new_suite = openml.study.create_benchmark_suite(
+ name="Test-Suite",
+ description="Test suite for the Python tutorial on benchmark suites",
+ task_ids=task_ids_for_suite,
+ alias=alias,
+)
+new_suite.publish()
+print(new_suite)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/tasks/functions.py:112: RuntimeWarning: Could not create task type id for 10 due to error 10 is not a valid TaskType
+ warnings.warn(
+/code/openml/tasks/functions.py:112: RuntimeWarning: Could not create task type id for 11 due to error 11 is not a valid TaskType
+ warnings.warn(
+/code/openml/tasks/functions.py:263: RuntimeWarning: Could not create task type id for 10 due to error 10 is not a valid TaskType
+ warnings.warn(
+OpenML Benchmark Suite
+======================
+ID..............: 332
+Name............: Test-Suite
+Status..........: None
+Main Entity Type: task
+Study URL.......: https://test.openml.org/s/332
+# of Tasks......: 20
+```
+:::
+
+``` default
+openml.config.stop_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-timing
+
+**Total running time of the script:** ( 0 minutes 30.398 seconds)
+:::
+
+::::::: {#sphx_glr_download_examples_30_extended_suites_tutorial.py}
+:::::: only
+html
+
+::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
+::: {.container .sphx-glr-download .sphx-glr-download-python}
+`Download Python source code: suites_tutorial.py <suites_tutorial.py>`{.interpreted-text
+role="download"}
+:::
+
+::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
+`Download Jupyter notebook: suites_tutorial.ipynb <suites_tutorial.ipynb>`{.interpreted-text
+role="download"}
+:::
+:::::
+::::::
+:::::::
+
+:::: only
+html
+
+::: rst-class
+sphx-glr-signature
+
+[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
+:::
+::::
diff --git a/docs/examples/30_extended/task_manual_iteration_tutorial.rst.md b/docs/examples/30_extended/task_manual_iteration_tutorial.rst.md
new file mode 100644
index 000000000..059b5cc6f
--- /dev/null
+++ b/docs/examples/30_extended/task_manual_iteration_tutorial.rst.md
@@ -0,0 +1,522 @@
+::::: only
+html
+
+:::: {.note .sphx-glr-download-link-note}
+::: title
+Note
+:::
+
+`Go to the end <sphx_glr_download_examples_30_extended_task_manual_iteration_tutorial.py>`{.interpreted-text
+role="ref"} to download the full example code
+::::
+:::::
+
+::: rst-class
+sphx-glr-example-title
+:::
+
+# Tasks: retrieving splits {#sphx_glr_examples_30_extended_task_manual_iteration_tutorial.py}
+
+Tasks define a target and a train/test split. Normally, they are the
+input to the function `openml.runs.run_model_on_task` which
+automatically runs the model on all splits of the task. However,
+sometimes it is necessary to manually split a dataset to perform
+experiments outside of the functions provided by OpenML. One such
+example is in the benchmark library
+[HPOBench](https://github.com/automl/HPOBench) which extensively uses
+data from OpenML, but not OpenML\'s functionality to conduct runs.
+
+``` default
+# License: BSD 3-Clause
+
+import openml
+```
+
+For this tutorial we will use the famous King+Rook versus King+Pawn on
+A7 dataset, which has the dataset ID 3 ([dataset on
+OpenML](https://www.openml.org/d/3)), and for which there exist tasks
+with all important estimation procedures. It is small enough (less than
+5000 samples) to efficiently use it in an example.
+
+We will first start with ([task 233](https://www.openml.org/t/233)),
+which is a task with a holdout estimation procedure.
+
+``` default
+task_id = 233
+task = openml.tasks.get_task(task_id)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+```
+:::
+
+Now that we have a task object we can obtain the number of repetitions,
+folds and samples as defined by the task:
+
+``` default
+n_repeats, n_folds, n_samples = task.get_split_dimensions()
+```
+
+- `n_repeats`: Number of times the model quality estimation is
+ performed
+- `n_folds`: Number of folds per repeat
+- `n_samples`: How many data points to use. This is only relevant for
+ learning curve tasks
+
+A list of all available estimation procedures is available
+[here](https://www.openml.org/search?q=%2520measure_type%3Aestimation_procedure&type=measure).
+
+Task `233` is a simple task using the holdout estimation procedure and
+therefore has only a single repeat, a single fold and a single sample
+size:
+
+``` default
+print(
+ "Task {}: number of repeats: {}, number of folds: {}, number of samples {}.".format(
+ task_id,
+ n_repeats,
+ n_folds,
+ n_samples,
+ )
+)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+Task 233: number of repeats: 1, number of folds: 1, number of samples 1.
+```
+:::
+
+We can now retrieve the train/test split for this combination of
+repeats, folds and number of samples (indexing is zero-based). Usually,
+one would loop over all repeats, folds and sample sizes, but we can
+neglect this here as there is only a single repetition.
+
+``` default
+train_indices, test_indices = task.get_train_test_split_indices(
+ repeat=0,
+ fold=0,
+ sample=0,
+)
+
+print(train_indices.shape, train_indices.dtype)
+print(test_indices.shape, test_indices.dtype)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+(2142,) int32
+(1054,) int32
+```
+:::
+
+And then split the data based on this:
+
+``` default
+X, y = task.get_X_and_y(dataset_format="dataframe")
+X_train = X.iloc[train_indices]
+y_train = y.iloc[train_indices]
+X_test = X.iloc[test_indices]
+y_test = y.iloc[test_indices]
+
+print(
+ "X_train.shape: {}, y_train.shape: {}, X_test.shape: {}, y_test.shape: {}".format(
+ X_train.shape,
+ y_train.shape,
+ X_test.shape,
+ y_test.shape,
+ )
+)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+X_train.shape: (2142, 36), y_train.shape: (2142,), X_test.shape: (1054, 36), y_test.shape: (1054,)
+```
+:::
+
+Obviously, we can also retrieve cross-validation versions of the dataset
+used in task `233`:
+
+``` default
+task_id = 3
+task = openml.tasks.get_task(task_id)
+X, y = task.get_X_and_y(dataset_format="dataframe")
+n_repeats, n_folds, n_samples = task.get_split_dimensions()
+print(
+ "Task {}: number of repeats: {}, number of folds: {}, number of samples {}.".format(
+ task_id,
+ n_repeats,
+ n_folds,
+ n_samples,
+ )
+)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+Task 3: number of repeats: 1, number of folds: 10, number of samples 1.
+```
+:::
+
+And then perform the aforementioned iteration over all splits:
+
+``` default
+for repeat_idx in range(n_repeats):
+ for fold_idx in range(n_folds):
+ for sample_idx in range(n_samples):
+ train_indices, test_indices = task.get_train_test_split_indices(
+ repeat=repeat_idx,
+ fold=fold_idx,
+ sample=sample_idx,
+ )
+ X_train = X.iloc[train_indices]
+ y_train = y.iloc[train_indices]
+ X_test = X.iloc[test_indices]
+ y_test = y.iloc[test_indices]
+
+ print(
+ "Repeat #{}, fold #{}, samples {}: X_train.shape: {}, "
+ "y_train.shape {}, X_test.shape {}, y_test.shape {}".format(
+ repeat_idx,
+ fold_idx,
+ sample_idx,
+ X_train.shape,
+ y_train.shape,
+ X_test.shape,
+ y_test.shape,
+ )
+ )
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+Repeat #0, fold #0, samples 0: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #1, samples 0: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #2, samples 0: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #3, samples 0: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #4, samples 0: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #5, samples 0: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #6, samples 0: X_train.shape: (2877, 36), y_train.shape (2877,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #7, samples 0: X_train.shape: (2877, 36), y_train.shape (2877,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #8, samples 0: X_train.shape: (2877, 36), y_train.shape (2877,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #9, samples 0: X_train.shape: (2877, 36), y_train.shape (2877,), X_test.shape (319, 36), y_test.shape (319,)
+```
+:::
+
+And also versions with multiple repeats:
+
+``` default
+task_id = 1767
+task = openml.tasks.get_task(task_id)
+X, y = task.get_X_and_y(dataset_format="dataframe")
+n_repeats, n_folds, n_samples = task.get_split_dimensions()
+print(
+ "Task {}: number of repeats: {}, number of folds: {}, number of samples {}.".format(
+ task_id,
+ n_repeats,
+ n_folds,
+ n_samples,
+ )
+)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+Task 1767: number of repeats: 5, number of folds: 2, number of samples 1.
+```
+:::
+
+And then again perform the aforementioned iteration over all splits:
+
+``` default
+for repeat_idx in range(n_repeats):
+ for fold_idx in range(n_folds):
+ for sample_idx in range(n_samples):
+ train_indices, test_indices = task.get_train_test_split_indices(
+ repeat=repeat_idx,
+ fold=fold_idx,
+ sample=sample_idx,
+ )
+ X_train = X.iloc[train_indices]
+ y_train = y.iloc[train_indices]
+ X_test = X.iloc[test_indices]
+ y_test = y.iloc[test_indices]
+
+ print(
+ "Repeat #{}, fold #{}, samples {}: X_train.shape: {}, "
+ "y_train.shape {}, X_test.shape {}, y_test.shape {}".format(
+ repeat_idx,
+ fold_idx,
+ sample_idx,
+ X_train.shape,
+ y_train.shape,
+ X_test.shape,
+ y_test.shape,
+ )
+ )
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+Repeat #0, fold #0, samples 0: X_train.shape: (1598, 36), y_train.shape (1598,), X_test.shape (1598, 36), y_test.shape (1598,)
+Repeat #0, fold #1, samples 0: X_train.shape: (1598, 36), y_train.shape (1598,), X_test.shape (1598, 36), y_test.shape (1598,)
+Repeat #1, fold #0, samples 0: X_train.shape: (1598, 36), y_train.shape (1598,), X_test.shape (1598, 36), y_test.shape (1598,)
+Repeat #1, fold #1, samples 0: X_train.shape: (1598, 36), y_train.shape (1598,), X_test.shape (1598, 36), y_test.shape (1598,)
+Repeat #2, fold #0, samples 0: X_train.shape: (1598, 36), y_train.shape (1598,), X_test.shape (1598, 36), y_test.shape (1598,)
+Repeat #2, fold #1, samples 0: X_train.shape: (1598, 36), y_train.shape (1598,), X_test.shape (1598, 36), y_test.shape (1598,)
+Repeat #3, fold #0, samples 0: X_train.shape: (1598, 36), y_train.shape (1598,), X_test.shape (1598, 36), y_test.shape (1598,)
+Repeat #3, fold #1, samples 0: X_train.shape: (1598, 36), y_train.shape (1598,), X_test.shape (1598, 36), y_test.shape (1598,)
+Repeat #4, fold #0, samples 0: X_train.shape: (1598, 36), y_train.shape (1598,), X_test.shape (1598, 36), y_test.shape (1598,)
+Repeat #4, fold #1, samples 0: X_train.shape: (1598, 36), y_train.shape (1598,), X_test.shape (1598, 36), y_test.shape (1598,)
+```
+:::
+
+And finally a task based on learning curves:
+
+``` default
+task_id = 1702
+task = openml.tasks.get_task(task_id)
+X, y = task.get_X_and_y(dataset_format="dataframe")
+n_repeats, n_folds, n_samples = task.get_split_dimensions()
+print(
+ "Task {}: number of repeats: {}, number of folds: {}, number of samples {}.".format(
+ task_id,
+ n_repeats,
+ n_folds,
+ n_samples,
+ )
+)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+Task 1702: number of repeats: 1, number of folds: 10, number of samples 12.
+```
+:::
+
+And then again perform the aforementioned iteration over all splits:
+
+``` default
+for repeat_idx in range(n_repeats):
+ for fold_idx in range(n_folds):
+ for sample_idx in range(n_samples):
+ train_indices, test_indices = task.get_train_test_split_indices(
+ repeat=repeat_idx,
+ fold=fold_idx,
+ sample=sample_idx,
+ )
+ X_train = X.iloc[train_indices]
+ y_train = y.iloc[train_indices]
+ X_test = X.iloc[test_indices]
+ y_test = y.iloc[test_indices]
+
+ print(
+ "Repeat #{}, fold #{}, samples {}: X_train.shape: {}, "
+ "y_train.shape {}, X_test.shape {}, y_test.shape {}".format(
+ repeat_idx,
+ fold_idx,
+ sample_idx,
+ X_train.shape,
+ y_train.shape,
+ X_test.shape,
+ y_test.shape,
+ )
+ )
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+Repeat #0, fold #0, samples 0: X_train.shape: (64, 36), y_train.shape (64,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #0, samples 1: X_train.shape: (91, 36), y_train.shape (91,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #0, samples 2: X_train.shape: (128, 36), y_train.shape (128,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #0, samples 3: X_train.shape: (181, 36), y_train.shape (181,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #0, samples 4: X_train.shape: (256, 36), y_train.shape (256,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #0, samples 5: X_train.shape: (362, 36), y_train.shape (362,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #0, samples 6: X_train.shape: (512, 36), y_train.shape (512,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #0, samples 7: X_train.shape: (724, 36), y_train.shape (724,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #0, samples 8: X_train.shape: (1024, 36), y_train.shape (1024,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #0, samples 9: X_train.shape: (1448, 36), y_train.shape (1448,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #0, samples 10: X_train.shape: (2048, 36), y_train.shape (2048,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #0, samples 11: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #1, samples 0: X_train.shape: (64, 36), y_train.shape (64,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #1, samples 1: X_train.shape: (91, 36), y_train.shape (91,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #1, samples 2: X_train.shape: (128, 36), y_train.shape (128,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #1, samples 3: X_train.shape: (181, 36), y_train.shape (181,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #1, samples 4: X_train.shape: (256, 36), y_train.shape (256,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #1, samples 5: X_train.shape: (362, 36), y_train.shape (362,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #1, samples 6: X_train.shape: (512, 36), y_train.shape (512,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #1, samples 7: X_train.shape: (724, 36), y_train.shape (724,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #1, samples 8: X_train.shape: (1024, 36), y_train.shape (1024,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #1, samples 9: X_train.shape: (1448, 36), y_train.shape (1448,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #1, samples 10: X_train.shape: (2048, 36), y_train.shape (2048,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #1, samples 11: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #2, samples 0: X_train.shape: (64, 36), y_train.shape (64,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #2, samples 1: X_train.shape: (91, 36), y_train.shape (91,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #2, samples 2: X_train.shape: (128, 36), y_train.shape (128,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #2, samples 3: X_train.shape: (181, 36), y_train.shape (181,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #2, samples 4: X_train.shape: (256, 36), y_train.shape (256,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #2, samples 5: X_train.shape: (362, 36), y_train.shape (362,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #2, samples 6: X_train.shape: (512, 36), y_train.shape (512,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #2, samples 7: X_train.shape: (724, 36), y_train.shape (724,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #2, samples 8: X_train.shape: (1024, 36), y_train.shape (1024,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #2, samples 9: X_train.shape: (1448, 36), y_train.shape (1448,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #2, samples 10: X_train.shape: (2048, 36), y_train.shape (2048,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #2, samples 11: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #3, samples 0: X_train.shape: (64, 36), y_train.shape (64,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #3, samples 1: X_train.shape: (91, 36), y_train.shape (91,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #3, samples 2: X_train.shape: (128, 36), y_train.shape (128,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #3, samples 3: X_train.shape: (181, 36), y_train.shape (181,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #3, samples 4: X_train.shape: (256, 36), y_train.shape (256,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #3, samples 5: X_train.shape: (362, 36), y_train.shape (362,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #3, samples 6: X_train.shape: (512, 36), y_train.shape (512,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #3, samples 7: X_train.shape: (724, 36), y_train.shape (724,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #3, samples 8: X_train.shape: (1024, 36), y_train.shape (1024,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #3, samples 9: X_train.shape: (1448, 36), y_train.shape (1448,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #3, samples 10: X_train.shape: (2048, 36), y_train.shape (2048,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #3, samples 11: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #4, samples 0: X_train.shape: (64, 36), y_train.shape (64,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #4, samples 1: X_train.shape: (91, 36), y_train.shape (91,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #4, samples 2: X_train.shape: (128, 36), y_train.shape (128,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #4, samples 3: X_train.shape: (181, 36), y_train.shape (181,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #4, samples 4: X_train.shape: (256, 36), y_train.shape (256,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #4, samples 5: X_train.shape: (362, 36), y_train.shape (362,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #4, samples 6: X_train.shape: (512, 36), y_train.shape (512,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #4, samples 7: X_train.shape: (724, 36), y_train.shape (724,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #4, samples 8: X_train.shape: (1024, 36), y_train.shape (1024,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #4, samples 9: X_train.shape: (1448, 36), y_train.shape (1448,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #4, samples 10: X_train.shape: (2048, 36), y_train.shape (2048,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #4, samples 11: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #5, samples 0: X_train.shape: (64, 36), y_train.shape (64,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #5, samples 1: X_train.shape: (91, 36), y_train.shape (91,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #5, samples 2: X_train.shape: (128, 36), y_train.shape (128,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #5, samples 3: X_train.shape: (181, 36), y_train.shape (181,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #5, samples 4: X_train.shape: (256, 36), y_train.shape (256,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #5, samples 5: X_train.shape: (362, 36), y_train.shape (362,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #5, samples 6: X_train.shape: (512, 36), y_train.shape (512,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #5, samples 7: X_train.shape: (724, 36), y_train.shape (724,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #5, samples 8: X_train.shape: (1024, 36), y_train.shape (1024,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #5, samples 9: X_train.shape: (1448, 36), y_train.shape (1448,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #5, samples 10: X_train.shape: (2048, 36), y_train.shape (2048,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #5, samples 11: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
+Repeat #0, fold #6, samples 0: X_train.shape: (64, 36), y_train.shape (64,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #6, samples 1: X_train.shape: (91, 36), y_train.shape (91,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #6, samples 2: X_train.shape: (128, 36), y_train.shape (128,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #6, samples 3: X_train.shape: (181, 36), y_train.shape (181,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #6, samples 4: X_train.shape: (256, 36), y_train.shape (256,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #6, samples 5: X_train.shape: (362, 36), y_train.shape (362,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #6, samples 6: X_train.shape: (512, 36), y_train.shape (512,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #6, samples 7: X_train.shape: (724, 36), y_train.shape (724,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #6, samples 8: X_train.shape: (1024, 36), y_train.shape (1024,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #6, samples 9: X_train.shape: (1448, 36), y_train.shape (1448,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #6, samples 10: X_train.shape: (2048, 36), y_train.shape (2048,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #6, samples 11: X_train.shape: (2877, 36), y_train.shape (2877,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #7, samples 0: X_train.shape: (64, 36), y_train.shape (64,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #7, samples 1: X_train.shape: (91, 36), y_train.shape (91,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #7, samples 2: X_train.shape: (128, 36), y_train.shape (128,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #7, samples 3: X_train.shape: (181, 36), y_train.shape (181,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #7, samples 4: X_train.shape: (256, 36), y_train.shape (256,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #7, samples 5: X_train.shape: (362, 36), y_train.shape (362,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #7, samples 6: X_train.shape: (512, 36), y_train.shape (512,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #7, samples 7: X_train.shape: (724, 36), y_train.shape (724,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #7, samples 8: X_train.shape: (1024, 36), y_train.shape (1024,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #7, samples 9: X_train.shape: (1448, 36), y_train.shape (1448,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #7, samples 10: X_train.shape: (2048, 36), y_train.shape (2048,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #7, samples 11: X_train.shape: (2877, 36), y_train.shape (2877,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #8, samples 0: X_train.shape: (64, 36), y_train.shape (64,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #8, samples 1: X_train.shape: (91, 36), y_train.shape (91,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #8, samples 2: X_train.shape: (128, 36), y_train.shape (128,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #8, samples 3: X_train.shape: (181, 36), y_train.shape (181,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #8, samples 4: X_train.shape: (256, 36), y_train.shape (256,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #8, samples 5: X_train.shape: (362, 36), y_train.shape (362,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #8, samples 6: X_train.shape: (512, 36), y_train.shape (512,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #8, samples 7: X_train.shape: (724, 36), y_train.shape (724,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #8, samples 8: X_train.shape: (1024, 36), y_train.shape (1024,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #8, samples 9: X_train.shape: (1448, 36), y_train.shape (1448,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #8, samples 10: X_train.shape: (2048, 36), y_train.shape (2048,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #8, samples 11: X_train.shape: (2877, 36), y_train.shape (2877,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #9, samples 0: X_train.shape: (64, 36), y_train.shape (64,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #9, samples 1: X_train.shape: (91, 36), y_train.shape (91,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #9, samples 2: X_train.shape: (128, 36), y_train.shape (128,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #9, samples 3: X_train.shape: (181, 36), y_train.shape (181,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #9, samples 4: X_train.shape: (256, 36), y_train.shape (256,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #9, samples 5: X_train.shape: (362, 36), y_train.shape (362,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #9, samples 6: X_train.shape: (512, 36), y_train.shape (512,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #9, samples 7: X_train.shape: (724, 36), y_train.shape (724,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #9, samples 8: X_train.shape: (1024, 36), y_train.shape (1024,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #9, samples 9: X_train.shape: (1448, 36), y_train.shape (1448,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #9, samples 10: X_train.shape: (2048, 36), y_train.shape (2048,), X_test.shape (319, 36), y_test.shape (319,)
+Repeat #0, fold #9, samples 11: X_train.shape: (2877, 36), y_train.shape (2877,), X_test.shape (319, 36), y_test.shape (319,)
+```
+:::
+
+::: rst-class
+sphx-glr-timing
+
+**Total running time of the script:** ( 0 minutes 3.602 seconds)
+:::
+
+::::::: {#sphx_glr_download_examples_30_extended_task_manual_iteration_tutorial.py}
+:::::: only
+html
+
+::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
+::: {.container .sphx-glr-download .sphx-glr-download-python}
+`Download Python source code: task_manual_iteration_tutorial.py <task_manual_iteration_tutorial.py>`{.interpreted-text
+role="download"}
+:::
+
+::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
+`Download Jupyter notebook: task_manual_iteration_tutorial.ipynb <task_manual_iteration_tutorial.ipynb>`{.interpreted-text
+role="download"}
+:::
+:::::
+::::::
+:::::::
+
+:::: only
+html
+
+::: rst-class
+sphx-glr-signature
+
+[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
+:::
+::::
diff --git a/docs/examples/30_extended/tasks_tutorial.rst.md b/docs/examples/30_extended/tasks_tutorial.rst.md
new file mode 100644
index 000000000..c1a07f965
--- /dev/null
+++ b/docs/examples/30_extended/tasks_tutorial.rst.md
@@ -0,0 +1,1292 @@
+::::: only
+html
+
+:::: {.note .sphx-glr-download-link-note}
+::: title
+Note
+:::
+
+`Go to the end <sphx_glr_download_examples_30_extended_tasks_tutorial.py>`{.interpreted-text
+role="ref"} to download the full example code
+::::
+:::::
+
+::: rst-class
+sphx-glr-example-title
+:::
+
+# Tasks {#sphx_glr_examples_30_extended_tasks_tutorial.py}
+
+A tutorial on how to list and download tasks.
+
+``` default
+# License: BSD 3-Clause
+
+import openml
+from openml.tasks import TaskType
+import pandas as pd
+```
+
+Tasks are identified by IDs and can be accessed in two different ways:
+
+1. In a list providing basic information on all tasks available on
+ OpenML. This function will not download the actual tasks, but will
+ instead download meta data that can be used to filter the tasks and
+ retrieve a set of IDs. We can filter this list, for example, we can
+ only list tasks having a special tag or only tasks for a specific
+ target such as *supervised classification*.
+2. A single task by its ID. It contains all meta information, the
+ target metric, the splits and an iterator which can be used to
+ access the splits in a useful manner.
+
+## Listing tasks
+
+We will start by simply listing only *supervised classification* tasks.
+**openml.tasks.list_tasks()** returns a dictionary of dictionaries by
+default, but we request a [pandas
+dataframe](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html)
+instead to have better visualization capabilities and easier access:
+
+``` default
+tasks = openml.tasks.list_tasks(
+ task_type=TaskType.SUPERVISED_CLASSIFICATION, output_format="dataframe"
+)
+print(tasks.columns)
+print(f"First 5 of {len(tasks)} tasks:")
+print(tasks.head())
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+Index(['tid', 'ttid', 'did', 'name', 'task_type', 'status',
+ 'estimation_procedure', 'evaluation_measures', 'source_data',
+ 'target_feature', 'MajorityClassSize', 'MaxNominalAttDistinctValues',
+ 'MinorityClassSize', 'NumberOfClasses', 'NumberOfFeatures',
+ 'NumberOfInstances', 'NumberOfInstancesWithMissingValues',
+ 'NumberOfMissingValues', 'NumberOfNumericFeatures',
+ 'NumberOfSymbolicFeatures', 'cost_matrix'],
+ dtype='object')
+First 5 of 4239 tasks:
+ tid ... cost_matrix
+2 2 ... NaN
+3 3 ... NaN
+4 4 ... NaN
+5 5 ... NaN
+6 6 ... NaN
+
+[5 rows x 21 columns]
+```
+:::
+
+We can filter the list of tasks to only contain datasets with more than
+500 samples, but less than 1000 samples:
+
+``` default
+filtered_tasks = tasks.query("NumberOfInstances > 500 and NumberOfInstances < 1000")
+print(list(filtered_tasks.index))
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+[2, 11, 15, 29, 37, 41, 49, 53, 232, 241, 245, 259, 267, 271, 279, 283, 1766, 1775, 1779, 1793, 1801, 1805, 1813, 1817, 1882, 1891, 1895, 1909, 1917, 1921, 1929, 1933, 1945, 1952, 1956, 1967, 1973, 1977, 1983, 1987, 2079, 2125, 2944, 3022, 3034, 3047, 3049, 3053, 3054, 3055, 3484, 3486, 3492, 3493, 3494, 3512, 3518, 3520, 3521, 3529, 3535, 3549, 3560, 3561, 3583, 3623, 3636, 3640, 3660, 3690, 3691, 3692, 3704, 3706, 3718, 3794, 3803, 3810, 3812, 3813, 3814, 3817, 3833, 3852, 3853, 3857, 3860, 3867, 3877, 3879, 3886, 3913, 3971, 3979, 3992, 3999, 4189, 4191, 4197, 4198, 4199, 4217, 4223, 4225, 4226, 4234, 4240, 4254, 4265, 4266, 4288, 4328, 4341, 4345, 4365, 4395, 4396, 4397, 4409, 4411, 4423, 4499, 4508, 4515, 4517, 4518, 4519, 4522, 4538, 4557, 4558, 4562, 4565, 4572, 4582, 4584, 4591, 4618, 4676, 4684, 4697, 4704, 7286, 7307, 7543, 7548, 7558, 9904, 9905, 9946, 9950, 9971, 9980, 9989, 9990, 10097, 10098, 10101, 12738, 12739, 14954, 14968, 145682, 145800, 145804, 145805, 145825, 145836, 145839, 145848, 145878, 145882, 145914, 145917, 145952, 145959, 145970, 145976, 145978, 146062, 146064, 146065, 146066, 146069, 146092, 146156, 146216, 146219, 146231, 146574, 146576, 146577, 146578, 146583, 146587, 146588, 146593, 146596, 146597, 146600, 146818, 146819, 166859, 166875, 166882, 166884, 166893, 166905, 166906, 166907, 166913, 166915, 166919, 166947, 166953, 166956, 166957, 166958, 166959, 166960, 166967, 166976, 166977, 166978, 166980, 166983, 166988, 166989, 166992, 167016, 167020, 167031, 167037, 167062, 167067, 167068, 167095, 167096, 167100, 167104, 167106, 167151, 167154, 167160, 167163, 167167, 167168, 167171, 167173, 167174, 167175, 167180, 167184, 167187, 167194, 167198, 168300, 168783, 168819, 168820, 168821, 168822, 168823, 168824, 168825, 168907, 189786, 189859, 189899, 189900, 189932, 189937, 189941, 190136, 190138, 190139, 190140, 190143, 190146, 233090, 233094, 233109, 233115, 233171, 233206, 359953, 359954, 359955, 360857, 360865, 360868, 360869, 360951, 360953, 360964, 361107, 361109, 361146, 361147, 361148, 361149, 361150, 361151, 361152, 361153, 361154, 361155, 361156, 361157, 361158, 361159, 361160, 361161, 361163, 361164, 361165, 361166, 361167, 361168, 361169, 361170, 361171, 361172, 361173, 361174, 361175, 361176, 361183, 361185, 361190, 361305, 361338, 361340, 361412, 361415, 361424, 361426, 361432, 361433, 361434, 361436, 361437, 361440, 361442, 361443, 361457, 361463, 361483, 361486, 361492, 361495, 361498, 361499, 361502, 361504, 361506, 361507, 361512, 361517, 361522, 361529, 361531, 361540, 361557, 361572, 361624, 361650, 361660]
+```
+:::
+
+``` default
+# Number of tasks
+print(len(filtered_tasks))
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+368
+```
+:::
+
+Then, we can further restrict the tasks to all have the same resampling
+strategy:
+
+``` default
+filtered_tasks = filtered_tasks.query('estimation_procedure == "10-fold Crossvalidation"')
+print(list(filtered_tasks.index))
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+[2, 11, 15, 29, 37, 41, 49, 53, 2079, 3022, 3484, 3486, 3492, 3493, 3494, 3512, 3518, 3520, 3521, 3529, 3535, 3549, 3560, 3561, 3583, 3623, 3636, 3640, 3660, 3690, 3691, 3692, 3704, 3706, 3718, 3794, 3803, 3810, 3812, 3813, 3814, 3817, 3833, 3852, 3853, 3857, 3860, 3867, 3877, 3879, 3886, 3913, 3971, 3979, 3992, 3999, 7286, 7307, 7548, 7558, 9904, 9905, 9946, 9950, 9971, 9980, 9989, 9990, 10097, 10098, 10101, 14954, 14968, 145682, 145800, 145804, 145805, 145825, 145836, 145839, 145848, 145878, 145882, 145914, 145917, 145952, 145959, 145970, 145976, 145978, 146062, 146064, 146065, 146066, 146069, 146092, 146156, 146216, 146219, 146231, 146818, 146819, 168300, 168907, 189932, 189937, 189941, 190136, 190138, 190139, 190140, 190143, 190146, 233171, 359953, 359954, 359955, 360857, 360865, 360868, 360869, 360951, 360953, 360964, 361107, 361109, 361146, 361147, 361148, 361149, 361150, 361151, 361152, 361153, 361154, 361155, 361156, 361157, 361158, 361159, 361160, 361161, 361163, 361164, 361165, 361166, 361167, 361168, 361169, 361170, 361171, 361172, 361173, 361174, 361175, 361176, 361183, 361185, 361190, 361305, 361338, 361340]
+```
+:::
+
+``` default
+# Number of tasks
+print(len(filtered_tasks))
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+162
+```
+:::
+
+Resampling strategies can be found on the [OpenML
+Website](https://www.openml.org/search?type=measure&q=estimation%20procedure).
+
+Similar to listing tasks by task type, we can list tasks by tags:
+
+``` default
+tasks = openml.tasks.list_tasks(tag="OpenML100", output_format="dataframe")
+print(f"First 5 of {len(tasks)} tasks:")
+print(tasks.head())
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+First 5 of 91 tasks:
+ tid ... NumberOfSymbolicFeatures
+3 3 ... 37
+6 6 ... 1
+11 11 ... 1
+12 12 ... 1
+14 14 ... 1
+
+[5 rows x 19 columns]
+```
+:::
+
+Furthermore, we can list tasks based on the dataset id:
+
+``` default
+tasks = openml.tasks.list_tasks(data_id=1471, output_format="dataframe")
+print(f"First 5 of {len(tasks)} tasks:")
+print(tasks.head())
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+First 5 of 25 tasks:
+ tid ... number_samples
+9983 9983 ... NaN
+14951 14951 ... NaN
+56483 56483 ... NaN
+56484 56484 ... NaN
+56485 56485 ... NaN
+
+[5 rows x 23 columns]
+```
+:::
+
+In addition, a size limit and an offset can be applied both separately
+and simultaneously:
+
+``` default
+tasks = openml.tasks.list_tasks(size=10, offset=50, output_format="dataframe")
+print(tasks)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+tid ... number_samples
+59 59 ... NaN
+60 60 ... NaN
+62 62 ... 9
+63 63 ... 12
+64 64 ... 1
+65 65 ... 7
+66 66 ... 5
+67 67 ... 6
+68 68 ... 5
+69 69 ... 4
+
+[10 rows x 21 columns]
+```
+:::
+
+**OpenML 100** is a curated list of 100 tasks to start using OpenML.
+They are all supervised classification tasks with more than 500
+instances and less than 50000 instances per task. To make things easier,
+the tasks do not contain highly unbalanced data and sparse data.
+However, the tasks include missing values and categorical features. You
+can find out more about the *OpenML 100* on [the OpenML benchmarking
+page](https://docs.openml.org/benchmark/).
+
+Finally, it is also possible to list all tasks on OpenML with:
+
+``` default
+tasks = openml.tasks.list_tasks(output_format="dataframe")
+print(len(tasks))
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+47087
+```
+:::
+
+### Exercise
+
+Search for the tasks on the \'eeg-eye-state\' dataset.
+
+``` default
+tasks.query('name=="eeg-eye-state"')
+```
+
+<div class="output_subarea output_html rendered_html output_result">
+<div>
+<style scoped>
+ .dataframe tbody tr th:only-of-type {
+ vertical-align: middle;
+ }
+
+ .dataframe tbody tr th {
+ vertical-align: top;
+ }
+
+ .dataframe thead th {
+ text-align: right;
+ }
+</style>
+<table border="1" class="dataframe">
+ <thead>
+ <tr style="text-align: right;">
+ <th></th>
+ <th>tid</th>
+ <th>ttid</th>
+ <th>did</th>
+ <th>name</th>
+ <th>task_type</th>
+ <th>status</th>
+ <th>estimation_procedure</th>
+ <th>evaluation_measures</th>
+ <th>source_data</th>
+ <th>target_feature</th>
+ <th>MajorityClassSize</th>
+ <th>MaxNominalAttDistinctValues</th>
+ <th>MinorityClassSize</th>
+ <th>NumberOfClasses</th>
+ <th>NumberOfFeatures</th>
+ <th>NumberOfInstances</th>
+ <th>NumberOfInstancesWithMissingValues</th>
+ <th>NumberOfMissingValues</th>
+ <th>NumberOfNumericFeatures</th>
+ <th>NumberOfSymbolicFeatures</th>
+ <th>number_samples</th>
+ <th>cost_matrix</th>
+ <th>source_data_labeled</th>
+ <th>target_feature_event</th>
+ <th>target_feature_left</th>
+ <th>target_feature_right</th>
+ <th>quality_measure</th>
+ <th>target_value</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <th>3511</th>
+ <td>9983</td>
+ <td>TaskType.SUPERVISED_CLASSIFICATION</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Supervised Classification</td>
+ <td>active</td>
+ <td>10-fold Crossvalidation</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>Class</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ </tr>
+ <tr>
+ <th>4692</th>
+ <td>14951</td>
+ <td>TaskType.SUPERVISED_CLASSIFICATION</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Supervised Classification</td>
+ <td>active</td>
+ <td>10-fold Crossvalidation</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>Class</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ </tr>
+ <tr>
+ <th>8032</th>
+ <td>56483</td>
+ <td>TaskType.SUBGROUP_DISCOVERY</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Subgroup Discovery</td>
+ <td>active</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>Class</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>Cortana Quality</td>
+ <td>1</td>
+ </tr>
+ <tr>
+ <th>8033</th>
+ <td>56484</td>
+ <td>TaskType.SUBGROUP_DISCOVERY</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Subgroup Discovery</td>
+ <td>active</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>Class</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>Information gain</td>
+ <td>1</td>
+ </tr>
+ <tr>
+ <th>8034</th>
+ <td>56485</td>
+ <td>TaskType.SUBGROUP_DISCOVERY</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Subgroup Discovery</td>
+ <td>active</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>Class</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>Binomial test</td>
+ <td>1</td>
+ </tr>
+ <tr>
+ <th>8035</th>
+ <td>56486</td>
+ <td>TaskType.SUBGROUP_DISCOVERY</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Subgroup Discovery</td>
+ <td>active</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>Class</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>Jaccard</td>
+ <td>1</td>
+ </tr>
+ <tr>
+ <th>8036</th>
+ <td>56487</td>
+ <td>TaskType.SUBGROUP_DISCOVERY</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Subgroup Discovery</td>
+ <td>active</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>Class</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>Cortana Quality</td>
+ <td>2</td>
+ </tr>
+ <tr>
+ <th>8037</th>
+ <td>56488</td>
+ <td>TaskType.SUBGROUP_DISCOVERY</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Subgroup Discovery</td>
+ <td>active</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>Class</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>Information gain</td>
+ <td>2</td>
+ </tr>
+ <tr>
+ <th>8038</th>
+ <td>56489</td>
+ <td>TaskType.SUBGROUP_DISCOVERY</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Subgroup Discovery</td>
+ <td>active</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>Class</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>Binomial test</td>
+ <td>2</td>
+ </tr>
+ <tr>
+ <th>8039</th>
+ <td>56490</td>
+ <td>TaskType.SUBGROUP_DISCOVERY</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Subgroup Discovery</td>
+ <td>active</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>Class</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>Jaccard</td>
+ <td>2</td>
+ </tr>
+ <tr>
+ <th>8581</th>
+ <td>75219</td>
+ <td>TaskType.SUPERVISED_CLASSIFICATION</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Supervised Classification</td>
+ <td>active</td>
+ <td>33% Holdout set</td>
+ <td>predictive_accuracy</td>
+ <td>1471</td>
+ <td>Class</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ </tr>
+ <tr>
+ <th>8663</th>
+ <td>125901</td>
+ <td>TaskType.LEARNING_CURVE</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Learning Curve</td>
+ <td>active</td>
+ <td>10-fold Learning Curve</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>Class</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>17</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ </tr>
+ <tr>
+ <th>9864</th>
+ <td>127251</td>
+ <td>TaskType.CLUSTERING</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Clustering</td>
+ <td>active</td>
+ <td>50 times Clustering</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>NaN</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ </tr>
+ <tr>
+ <th>11920</th>
+ <td>146761</td>
+ <td>TaskType.CLUSTERING</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Clustering</td>
+ <td>active</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>Class</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ </tr>
+ <tr>
+ <th>13204</th>
+ <td>148117</td>
+ <td>TaskType.CLUSTERING</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Clustering</td>
+ <td>active</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>NaN</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ </tr>
+ <tr>
+ <th>17309</th>
+ <td>170139</td>
+ <td>TaskType.CLUSTERING</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Clustering</td>
+ <td>active</td>
+ <td>50 times Clustering</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>NaN</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ </tr>
+ <tr>
+ <th>20760</th>
+ <td>191646</td>
+ <td>TaskType.CLUSTERING</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Clustering</td>
+ <td>active</td>
+ <td>50 times Clustering</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>NaN</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ </tr>
+ <tr>
+ <th>24446</th>
+ <td>213320</td>
+ <td>TaskType.CLUSTERING</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Clustering</td>
+ <td>active</td>
+ <td>50 times Clustering</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>NaN</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ </tr>
+ <tr>
+ <th>27655</th>
+ <td>234435</td>
+ <td>TaskType.CLUSTERING</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Clustering</td>
+ <td>active</td>
+ <td>50 times Clustering</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>NaN</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ </tr>
+ <tr>
+ <th>31979</th>
+ <td>256739</td>
+ <td>TaskType.CLUSTERING</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Clustering</td>
+ <td>active</td>
+ <td>50 times Clustering</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>NaN</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ </tr>
+ <tr>
+ <th>31989</th>
+ <td>256750</td>
+ <td>TaskType.CLUSTERING</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Clustering</td>
+ <td>active</td>
+ <td>50 times Clustering</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>NaN</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ </tr>
+ <tr>
+ <th>37166</th>
+ <td>297708</td>
+ <td>TaskType.CLUSTERING</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Clustering</td>
+ <td>active</td>
+ <td>50 times Clustering</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>NaN</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ </tr>
+ <tr>
+ <th>40370</th>
+ <td>318835</td>
+ <td>TaskType.CLUSTERING</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Clustering</td>
+ <td>active</td>
+ <td>50 times Clustering</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>NaN</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ </tr>
+ <tr>
+ <th>43587</th>
+ <td>339990</td>
+ <td>TaskType.CLUSTERING</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Clustering</td>
+ <td>active</td>
+ <td>50 times Clustering</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>NaN</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ </tr>
+ <tr>
+ <th>46851</th>
+ <td>361459</td>
+ <td>TaskType.SUPERVISED_CLASSIFICATION</td>
+ <td>1471</td>
+ <td>eeg-eye-state</td>
+ <td>Supervised Classification</td>
+ <td>active</td>
+ <td>5 times 2-fold Crossvalidation</td>
+ <td>NaN</td>
+ <td>1471</td>
+ <td>Class</td>
+ <td>8257.0</td>
+ <td>2.0</td>
+ <td>6723.0</td>
+ <td>2.0</td>
+ <td>15.0</td>
+ <td>14980.0</td>
+ <td>0.0</td>
+ <td>0.0</td>
+ <td>14.0</td>
+ <td>1.0</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ <td>NaN</td>
+ </tr>
+ </tbody>
+</table>
+</div>
+</div>
+<br />
+<br />
+
+## Downloading tasks
+
+We provide two functions to download tasks, one which downloads only a
+single task by its ID, and one which takes a list of IDs and downloads
+all of these tasks:
+
+``` default
+task_id = 31
+task = openml.tasks.get_task(task_id)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
+ warnings.warn(
+/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
+ warnings.warn(
+```
+:::
+
+Properties of the task are stored as member variables:
+
+``` default
+print(task)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+OpenML Classification Task
+==========================
+Task Type Description: https://www.openml.org/tt/TaskType.SUPERVISED_CLASSIFICATION
+Task ID..............: 31
+Task URL.............: https://www.openml.org/t/31
+Estimation Procedure.: crossvalidation
+Target Feature.......: class
+# of Classes.........: 2
+Cost Matrix..........: Available
+```
+:::
+
+And:
+
+``` default
+ids = [2, 1891, 31, 9983]
+tasks = openml.tasks.get_tasks(ids)
+print(tasks[0])
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+OpenML Classification Task
+==========================
+Task Type Description: https://www.openml.org/tt/TaskType.SUPERVISED_CLASSIFICATION
+Task ID..............: 2
+Task URL.............: https://www.openml.org/t/2
+Estimation Procedure.: crossvalidation
+Evaluation Measure...: predictive_accuracy
+Target Feature.......: class
+# of Classes.........: 6
+Cost Matrix..........: Available
+```
+:::
+
+## Creating tasks
+
+You can also create new tasks. Take the following into account:
+
+- You can only create tasks on *active* datasets
+- For now, only the following tasks are supported: classification,
+ regression, clustering, and learning curve analysis.
+- For now, tasks can only be created on a single dataset.
+- The exact same task must not already exist.
+
+Creating a task requires the following input:
+
+- task_type: The task type ID, required (see below). Required.
+- dataset_id: The dataset ID. Required.
+- target_name: The name of the attribute you aim to predict. Optional.
+- estimation_procedure_id : The ID of the estimation procedure used to
+ create train-test splits. Optional.
+- evaluation_measure: The name of the evaluation measure. Optional.
+- Any additional inputs for specific tasks
+
+It is best to leave the evaluation measure open if there is no strong
+prerequisite for a specific measure. OpenML will always compute all
+appropriate measures and you can filter or sort results on your
+favourite measure afterwards. Only add an evaluation measure if
+necessary (e.g. when other measure make no sense), since it will create
+a new task, which scatters results across tasks.
+
+We\'ll use the test server for the rest of this tutorial.
+
+:::: warning
+::: title
+Warning
+:::
+
+.. include:: ../../test_server_usage_warning.txt
+::::
+
+``` default
+openml.config.start_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!
+ warnings.warn(
+```
+:::
+
+### Example
+
+Let\'s create a classification task on a dataset. In this example we
+will do this on the Iris dataset (ID=128 (on test server)). We\'ll use
+10-fold cross-validation (ID=1), and *predictive accuracy* as the
+predefined measure (this can also be left open). If a task with these
+parameters exists, we will get an appropriate exception. If such a task
+doesn\'t exist, a task will be created and the corresponding task_id
+will be returned.
+
+``` default
+try:
+ my_task = openml.tasks.create_task(
+ task_type=TaskType.SUPERVISED_CLASSIFICATION,
+ dataset_id=128,
+ target_name="class",
+ evaluation_measure="predictive_accuracy",
+ estimation_procedure_id=1,
+ )
+ my_task.publish()
+except openml.exceptions.OpenMLServerException as e:
+ # Error code for 'task already exists'
+ if e.code == 614:
+ # Lookup task
+ tasks = openml.tasks.list_tasks(data_id=128, output_format="dataframe")
+ tasks = tasks.query(
+ 'task_type == "Supervised Classification" '
+ 'and estimation_procedure == "10-fold Crossvalidation" '
+ 'and evaluation_measures == "predictive_accuracy"'
+ )
+ task_id = tasks.loc[:, "tid"].values[0]
+ print("Task already exists. Task ID is", task_id)
+
+# reverting to prod server
+openml.config.stop_using_configuration_for_example()
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+/code/openml/tasks/functions.py:112: RuntimeWarning: Could not create task type id for 10 due to error 10 is not a valid TaskType
+ warnings.warn(
+/code/openml/tasks/functions.py:112: RuntimeWarning: Could not create task type id for 11 due to error 11 is not a valid TaskType
+ warnings.warn(
+/code/openml/tasks/functions.py:263: RuntimeWarning: Could not create task type id for 10 due to error 10 is not a valid TaskType
+ warnings.warn(
+Task already exists. Task ID is 1306
+```
+:::
+
+- [Complete list of task
+ types](https://www.openml.org/search?type=task_type).
+- [Complete list of model estimation
+ procedures](https://www.openml.org/search?q=%2520measure_type%3Aestimation_procedure&type=measure).
+- [Complete list of evaluation
+ measures](https://www.openml.org/search?q=measure_type%3Aevaluation_measure&type=measure).
+
+::: rst-class
+sphx-glr-timing
+
+**Total running time of the script:** ( 0 minutes 47.382 seconds)
+:::
+
+::::::: {#sphx_glr_download_examples_30_extended_tasks_tutorial.py}
+:::::: only
+html
+
+::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
+::: {.container .sphx-glr-download .sphx-glr-download-python}
+`Download Python source code: tasks_tutorial.py <tasks_tutorial.py>`{.interpreted-text
+role="download"}
+:::
+
+::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
+`Download Jupyter notebook: tasks_tutorial.ipynb <tasks_tutorial.ipynb>`{.interpreted-text
+role="download"}
+:::
+:::::
+::::::
+:::::::
+
+:::: only
+html
+
+::: rst-class
+sphx-glr-signature
+
+[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
+:::
+::::
diff --git a/docs/examples/40_paper/2015_neurips_feurer_example.rst.md b/docs/examples/40_paper/2015_neurips_feurer_example.rst.md
new file mode 100644
index 000000000..5972b4b00
--- /dev/null
+++ b/docs/examples/40_paper/2015_neurips_feurer_example.rst.md
@@ -0,0 +1,167 @@
+::::: only
+html
+
+:::: {.note .sphx-glr-download-link-note}
+::: title
+Note
+:::
+
+`Go to the end <sphx_glr_download_examples_40_paper_2015_neurips_feurer_example.py>`{.interpreted-text
+role="ref"} to download the full example code
+::::
+:::::
+
+::: rst-class
+sphx-glr-example-title
+:::
+
+# Feurer et al. (2015) {#sphx_glr_examples_40_paper_2015_neurips_feurer_example.py}
+
+A tutorial on how to get the datasets used in the paper introducing
+*Auto-sklearn* by Feurer et al..
+
+Auto-sklearn website: <https://automl.github.io/auto-sklearn/>
+
+## Publication
+
+| Efficient and Robust Automated Machine Learning
+| Matthias Feurer, Aaron Klein, Katharina Eggensperger, Jost
+ Springenberg, Manuel Blum and Frank Hutter
+| In *Advances in Neural Information Processing Systems 28*, 2015
+| Available at
+ <https://papers.nips.cc/paper/5872-efficient-and-robust-automated-machine-learning.pdf>
+
+``` default
+# License: BSD 3-Clause
+
+import pandas as pd
+
+import openml
+```
+
+List of dataset IDs given in the supplementary material of Feurer et
+al.:
+<https://papers.nips.cc/paper/5872-efficient-and-robust-automated-machine-learning-supplemental.zip>
+fmt: off
+
+``` default
+dataset_ids = [
+ 3, 6, 12, 14, 16, 18, 21, 22, 23, 24, 26, 28, 30, 31, 32, 36, 38, 44, 46,
+ 57, 60, 179, 180, 181, 182, 184, 185, 273, 293, 300, 351, 354, 357, 389,
+ 390, 391, 392, 393, 395, 396, 398, 399, 401, 554, 679, 715, 718, 720, 722,
+ 723, 727, 728, 734, 735, 737, 740, 741, 743, 751, 752, 761, 772, 797, 799,
+ 803, 806, 807, 813, 816, 819, 821, 822, 823, 833, 837, 843, 845, 846, 847,
+ 849, 866, 871, 881, 897, 901, 903, 904, 910, 912, 913, 914, 917, 923, 930,
+ 934, 953, 958, 959, 962, 966, 971, 976, 977, 978, 979, 980, 991, 993, 995,
+ 1000, 1002, 1018, 1019, 1020, 1021, 1036, 1040, 1041, 1049, 1050, 1053,
+ 1056, 1067, 1068, 1069, 1111, 1112, 1114, 1116, 1119, 1120, 1128, 1130,
+ 1134, 1138, 1139, 1142, 1146, 1161, 1166,
+]
+# fmt: on
+```
+
+The dataset IDs could be used directly to load the dataset and split the
+data into a training set and a test set. However, to be reproducible, we
+will first obtain the respective tasks from OpenML, which define both
+the target feature and the train/test split.
+
+:::: note
+::: title
+Note
+:::
+
+It is discouraged to work directly on datasets and only provide dataset
+IDs in a paper as this does not allow reproducibility (unclear
+splitting). Please do not use datasets but the respective tasks as basis
+for a paper and publish task IDS. This example is only given to showcase
+the use of OpenML-Python for a published paper and as a warning on how
+not to do it. Please check the [OpenML documentation of
+tasks](https://docs.openml.org/#tasks) if you want to learn more about
+them.
+::::
+
+This lists both active and inactive tasks (because of `status='all'`).
+Unfortunately, this is necessary as some of the datasets contain issues
+found after the publication and became deactivated, which also
+deactivated the tasks on them. More information on active or inactive
+datasets can be found in the [online
+docs](https://docs.openml.org/#dataset-status).
+
+``` default
+tasks = openml.tasks.list_tasks(
+ task_type=openml.tasks.TaskType.SUPERVISED_CLASSIFICATION,
+ status="all",
+ output_format="dataframe",
+)
+
+# Query only those with holdout as the resampling startegy.
+tasks = tasks.query('estimation_procedure == "33% Holdout set"')
+
+task_ids = []
+for did in dataset_ids:
+ tasks_ = list(tasks.query("did == {}".format(did)).tid)
+ if len(tasks_) >= 1: # if there are multiple task, take the one with lowest ID (oldest).
+ task_id = min(tasks_)
+ else:
+ raise ValueError(did)
+
+ # Optional - Check that the task has the same target attribute as the
+ # dataset default target attribute
+ # (disabled for this example as it needs to run fast to be rendered online)
+ # task = openml.tasks.get_task(task_id)
+ # dataset = task.get_dataset()
+ # if task.target_name != dataset.default_target_attribute:
+ # raise ValueError(
+ # (task.target_name, dataset.default_target_attribute)
+ # )
+
+ task_ids.append(task_id)
+
+assert len(task_ids) == 140
+task_ids.sort()
+
+# These are the tasks to work with:
+print(task_ids)
+```
+
+::: rst-class
+sphx-glr-script-out
+
+``` none
+[233, 236, 242, 244, 246, 248, 251, 252, 253, 254, 256, 258, 260, 261, 262, 266, 273, 275, 288, 2117, 2118, 2119, 2120, 2122, 2123, 2350, 3043, 3044, 75090, 75092, 75093, 75098, 75099, 75100, 75103, 75104, 75105, 75106, 75107, 75108, 75111, 75112, 75113, 75114, 75115, 75116, 75117, 75119, 75120, 75121, 75122, 75125, 75126, 75129, 75131, 75133, 75136, 75137, 75138, 75139, 75140, 75142, 75143, 75146, 75147, 75148, 75149, 75150, 75151, 75152, 75153, 75155, 75157, 75159, 75160, 75161, 75162, 75163, 75164, 75165, 75166, 75168, 75169, 75170, 75171, 75172, 75173, 75174, 75175, 75176, 75179, 75180, 75182, 75183, 75184, 75185, 75186, 75188, 75189, 75190, 75191, 75192, 75194, 75195, 75196, 75197, 75198, 75199, 75200, 75201, 75202, 75203, 75204, 75205, 75206, 75207, 75208, 75209, 75210, 75212, 75213, 75216, 75218, 75220, 75222, 75224, 75226, 75228, 75229, 75233, 75238, 75240, 75244, 75245, 75246, 75247, 75248, 75249, 75251, 190400]
+```
+:::
+
+::: rst-class
+sphx-glr-timing
+
+**Total running time of the script:** ( 0 minutes 1.752 seconds)
+:::
+
+::::::: {#sphx_glr_download_examples_40_paper_2015_neurips_feurer_example.py}
+:::::: only
+html
+
+::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
+::: {.container .sphx-glr-download .sphx-glr-download-python}
+`Download Python source code: 2015_neurips_feurer_example.py <2015_neurips_feurer_example.py>`{.interpreted-text
+role="download"}
+:::
+
+::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
+`Download Jupyter notebook: 2015_neurips_feurer_example.ipynb <2015_neurips_feurer_example.ipynb>`{.interpreted-text
+role="download"}
+:::
+:::::
+::::::
+:::::::
+
+:::: only
+html
+
+::: rst-class
+sphx-glr-signature
+
+[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
+:::
+::::
diff --git a/docs/examples/40_paper/2018_ida_strang_example.rst.md b/docs/examples/40_paper/2018_ida_strang_example.rst.md
new file mode 100644
index 000000000..42f5cb09a
--- /dev/null
+++ b/docs/examples/40_paper/2018_ida_strang_example.rst.md
@@ -0,0 +1,191 @@
+::::: only
+html
+
+:::: {.note .sphx-glr-download-link-note}
+::: title
+Note
+:::
+
+`Go to the end <sphx_glr_download_examples_40_paper_2018_ida_strang_example.py>`{.interpreted-text
+role="ref"} to download the full example code
+::::
+:::::
+
+::: rst-class
+sphx-glr-example-title
+:::
+
+# Strang et al. (2018) {#sphx_glr_examples_40_paper_2018_ida_strang_example.py}
+
+A tutorial on how to reproduce the analysis conducted for *Don\'t Rule
+Out Simple Models Prematurely: A Large Scale Benchmark Comparing Linear
+and Non-linear Classifiers in OpenML*.
+
+## Publication
+
+| Don\'t Rule Out Simple Models Prematurely: A Large Scale Benchmark
+ Comparing Linear and Non-linear Classifiers in OpenML
+| Benjamin Strang, Peter van der Putten, Jan N. van Rijn and Frank
+ Hutter
+| In *Advances in Intelligent Data Analysis XVII 17th International
+ Symposium*, 2018
+| Available at
+ <https://link.springer.com/chapter/10.1007%2F978-3-030-01768-2_25>
+
+``` default
+# License: BSD 3-Clause
+
+import matplotlib.pyplot as plt
+import openml
+import pandas as pd
+```
+
+A basic step for each data-mining or machine learning task is to
+determine which model to choose based on the problem and the data at
+hand. In this work we investigate when non-linear classifiers outperform
+linear classifiers by means of a large scale experiment.
+
+The paper is accompanied with a study object, containing all relevant
+tasks and runs (`study_id=123`). The paper features three experiment
+classes: Support Vector Machines (SVM), Neural Networks (NN) and
+Decision Trees (DT). This example demonstrates how to reproduce the
+plots, comparing two classifiers given the OpenML flow ids. Note that
+this allows us to reproduce the SVM and NN experiment, but not the DT
+experiment, as this requires a bit more effort to distinguish the same
+flow with different hyperparameter values.
+
+``` default
+study_id = 123
+# for comparing svms: flow_ids = [7754, 7756]
+# for comparing nns: flow_ids = [7722, 7729]
+# for comparing dts: flow_ids = [7725], differentiate on hyper-parameter value
+classifier_family = "SVM"
+flow_ids = [7754, 7756]
+measure = "predictive_accuracy"
+meta_features = ["NumberOfInstances", "NumberOfFeatures"]
+class_values = ["non-linear better", "linear better", "equal"]
+
+# Downloads all evaluation records related to this study
+evaluations = openml.evaluations.list_evaluations(
+ measure, size=None, flows=flow_ids, study=study_id, output_format="dataframe"
+)
+# gives us a table with columns data_id, flow1_value, flow2_value
+evaluations = evaluations.pivot(index="data_id", columns="flow_id", values="value").dropna()
+# downloads all data qualities (for scatter plot)
+data_qualities = openml.datasets.list_datasets(
+ data_id=list(evaluations.index.values), output_format="dataframe"
+)
+# removes irrelevant data qualities
+data_qualities = data_qualities[meta_features]
+# makes a join between evaluation table and data qualities table,
+# now we have columns data_id, flow1_value, flow2_value, meta_feature_1,
+# meta_feature_2
+evaluations = evaluations.join(data_qualities, how="inner")
+
+# adds column that indicates the difference between the two classifiers
+evaluations["diff"] = evaluations[flow_ids[0]] - evaluations[flow_ids[1]]
+```
+
+makes the s-plot
+
+``` default
+fig_splot, ax_splot = plt.subplots()
+ax_splot.plot(range(len(evaluations)), sorted(evaluations["diff"]))
+ax_splot.set_title(classifier_family)
+ax_splot.set_xlabel("Dataset (sorted)")
+ax_splot.set_ylabel("difference between linear and non-linear classifier")
+ax_splot.grid(linestyle="--", axis="y")
+plt.show()
+```
+
+::: {.image-sg .sphx-glr-single-img alt="SVM" srcset="/examples/40_paper/images/sphx_glr_2018_ida_strang_example_001.png"}
+/examples/40_paper/images/sphx_glr_2018_ida_strang_example_001.png
+:::
+
+adds column that indicates the difference between the two classifiers,
+needed for the scatter plot
+
+``` default
+def determine_class(val_lin, val_nonlin):
+ if val_lin < val_nonlin:
+ return class_values[0]
+ elif val_nonlin < val_lin:
+ return class_values[1]
+ else:
+ return class_values[2]
+
+
+evaluations["class"] = evaluations.apply(
+ lambda row: determine_class(row[flow_ids[0]], row[flow_ids[1]]), axis=1
+)
+
+# does the plotting and formatting
+fig_scatter, ax_scatter = plt.subplots()
+for class_val in class_values:
+ df_class = evaluations[evaluations["class"] == class_val]
+ plt.scatter(df_class[meta_features[0]], df_class[meta_features[1]], label=class_val)
+ax_scatter.set_title(classifier_family)
+ax_scatter.set_xlabel(meta_features[0])
+ax_scatter.set_ylabel(meta_features[1])
+ax_scatter.legend()
+ax_scatter.set_xscale("log")
+ax_scatter.set_yscale("log")
+plt.show()
+```
+
+::: {.image-sg .sphx-glr-single-img alt="SVM" srcset="/examples/40_paper/images/sphx_glr_2018_ida_strang_example_002.png"}
+/examples/40_paper/images/sphx_glr_2018_ida_strang_example_002.png
+:::
+
+makes a scatter plot where each data point represents the performance of
+the two algorithms on various axis (not in the paper)
+
+``` default
+fig_diagplot, ax_diagplot = plt.subplots()
+ax_diagplot.grid(linestyle="--")
+ax_diagplot.plot([0, 1], ls="-", color="black")
+ax_diagplot.plot([0.2, 1.2], ls="--", color="black")
+ax_diagplot.plot([-0.2, 0.8], ls="--", color="black")
+ax_diagplot.scatter(evaluations[flow_ids[0]], evaluations[flow_ids[1]])
+ax_diagplot.set_xlabel(measure)
+ax_diagplot.set_ylabel(measure)
+plt.show()
+```
+
+::: {.image-sg .sphx-glr-single-img alt="2018 ida strang example" srcset="/examples/40_paper/images/sphx_glr_2018_ida_strang_example_003.png"}
+/examples/40_paper/images/sphx_glr_2018_ida_strang_example_003.png
+:::
+
+::: rst-class
+sphx-glr-timing
+
+**Total running time of the script:** ( 0 minutes 1.507 seconds)
+:::
+
+::::::: {#sphx_glr_download_examples_40_paper_2018_ida_strang_example.py}
+:::::: only
+html
+
+::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
+::: {.container .sphx-glr-download .sphx-glr-download-python}
+`Download Python source code: 2018_ida_strang_example.py <2018_ida_strang_example.py>`{.interpreted-text
+role="download"}
+:::
+
+::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
+`Download Jupyter notebook: 2018_ida_strang_example.ipynb <2018_ida_strang_example.ipynb>`{.interpreted-text
+role="download"}
+:::
+:::::
+::::::
+:::::::
+
+:::: only
+html
+
+::: rst-class
+sphx-glr-signature
+
+[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
+:::
+::::
diff --git a/docs/extensions.md b/docs/extensions.md
new file mode 100644
index 000000000..dbe7b1940
--- /dev/null
+++ b/docs/extensions.md
@@ -0,0 +1,183 @@
+orphan
+
+:
+
+# Extensions
+
+OpenML-Python provides an extension interface to connect other machine
+learning libraries than scikit-learn to OpenML. Please check the
+`api_extensions`{.interpreted-text role="ref"} and use the scikit-learn
+extension in
+`openml.extensions.sklearn.SklearnExtension`{.interpreted-text
+role="class"} as a starting point.
+
+## List of extensions
+
+Here is a list of currently maintained OpenML extensions:
+
+- `openml.extensions.sklearn.SklearnExtension`{.interpreted-text
+ role="class"}
+- [openml-keras](https://github.com/openml/openml-keras)
+- [openml-pytorch](https://github.com/openml/openml-pytorch)
+- [openml-tensorflow (for tensorflow
+ 2+)](https://github.com/openml/openml-tensorflow)
+
+## Connecting new machine learning libraries
+
+### Content of the Library
+
+To leverage support from the community and to tap in the potential of
+OpenML, interfacing with popular machine learning libraries is
+essential. The OpenML-Python package is capable of downloading meta-data
+and results (data, flows, runs), regardless of the library that was used
+to upload it. However, in order to simplify the process of uploading
+flows and runs from a specific library, an additional interface can be
+built. The OpenML-Python team does not have the capacity to develop and
+maintain such interfaces on its own. For this reason, we have built an
+extension interface to allows others to contribute back. Building a
+suitable extension for therefore requires an understanding of the
+current OpenML-Python support.
+
+The
+`sphx_glr_examples_20_basic_simple_flows_and_runs_tutorial.py`{.interpreted-text
+role="ref"} tutorial shows how scikit-learn currently works with
+OpenML-Python as an extension. The *sklearn* extension packaged with the
+[openml-python](https://github.com/openml/openml-python) repository can
+be used as a template/benchmark to build the new extension.
+
+#### API
+
+- The extension scripts must import the [openml]{.title-ref} package
+ and be able to interface with any function from the OpenML-Python
+ `api`{.interpreted-text role="ref"}.
+- The extension has to be defined as a Python class and must inherit
+ from `openml.extensions.Extension`{.interpreted-text role="class"}.
+- This class needs to have all the functions from [class
+ Extension]{.title-ref} overloaded as required.
+- The redefined functions should have adequate and appropriate
+ docstrings. The [Sklearn Extension API
+ :class:\`openml.extensions.sklearn.SklearnExtension.html]{.title-ref}
+ is a good example to follow.
+
+#### Interfacing with OpenML-Python
+
+Once the new extension class has been defined, the openml-python module
+to `openml.extensions.register_extension`{.interpreted-text role="meth"}
+must be called to allow OpenML-Python to interface the new extension.
+
+The following methods should get implemented. Although the documentation
+in the [Extension]{.title-ref} interface should always be leading, here
+we list some additional information and best practices. The [Sklearn
+Extension API
+:class:\`openml.extensions.sklearn.SklearnExtension.html]{.title-ref} is
+a good example to follow. Note that most methods are relatively simple
+and can be implemented in several lines of code.
+
+- General setup (required)
+ - `can_handle_flow`{.interpreted-text role="meth"}: Takes as
+ argument an OpenML flow, and checks whether this can be handled
+ by the current extension. The OpenML database consists of many
+ flows, from various workbenches (e.g., scikit-learn, Weka, mlr).
+ This method is called before a model is being deserialized.
+ Typically, the flow-dependency field is used to check whether
+ the specific library is present, and no unknown libraries are
+ present there.
+ - `can_handle_model`{.interpreted-text role="meth"}: Similar as
+ `can_handle_flow`{.interpreted-text role="meth"}, except that in
+ this case a Python object is given. As such, in many cases, this
+ method can be implemented by checking whether this adheres to a
+ certain base class.
+- Serialization and De-serialization (required)
+ - `flow_to_model`{.interpreted-text role="meth"}: deserializes the
+ OpenML Flow into a model (if the library can indeed handle the
+ flow). This method has an important interplay with
+ `model_to_flow`{.interpreted-text role="meth"}. Running these
+ two methods in succession should result in exactly the same
+ model (or flow). This property can be used for unit testing
+ (e.g., build a model with hyperparameters, make predictions on a
+ task, serialize it to a flow, deserialize it back, make it
+ predict on the same task, and check whether the predictions are
+ exactly the same.) The example in the scikit-learn interface
+ might seem daunting, but note that here some complicated design
+ choices were made, that allow for all sorts of interesting
+ research questions. It is probably good practice to start easy.
+ - `model_to_flow`{.interpreted-text role="meth"}: The inverse of
+ `flow_to_model`{.interpreted-text role="meth"}. Serializes a
+ model into an OpenML Flow. The flow should preserve the class,
+ the library version, and the tunable hyperparameters.
+ - `get_version_information`{.interpreted-text role="meth"}: Return
+ a tuple with the version information of the important libraries.
+ - `create_setup_string`{.interpreted-text role="meth"}: No longer
+ used, and will be deprecated soon.
+- Performing runs (required)
+ - `is_estimator`{.interpreted-text role="meth"}: Gets as input a
+ class, and checks whether it has the status of estimator in the
+ library (typically, whether it has a train method and a predict
+ method).
+ - `seed_model`{.interpreted-text role="meth"}: Sets a random seed
+ to the model.
+ - `_run_model_on_fold`{.interpreted-text role="meth"}: One of the
+ main requirements for a library to generate run objects for the
+ OpenML server. Obtains a train split (with labels) and a test
+ split (without labels) and the goal is to train a model on the
+ train split and return the predictions on the test split. On top
+ of the actual predictions, also the class probabilities should
+ be determined. For classifiers that do not return class
+ probabilities, this can just be the hot-encoded predicted label.
+ The predictions will be evaluated on the OpenML server. Also,
+ additional information can be returned, for example,
+ user-defined measures (such as runtime information, as this can
+ not be inferred on the server). Additionally, information about
+ a hyperparameter optimization trace can be provided.
+ - `obtain_parameter_values`{.interpreted-text role="meth"}:
+ Obtains the hyperparameters of a given model and the current
+ values. Please note that in the case of a hyperparameter
+ optimization procedure (e.g., random search), you only should
+ return the hyperparameters of this procedure (e.g., the
+ hyperparameter grid, budget, etc) and that the chosen model will
+ be inferred from the optimization trace.
+ - `check_if_model_fitted`{.interpreted-text role="meth"}: Check
+ whether the train method of the model has been called (and as
+ such, whether the predict method can be used).
+- Hyperparameter optimization (optional)
+ - `instantiate_model_from_hpo_class`{.interpreted-text
+ role="meth"}: If a given run has recorded the hyperparameter
+ optimization trace, then this method can be used to
+ reinstantiate the model with hyperparameters of a given
+ hyperparameter optimization iteration. Has some similarities
+ with `flow_to_model`{.interpreted-text role="meth"} (as this
+ method also sets the hyperparameters of a model). Note that
+ although this method is required, it is not necessary to
+ implement any logic if hyperparameter optimization is not
+ implemented. Simply raise a [NotImplementedError]{.title-ref}
+ then.
+
+### Hosting the library
+
+Each extension created should be a stand-alone repository, compatible
+with the [OpenML-Python
+repository](https://github.com/openml/openml-python). The extension
+repository should work off-the-shelf with *OpenML-Python* installed.
+
+Create a [public Github
+repo](https://docs.github.com/en/github/getting-started-with-github/create-a-repo)
+with the following directory structure:
+
+ | [repo name]
+ | |-- [extension name]
+ | | |-- __init__.py
+ | | |-- extension.py
+ | | |-- config.py (optionally)
+
+### Recommended
+
+- Test cases to keep the extension up to date with the
+ [openml-python]{.title-ref} upstream changes.
+- Documentation of the extension API, especially if any new
+ functionality added to OpenML-Python\'s extension design.
+- Examples to show how the new extension interfaces and works with
+ OpenML-Python.
+- Create a PR to add the new extension to the OpenML-Python API
+ documentation.
+
+Happy contributing!
diff --git a/docs/generated/openml.datasets.OpenMLDataFeature.rst.md b/docs/generated/openml.datasets.OpenMLDataFeature.rst.md
new file mode 100644
index 000000000..5e049b341
--- /dev/null
+++ b/docs/generated/openml.datasets.OpenMLDataFeature.rst.md
@@ -0,0 +1,13 @@
+orphan
+
+:
+
+# `openml.datasets`{.interpreted-text role="mod"}.OpenMLDataFeature
+
+::: currentmodule
+openml.datasets
+:::
+
+::: autoclass
+OpenMLDataFeature
+:::
diff --git a/docs/generated/openml.datasets.OpenMLDataset.rst.md b/docs/generated/openml.datasets.OpenMLDataset.rst.md
new file mode 100644
index 000000000..63b34cad3
--- /dev/null
+++ b/docs/generated/openml.datasets.OpenMLDataset.rst.md
@@ -0,0 +1,13 @@
+orphan
+
+:
+
+# `openml.datasets`{.interpreted-text role="mod"}.OpenMLDataset
+
+::: currentmodule
+openml.datasets
+:::
+
+::: autoclass
+OpenMLDataset
+:::
diff --git a/docs/generated/openml.datasets.attributes_arff_from_df.rst.md b/docs/generated/openml.datasets.attributes_arff_from_df.rst.md
new file mode 100644
index 000000000..f923a447c
--- /dev/null
+++ b/docs/generated/openml.datasets.attributes_arff_from_df.rst.md
@@ -0,0 +1,11 @@
+# `openml.datasets`{.interpreted-text role="mod"}.attributes_arff_from_df
+
+::: currentmodule
+openml.datasets
+:::
+
+::: autofunction
+attributes_arff_from_df
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.datasets.check_datasets_active.rst.md b/docs/generated/openml.datasets.check_datasets_active.rst.md
new file mode 100644
index 000000000..9b12e9b7c
--- /dev/null
+++ b/docs/generated/openml.datasets.check_datasets_active.rst.md
@@ -0,0 +1,11 @@
+# `openml.datasets`{.interpreted-text role="mod"}.check_datasets_active
+
+::: currentmodule
+openml.datasets
+:::
+
+::: autofunction
+check_datasets_active
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.datasets.create_dataset.rst.md b/docs/generated/openml.datasets.create_dataset.rst.md
new file mode 100644
index 000000000..d0a7d9e13
--- /dev/null
+++ b/docs/generated/openml.datasets.create_dataset.rst.md
@@ -0,0 +1,11 @@
+# `openml.datasets`{.interpreted-text role="mod"}.create_dataset
+
+::: currentmodule
+openml.datasets
+:::
+
+::: autofunction
+create_dataset
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.datasets.delete_dataset.rst.md b/docs/generated/openml.datasets.delete_dataset.rst.md
new file mode 100644
index 000000000..fd90d06c5
--- /dev/null
+++ b/docs/generated/openml.datasets.delete_dataset.rst.md
@@ -0,0 +1,11 @@
+# `openml.datasets`{.interpreted-text role="mod"}.delete_dataset
+
+::: currentmodule
+openml.datasets
+:::
+
+::: autofunction
+delete_dataset
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.datasets.edit_dataset.rst.md b/docs/generated/openml.datasets.edit_dataset.rst.md
new file mode 100644
index 000000000..251a54ba0
--- /dev/null
+++ b/docs/generated/openml.datasets.edit_dataset.rst.md
@@ -0,0 +1,11 @@
+# `openml.datasets`{.interpreted-text role="mod"}.edit_dataset
+
+::: currentmodule
+openml.datasets
+:::
+
+::: autofunction
+edit_dataset
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.datasets.fork_dataset.rst.md b/docs/generated/openml.datasets.fork_dataset.rst.md
new file mode 100644
index 000000000..43e36c964
--- /dev/null
+++ b/docs/generated/openml.datasets.fork_dataset.rst.md
@@ -0,0 +1,11 @@
+# `openml.datasets`{.interpreted-text role="mod"}.fork_dataset
+
+::: currentmodule
+openml.datasets
+:::
+
+::: autofunction
+fork_dataset
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.datasets.get_dataset.rst.md b/docs/generated/openml.datasets.get_dataset.rst.md
new file mode 100644
index 000000000..a8b5001b2
--- /dev/null
+++ b/docs/generated/openml.datasets.get_dataset.rst.md
@@ -0,0 +1,11 @@
+# `openml.datasets`{.interpreted-text role="mod"}.get_dataset
+
+::: currentmodule
+openml.datasets
+:::
+
+::: autofunction
+get_dataset
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.datasets.get_datasets.rst.md b/docs/generated/openml.datasets.get_datasets.rst.md
new file mode 100644
index 000000000..2585288a5
--- /dev/null
+++ b/docs/generated/openml.datasets.get_datasets.rst.md
@@ -0,0 +1,11 @@
+# `openml.datasets`{.interpreted-text role="mod"}.get_datasets
+
+::: currentmodule
+openml.datasets
+:::
+
+::: autofunction
+get_datasets
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.datasets.list_datasets.rst.md b/docs/generated/openml.datasets.list_datasets.rst.md
new file mode 100644
index 000000000..98832708b
--- /dev/null
+++ b/docs/generated/openml.datasets.list_datasets.rst.md
@@ -0,0 +1,11 @@
+# `openml.datasets`{.interpreted-text role="mod"}.list_datasets
+
+::: currentmodule
+openml.datasets
+:::
+
+::: autofunction
+list_datasets
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.datasets.list_qualities.rst.md b/docs/generated/openml.datasets.list_qualities.rst.md
new file mode 100644
index 000000000..659f91410
--- /dev/null
+++ b/docs/generated/openml.datasets.list_qualities.rst.md
@@ -0,0 +1,11 @@
+# `openml.datasets`{.interpreted-text role="mod"}.list_qualities
+
+::: currentmodule
+openml.datasets
+:::
+
+::: autofunction
+list_qualities
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.datasets.status_update.rst.md b/docs/generated/openml.datasets.status_update.rst.md
new file mode 100644
index 000000000..edb8df9a6
--- /dev/null
+++ b/docs/generated/openml.datasets.status_update.rst.md
@@ -0,0 +1,11 @@
+# `openml.datasets`{.interpreted-text role="mod"}.status_update
+
+::: currentmodule
+openml.datasets
+:::
+
+::: autofunction
+status_update
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.evaluations.OpenMLEvaluation.rst.md b/docs/generated/openml.evaluations.OpenMLEvaluation.rst.md
new file mode 100644
index 000000000..e953137eb
--- /dev/null
+++ b/docs/generated/openml.evaluations.OpenMLEvaluation.rst.md
@@ -0,0 +1,13 @@
+orphan
+
+:
+
+# `openml.evaluations`{.interpreted-text role="mod"}.OpenMLEvaluation
+
+::: currentmodule
+openml.evaluations
+:::
+
+::: autoclass
+OpenMLEvaluation
+:::
diff --git a/docs/generated/openml.evaluations.list_evaluation_measures.rst.md b/docs/generated/openml.evaluations.list_evaluation_measures.rst.md
new file mode 100644
index 000000000..81a25dd54
--- /dev/null
+++ b/docs/generated/openml.evaluations.list_evaluation_measures.rst.md
@@ -0,0 +1,11 @@
+# `openml.evaluations`{.interpreted-text role="mod"}.list_evaluation_measures
+
+::: currentmodule
+openml.evaluations
+:::
+
+::: autofunction
+list_evaluation_measures
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.evaluations.list_evaluations.rst.md b/docs/generated/openml.evaluations.list_evaluations.rst.md
new file mode 100644
index 000000000..d826310a9
--- /dev/null
+++ b/docs/generated/openml.evaluations.list_evaluations.rst.md
@@ -0,0 +1,11 @@
+# `openml.evaluations`{.interpreted-text role="mod"}.list_evaluations
+
+::: currentmodule
+openml.evaluations
+:::
+
+::: autofunction
+list_evaluations
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.evaluations.list_evaluations_setups.rst.md b/docs/generated/openml.evaluations.list_evaluations_setups.rst.md
new file mode 100644
index 000000000..a2f37e5a9
--- /dev/null
+++ b/docs/generated/openml.evaluations.list_evaluations_setups.rst.md
@@ -0,0 +1,11 @@
+# `openml.evaluations`{.interpreted-text role="mod"}.list_evaluations_setups
+
+::: currentmodule
+openml.evaluations
+:::
+
+::: autofunction
+list_evaluations_setups
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.extensions.Extension.rst.md b/docs/generated/openml.extensions.Extension.rst.md
new file mode 100644
index 000000000..8ea18ecf4
--- /dev/null
+++ b/docs/generated/openml.extensions.Extension.rst.md
@@ -0,0 +1,13 @@
+orphan
+
+:
+
+# `openml.extensions`{.interpreted-text role="mod"}.Extension
+
+::: currentmodule
+openml.extensions
+:::
+
+::: autoclass
+Extension
+:::
diff --git a/docs/generated/openml.extensions.get_extension_by_flow.rst.md b/docs/generated/openml.extensions.get_extension_by_flow.rst.md
new file mode 100644
index 000000000..f86230751
--- /dev/null
+++ b/docs/generated/openml.extensions.get_extension_by_flow.rst.md
@@ -0,0 +1,11 @@
+# `openml.extensions`{.interpreted-text role="mod"}.get_extension_by_flow
+
+::: currentmodule
+openml.extensions
+:::
+
+::: autofunction
+get_extension_by_flow
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.extensions.get_extension_by_model.rst.md b/docs/generated/openml.extensions.get_extension_by_model.rst.md
new file mode 100644
index 000000000..057a0812e
--- /dev/null
+++ b/docs/generated/openml.extensions.get_extension_by_model.rst.md
@@ -0,0 +1,11 @@
+# `openml.extensions`{.interpreted-text role="mod"}.get_extension_by_model
+
+::: currentmodule
+openml.extensions
+:::
+
+::: autofunction
+get_extension_by_model
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.extensions.register_extension.rst.md b/docs/generated/openml.extensions.register_extension.rst.md
new file mode 100644
index 000000000..8304875c0
--- /dev/null
+++ b/docs/generated/openml.extensions.register_extension.rst.md
@@ -0,0 +1,11 @@
+# `openml.extensions`{.interpreted-text role="mod"}.register_extension
+
+::: currentmodule
+openml.extensions
+:::
+
+::: autofunction
+register_extension
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.extensions.sklearn.SklearnExtension.rst.md b/docs/generated/openml.extensions.sklearn.SklearnExtension.rst.md
new file mode 100644
index 000000000..8cd1c9714
--- /dev/null
+++ b/docs/generated/openml.extensions.sklearn.SklearnExtension.rst.md
@@ -0,0 +1,13 @@
+orphan
+
+:
+
+# `openml.extensions.sklearn`{.interpreted-text role="mod"}.SklearnExtension
+
+::: currentmodule
+openml.extensions.sklearn
+:::
+
+::: autoclass
+SklearnExtension
+:::
diff --git a/docs/generated/openml.flows.OpenMLFlow.rst.md b/docs/generated/openml.flows.OpenMLFlow.rst.md
new file mode 100644
index 000000000..c3ba4f80f
--- /dev/null
+++ b/docs/generated/openml.flows.OpenMLFlow.rst.md
@@ -0,0 +1,13 @@
+orphan
+
+:
+
+# `openml.flows`{.interpreted-text role="mod"}.OpenMLFlow
+
+::: currentmodule
+openml.flows
+:::
+
+::: autoclass
+OpenMLFlow
+:::
diff --git a/docs/generated/openml.flows.assert_flows_equal.rst.md b/docs/generated/openml.flows.assert_flows_equal.rst.md
new file mode 100644
index 000000000..0e55a16ef
--- /dev/null
+++ b/docs/generated/openml.flows.assert_flows_equal.rst.md
@@ -0,0 +1,11 @@
+# `openml.flows`{.interpreted-text role="mod"}.assert_flows_equal
+
+::: currentmodule
+openml.flows
+:::
+
+::: autofunction
+assert_flows_equal
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.flows.delete_flow.rst.md b/docs/generated/openml.flows.delete_flow.rst.md
new file mode 100644
index 000000000..d3abe3503
--- /dev/null
+++ b/docs/generated/openml.flows.delete_flow.rst.md
@@ -0,0 +1,11 @@
+# `openml.flows`{.interpreted-text role="mod"}.delete_flow
+
+::: currentmodule
+openml.flows
+:::
+
+::: autofunction
+delete_flow
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.flows.flow_exists.rst.md b/docs/generated/openml.flows.flow_exists.rst.md
new file mode 100644
index 000000000..bf766b1ac
--- /dev/null
+++ b/docs/generated/openml.flows.flow_exists.rst.md
@@ -0,0 +1,11 @@
+# `openml.flows`{.interpreted-text role="mod"}.flow_exists
+
+::: currentmodule
+openml.flows
+:::
+
+::: autofunction
+flow_exists
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.flows.get_flow.rst.md b/docs/generated/openml.flows.get_flow.rst.md
new file mode 100644
index 000000000..10b800c80
--- /dev/null
+++ b/docs/generated/openml.flows.get_flow.rst.md
@@ -0,0 +1,11 @@
+# `openml.flows`{.interpreted-text role="mod"}.get_flow
+
+::: currentmodule
+openml.flows
+:::
+
+::: autofunction
+get_flow
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.flows.list_flows.rst.md b/docs/generated/openml.flows.list_flows.rst.md
new file mode 100644
index 000000000..d557b7029
--- /dev/null
+++ b/docs/generated/openml.flows.list_flows.rst.md
@@ -0,0 +1,11 @@
+# `openml.flows`{.interpreted-text role="mod"}.list_flows
+
+::: currentmodule
+openml.flows
+:::
+
+::: autofunction
+list_flows
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.runs.OpenMLRun.rst.md b/docs/generated/openml.runs.OpenMLRun.rst.md
new file mode 100644
index 000000000..36230c66c
--- /dev/null
+++ b/docs/generated/openml.runs.OpenMLRun.rst.md
@@ -0,0 +1,13 @@
+orphan
+
+:
+
+# `openml.runs`{.interpreted-text role="mod"}.OpenMLRun
+
+::: currentmodule
+openml.runs
+:::
+
+::: autoclass
+OpenMLRun
+:::
diff --git a/docs/generated/openml.runs.delete_run.rst.md b/docs/generated/openml.runs.delete_run.rst.md
new file mode 100644
index 000000000..7a1603c1b
--- /dev/null
+++ b/docs/generated/openml.runs.delete_run.rst.md
@@ -0,0 +1,11 @@
+# `openml.runs`{.interpreted-text role="mod"}.delete_run
+
+::: currentmodule
+openml.runs
+:::
+
+::: autofunction
+delete_run
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.runs.get_run.rst.md b/docs/generated/openml.runs.get_run.rst.md
new file mode 100644
index 000000000..8957d4dc8
--- /dev/null
+++ b/docs/generated/openml.runs.get_run.rst.md
@@ -0,0 +1,11 @@
+# `openml.runs`{.interpreted-text role="mod"}.get_run
+
+::: currentmodule
+openml.runs
+:::
+
+::: autofunction
+get_run
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.runs.get_run_trace.rst.md b/docs/generated/openml.runs.get_run_trace.rst.md
new file mode 100644
index 000000000..67f4a31f6
--- /dev/null
+++ b/docs/generated/openml.runs.get_run_trace.rst.md
@@ -0,0 +1,11 @@
+# `openml.runs`{.interpreted-text role="mod"}.get_run_trace
+
+::: currentmodule
+openml.runs
+:::
+
+::: autofunction
+get_run_trace
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.runs.get_runs.rst.md b/docs/generated/openml.runs.get_runs.rst.md
new file mode 100644
index 000000000..7150a72f5
--- /dev/null
+++ b/docs/generated/openml.runs.get_runs.rst.md
@@ -0,0 +1,11 @@
+# `openml.runs`{.interpreted-text role="mod"}.get_runs
+
+::: currentmodule
+openml.runs
+:::
+
+::: autofunction
+get_runs
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.runs.initialize_model_from_run.rst.md b/docs/generated/openml.runs.initialize_model_from_run.rst.md
new file mode 100644
index 000000000..6eed93268
--- /dev/null
+++ b/docs/generated/openml.runs.initialize_model_from_run.rst.md
@@ -0,0 +1,11 @@
+# `openml.runs`{.interpreted-text role="mod"}.initialize_model_from_run
+
+::: currentmodule
+openml.runs
+:::
+
+::: autofunction
+initialize_model_from_run
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.runs.initialize_model_from_trace.rst.md b/docs/generated/openml.runs.initialize_model_from_trace.rst.md
new file mode 100644
index 000000000..0edd6f439
--- /dev/null
+++ b/docs/generated/openml.runs.initialize_model_from_trace.rst.md
@@ -0,0 +1,11 @@
+# `openml.runs`{.interpreted-text role="mod"}.initialize_model_from_trace
+
+::: currentmodule
+openml.runs
+:::
+
+::: autofunction
+initialize_model_from_trace
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.runs.list_runs.rst.md b/docs/generated/openml.runs.list_runs.rst.md
new file mode 100644
index 000000000..51a556b62
--- /dev/null
+++ b/docs/generated/openml.runs.list_runs.rst.md
@@ -0,0 +1,11 @@
+# `openml.runs`{.interpreted-text role="mod"}.list_runs
+
+::: currentmodule
+openml.runs
+:::
+
+::: autofunction
+list_runs
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.runs.run_exists.rst.md b/docs/generated/openml.runs.run_exists.rst.md
new file mode 100644
index 000000000..9d2b8b9e7
--- /dev/null
+++ b/docs/generated/openml.runs.run_exists.rst.md
@@ -0,0 +1,11 @@
+# `openml.runs`{.interpreted-text role="mod"}.run_exists
+
+::: currentmodule
+openml.runs
+:::
+
+::: autofunction
+run_exists
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.runs.run_flow_on_task.rst.md b/docs/generated/openml.runs.run_flow_on_task.rst.md
new file mode 100644
index 000000000..081b550a8
--- /dev/null
+++ b/docs/generated/openml.runs.run_flow_on_task.rst.md
@@ -0,0 +1,11 @@
+# `openml.runs`{.interpreted-text role="mod"}.run_flow_on_task
+
+::: currentmodule
+openml.runs
+:::
+
+::: autofunction
+run_flow_on_task
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.runs.run_model_on_task.rst.md b/docs/generated/openml.runs.run_model_on_task.rst.md
new file mode 100644
index 000000000..bda77fc1f
--- /dev/null
+++ b/docs/generated/openml.runs.run_model_on_task.rst.md
@@ -0,0 +1,11 @@
+# `openml.runs`{.interpreted-text role="mod"}.run_model_on_task
+
+::: currentmodule
+openml.runs
+:::
+
+::: autofunction
+run_model_on_task
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.setups.OpenMLParameter.rst.md b/docs/generated/openml.setups.OpenMLParameter.rst.md
new file mode 100644
index 000000000..6c5468b1a
--- /dev/null
+++ b/docs/generated/openml.setups.OpenMLParameter.rst.md
@@ -0,0 +1,13 @@
+orphan
+
+:
+
+# `openml.setups`{.interpreted-text role="mod"}.OpenMLParameter
+
+::: currentmodule
+openml.setups
+:::
+
+::: autoclass
+OpenMLParameter
+:::
diff --git a/docs/generated/openml.setups.OpenMLSetup.rst.md b/docs/generated/openml.setups.OpenMLSetup.rst.md
new file mode 100644
index 000000000..6d372fac0
--- /dev/null
+++ b/docs/generated/openml.setups.OpenMLSetup.rst.md
@@ -0,0 +1,13 @@
+orphan
+
+:
+
+# `openml.setups`{.interpreted-text role="mod"}.OpenMLSetup
+
+::: currentmodule
+openml.setups
+:::
+
+::: autoclass
+OpenMLSetup
+:::
diff --git a/docs/generated/openml.setups.get_setup.rst.md b/docs/generated/openml.setups.get_setup.rst.md
new file mode 100644
index 000000000..bb65d1edf
--- /dev/null
+++ b/docs/generated/openml.setups.get_setup.rst.md
@@ -0,0 +1,11 @@
+# `openml.setups`{.interpreted-text role="mod"}.get_setup
+
+::: currentmodule
+openml.setups
+:::
+
+::: autofunction
+get_setup
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.setups.initialize_model.rst.md b/docs/generated/openml.setups.initialize_model.rst.md
new file mode 100644
index 000000000..4dd66f52c
--- /dev/null
+++ b/docs/generated/openml.setups.initialize_model.rst.md
@@ -0,0 +1,11 @@
+# `openml.setups`{.interpreted-text role="mod"}.initialize_model
+
+::: currentmodule
+openml.setups
+:::
+
+::: autofunction
+initialize_model
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.setups.list_setups.rst.md b/docs/generated/openml.setups.list_setups.rst.md
new file mode 100644
index 000000000..4be3cebe0
--- /dev/null
+++ b/docs/generated/openml.setups.list_setups.rst.md
@@ -0,0 +1,11 @@
+# `openml.setups`{.interpreted-text role="mod"}.list_setups
+
+::: currentmodule
+openml.setups
+:::
+
+::: autofunction
+list_setups
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.setups.setup_exists.rst.md b/docs/generated/openml.setups.setup_exists.rst.md
new file mode 100644
index 000000000..1027524e4
--- /dev/null
+++ b/docs/generated/openml.setups.setup_exists.rst.md
@@ -0,0 +1,11 @@
+# `openml.setups`{.interpreted-text role="mod"}.setup_exists
+
+::: currentmodule
+openml.setups
+:::
+
+::: autofunction
+setup_exists
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.study.OpenMLBenchmarkSuite.rst.md b/docs/generated/openml.study.OpenMLBenchmarkSuite.rst.md
new file mode 100644
index 000000000..2240023ce
--- /dev/null
+++ b/docs/generated/openml.study.OpenMLBenchmarkSuite.rst.md
@@ -0,0 +1,13 @@
+orphan
+
+:
+
+# `openml.study`{.interpreted-text role="mod"}.OpenMLBenchmarkSuite
+
+::: currentmodule
+openml.study
+:::
+
+::: autoclass
+OpenMLBenchmarkSuite
+:::
diff --git a/docs/generated/openml.study.OpenMLStudy.rst.md b/docs/generated/openml.study.OpenMLStudy.rst.md
new file mode 100644
index 000000000..3f9fb44dc
--- /dev/null
+++ b/docs/generated/openml.study.OpenMLStudy.rst.md
@@ -0,0 +1,13 @@
+orphan
+
+:
+
+# `openml.study`{.interpreted-text role="mod"}.OpenMLStudy
+
+::: currentmodule
+openml.study
+:::
+
+::: autoclass
+OpenMLStudy
+:::
diff --git a/docs/generated/openml.study.attach_to_study.rst.md b/docs/generated/openml.study.attach_to_study.rst.md
new file mode 100644
index 000000000..87d02d70c
--- /dev/null
+++ b/docs/generated/openml.study.attach_to_study.rst.md
@@ -0,0 +1,11 @@
+# `openml.study`{.interpreted-text role="mod"}.attach_to_study
+
+::: currentmodule
+openml.study
+:::
+
+::: autofunction
+attach_to_study
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.study.attach_to_suite.rst.md b/docs/generated/openml.study.attach_to_suite.rst.md
new file mode 100644
index 000000000..752f0c438
--- /dev/null
+++ b/docs/generated/openml.study.attach_to_suite.rst.md
@@ -0,0 +1,11 @@
+# `openml.study`{.interpreted-text role="mod"}.attach_to_suite
+
+::: currentmodule
+openml.study
+:::
+
+::: autofunction
+attach_to_suite
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.study.create_benchmark_suite.rst.md b/docs/generated/openml.study.create_benchmark_suite.rst.md
new file mode 100644
index 000000000..2c2b22c6a
--- /dev/null
+++ b/docs/generated/openml.study.create_benchmark_suite.rst.md
@@ -0,0 +1,11 @@
+# `openml.study`{.interpreted-text role="mod"}.create_benchmark_suite
+
+::: currentmodule
+openml.study
+:::
+
+::: autofunction
+create_benchmark_suite
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.study.create_study.rst.md b/docs/generated/openml.study.create_study.rst.md
new file mode 100644
index 000000000..dc7ed4595
--- /dev/null
+++ b/docs/generated/openml.study.create_study.rst.md
@@ -0,0 +1,11 @@
+# `openml.study`{.interpreted-text role="mod"}.create_study
+
+::: currentmodule
+openml.study
+:::
+
+::: autofunction
+create_study
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.study.delete_study.rst.md b/docs/generated/openml.study.delete_study.rst.md
new file mode 100644
index 000000000..c1fed242b
--- /dev/null
+++ b/docs/generated/openml.study.delete_study.rst.md
@@ -0,0 +1,11 @@
+# `openml.study`{.interpreted-text role="mod"}.delete_study
+
+::: currentmodule
+openml.study
+:::
+
+::: autofunction
+delete_study
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.study.delete_suite.rst.md b/docs/generated/openml.study.delete_suite.rst.md
new file mode 100644
index 000000000..f65e310a7
--- /dev/null
+++ b/docs/generated/openml.study.delete_suite.rst.md
@@ -0,0 +1,11 @@
+# `openml.study`{.interpreted-text role="mod"}.delete_suite
+
+::: currentmodule
+openml.study
+:::
+
+::: autofunction
+delete_suite
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.study.detach_from_study.rst.md b/docs/generated/openml.study.detach_from_study.rst.md
new file mode 100644
index 000000000..25b4a4e62
--- /dev/null
+++ b/docs/generated/openml.study.detach_from_study.rst.md
@@ -0,0 +1,11 @@
+# `openml.study`{.interpreted-text role="mod"}.detach_from_study
+
+::: currentmodule
+openml.study
+:::
+
+::: autofunction
+detach_from_study
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.study.detach_from_suite.rst.md b/docs/generated/openml.study.detach_from_suite.rst.md
new file mode 100644
index 000000000..44f6751ae
--- /dev/null
+++ b/docs/generated/openml.study.detach_from_suite.rst.md
@@ -0,0 +1,11 @@
+# `openml.study`{.interpreted-text role="mod"}.detach_from_suite
+
+::: currentmodule
+openml.study
+:::
+
+::: autofunction
+detach_from_suite
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.study.get_study.rst.md b/docs/generated/openml.study.get_study.rst.md
new file mode 100644
index 000000000..34e385e7e
--- /dev/null
+++ b/docs/generated/openml.study.get_study.rst.md
@@ -0,0 +1,11 @@
+# `openml.study`{.interpreted-text role="mod"}.get_study
+
+::: currentmodule
+openml.study
+:::
+
+::: autofunction
+get_study
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.study.get_suite.rst.md b/docs/generated/openml.study.get_suite.rst.md
new file mode 100644
index 000000000..a3c0a6008
--- /dev/null
+++ b/docs/generated/openml.study.get_suite.rst.md
@@ -0,0 +1,11 @@
+# `openml.study`{.interpreted-text role="mod"}.get_suite
+
+::: currentmodule
+openml.study
+:::
+
+::: autofunction
+get_suite
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.study.list_studies.rst.md b/docs/generated/openml.study.list_studies.rst.md
new file mode 100644
index 000000000..f98fd0410
--- /dev/null
+++ b/docs/generated/openml.study.list_studies.rst.md
@@ -0,0 +1,11 @@
+# `openml.study`{.interpreted-text role="mod"}.list_studies
+
+::: currentmodule
+openml.study
+:::
+
+::: autofunction
+list_studies
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.study.list_suites.rst.md b/docs/generated/openml.study.list_suites.rst.md
new file mode 100644
index 000000000..13ee56751
--- /dev/null
+++ b/docs/generated/openml.study.list_suites.rst.md
@@ -0,0 +1,11 @@
+# `openml.study`{.interpreted-text role="mod"}.list_suites
+
+::: currentmodule
+openml.study
+:::
+
+::: autofunction
+list_suites
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.study.update_study_status.rst.md b/docs/generated/openml.study.update_study_status.rst.md
new file mode 100644
index 000000000..19c4a01a1
--- /dev/null
+++ b/docs/generated/openml.study.update_study_status.rst.md
@@ -0,0 +1,11 @@
+# `openml.study`{.interpreted-text role="mod"}.update_study_status
+
+::: currentmodule
+openml.study
+:::
+
+::: autofunction
+update_study_status
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.study.update_suite_status.rst.md b/docs/generated/openml.study.update_suite_status.rst.md
new file mode 100644
index 000000000..36698c599
--- /dev/null
+++ b/docs/generated/openml.study.update_suite_status.rst.md
@@ -0,0 +1,11 @@
+# `openml.study`{.interpreted-text role="mod"}.update_suite_status
+
+::: currentmodule
+openml.study
+:::
+
+::: autofunction
+update_suite_status
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.tasks.OpenMLClassificationTask.rst.md b/docs/generated/openml.tasks.OpenMLClassificationTask.rst.md
new file mode 100644
index 000000000..4dd7ad30b
--- /dev/null
+++ b/docs/generated/openml.tasks.OpenMLClassificationTask.rst.md
@@ -0,0 +1,13 @@
+orphan
+
+:
+
+# `openml.tasks`{.interpreted-text role="mod"}.OpenMLClassificationTask
+
+::: currentmodule
+openml.tasks
+:::
+
+::: autoclass
+OpenMLClassificationTask
+:::
diff --git a/docs/generated/openml.tasks.OpenMLClusteringTask.rst.md b/docs/generated/openml.tasks.OpenMLClusteringTask.rst.md
new file mode 100644
index 000000000..98d21cec1
--- /dev/null
+++ b/docs/generated/openml.tasks.OpenMLClusteringTask.rst.md
@@ -0,0 +1,13 @@
+orphan
+
+:
+
+# `openml.tasks`{.interpreted-text role="mod"}.OpenMLClusteringTask
+
+::: currentmodule
+openml.tasks
+:::
+
+::: autoclass
+OpenMLClusteringTask
+:::
diff --git a/docs/generated/openml.tasks.OpenMLLearningCurveTask.rst.md b/docs/generated/openml.tasks.OpenMLLearningCurveTask.rst.md
new file mode 100644
index 000000000..bf7e68d57
--- /dev/null
+++ b/docs/generated/openml.tasks.OpenMLLearningCurveTask.rst.md
@@ -0,0 +1,13 @@
+orphan
+
+:
+
+# `openml.tasks`{.interpreted-text role="mod"}.OpenMLLearningCurveTask
+
+::: currentmodule
+openml.tasks
+:::
+
+::: autoclass
+OpenMLLearningCurveTask
+:::
diff --git a/docs/generated/openml.tasks.OpenMLRegressionTask.rst.md b/docs/generated/openml.tasks.OpenMLRegressionTask.rst.md
new file mode 100644
index 000000000..edea0fa16
--- /dev/null
+++ b/docs/generated/openml.tasks.OpenMLRegressionTask.rst.md
@@ -0,0 +1,13 @@
+orphan
+
+:
+
+# `openml.tasks`{.interpreted-text role="mod"}.OpenMLRegressionTask
+
+::: currentmodule
+openml.tasks
+:::
+
+::: autoclass
+OpenMLRegressionTask
+:::
diff --git a/docs/generated/openml.tasks.OpenMLSplit.rst.md b/docs/generated/openml.tasks.OpenMLSplit.rst.md
new file mode 100644
index 000000000..893fddb49
--- /dev/null
+++ b/docs/generated/openml.tasks.OpenMLSplit.rst.md
@@ -0,0 +1,13 @@
+orphan
+
+:
+
+# `openml.tasks`{.interpreted-text role="mod"}.OpenMLSplit
+
+::: currentmodule
+openml.tasks
+:::
+
+::: autoclass
+OpenMLSplit
+:::
diff --git a/docs/generated/openml.tasks.OpenMLSupervisedTask.rst.md b/docs/generated/openml.tasks.OpenMLSupervisedTask.rst.md
new file mode 100644
index 000000000..c8e327eea
--- /dev/null
+++ b/docs/generated/openml.tasks.OpenMLSupervisedTask.rst.md
@@ -0,0 +1,13 @@
+orphan
+
+:
+
+# `openml.tasks`{.interpreted-text role="mod"}.OpenMLSupervisedTask
+
+::: currentmodule
+openml.tasks
+:::
+
+::: autoclass
+OpenMLSupervisedTask
+:::
diff --git a/docs/generated/openml.tasks.OpenMLTask.rst.md b/docs/generated/openml.tasks.OpenMLTask.rst.md
new file mode 100644
index 000000000..467633e83
--- /dev/null
+++ b/docs/generated/openml.tasks.OpenMLTask.rst.md
@@ -0,0 +1,13 @@
+orphan
+
+:
+
+# `openml.tasks`{.interpreted-text role="mod"}.OpenMLTask
+
+::: currentmodule
+openml.tasks
+:::
+
+::: autoclass
+OpenMLTask
+:::
diff --git a/docs/generated/openml.tasks.TaskType.rst.md b/docs/generated/openml.tasks.TaskType.rst.md
new file mode 100644
index 000000000..5692454db
--- /dev/null
+++ b/docs/generated/openml.tasks.TaskType.rst.md
@@ -0,0 +1,13 @@
+orphan
+
+:
+
+# `openml.tasks`{.interpreted-text role="mod"}.TaskType
+
+::: currentmodule
+openml.tasks
+:::
+
+::: autoclass
+TaskType
+:::
diff --git a/docs/generated/openml.tasks.create_task.rst.md b/docs/generated/openml.tasks.create_task.rst.md
new file mode 100644
index 000000000..c049a4e7c
--- /dev/null
+++ b/docs/generated/openml.tasks.create_task.rst.md
@@ -0,0 +1,11 @@
+# `openml.tasks`{.interpreted-text role="mod"}.create_task
+
+::: currentmodule
+openml.tasks
+:::
+
+::: autofunction
+create_task
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.tasks.delete_task.rst.md b/docs/generated/openml.tasks.delete_task.rst.md
new file mode 100644
index 000000000..eb810dcb8
--- /dev/null
+++ b/docs/generated/openml.tasks.delete_task.rst.md
@@ -0,0 +1,11 @@
+# `openml.tasks`{.interpreted-text role="mod"}.delete_task
+
+::: currentmodule
+openml.tasks
+:::
+
+::: autofunction
+delete_task
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.tasks.get_task.rst.md b/docs/generated/openml.tasks.get_task.rst.md
new file mode 100644
index 000000000..985711a33
--- /dev/null
+++ b/docs/generated/openml.tasks.get_task.rst.md
@@ -0,0 +1,11 @@
+# `openml.tasks`{.interpreted-text role="mod"}.get_task
+
+::: currentmodule
+openml.tasks
+:::
+
+::: autofunction
+get_task
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.tasks.get_tasks.rst.md b/docs/generated/openml.tasks.get_tasks.rst.md
new file mode 100644
index 000000000..396be2131
--- /dev/null
+++ b/docs/generated/openml.tasks.get_tasks.rst.md
@@ -0,0 +1,11 @@
+# `openml.tasks`{.interpreted-text role="mod"}.get_tasks
+
+::: currentmodule
+openml.tasks
+:::
+
+::: autofunction
+get_tasks
+:::
+
+<div class="clearer"></div>
diff --git a/docs/generated/openml.tasks.list_tasks.rst.md b/docs/generated/openml.tasks.list_tasks.rst.md
new file mode 100644
index 000000000..1e24c7b9a
--- /dev/null
+++ b/docs/generated/openml.tasks.list_tasks.rst.md
@@ -0,0 +1,11 @@
+# `openml.tasks`{.interpreted-text role="mod"}.list_tasks
+
+::: currentmodule
+openml.tasks
+:::
+
+::: autofunction
+list_tasks
+:::
+
+<div class="clearer"></div>
diff --git a/docs/index.md b/docs/index.md
new file mode 100644
index 000000000..ddf28196b
--- /dev/null
+++ b/docs/index.md
@@ -0,0 +1,99 @@
+# OpenML
+
+**Collaborative Machine Learning in Python**
+
+Welcome to the documentation of the OpenML Python API, a connector to
+the collaborative machine learning platform
+[OpenML.org](https://www.openml.org). The OpenML Python package allows
+to use datasets and tasks from OpenML together with scikit-learn and
+share the results online.
+
+## Example
+
+```python
+import openml
+from sklearn import impute, tree, pipeline
+
+# Define a scikit-learn classifier or pipeline
+clf = pipeline.Pipeline(
+ steps=[
+ ('imputer', impute.SimpleImputer()),
+ ('estimator', tree.DecisionTreeClassifier())
+ ]
+)
+# Download the OpenML task for the pendigits dataset with 10-fold
+# cross-validation.
+task = openml.tasks.get_task(32)
+# Run the scikit-learn model on the task.
+run = openml.runs.run_model_on_task(clf, task)
+# Publish the experiment on OpenML (optional, requires an API key.
+# You can get your own API key by signing up to OpenML.org)
+run.publish()
+print(f'View the run online: {run.openml_url}')
+```
+
+You can find more examples in our `examples-index`{.interpreted-text
+role="ref"}.
+
+## How to get OpenML for python
+
+You can install the OpenML package via `pip` (we recommend using a virtual environment):
+
+```bash
+python -m pip install openml
+```
+
+For more advanced installation information, please see the
+["Installation"][installation] section.
+
+## Content
+
+- `usage`{.interpreted-text role="ref"}
+- `api`{.interpreted-text role="ref"}
+- `examples-index`{.interpreted-text role="ref"}
+- `extensions`{.interpreted-text role="ref"}
+- `contributing`{.interpreted-text role="ref"}
+- `progress`{.interpreted-text role="ref"}
+
+## Further information
+
+- [OpenML documentation](https://docs.openml.org/)
+- [OpenML client APIs](https://docs.openml.org/APIs/)
+- [OpenML developer guide](https://docs.openml.org/Contributing/)
+- [Contact information](https://www.openml.org/contact)
+- [Citation request](https://www.openml.org/cite)
+- [OpenML blog](https://medium.com/open-machine-learning)
+- [OpenML twitter account](https://twitter.com/open_ml)
+
+## Contributing
+
+Contribution to the OpenML package is highly appreciated. The OpenML
+package currently has a 1/4 position for the development and all help
+possible is needed to extend and maintain the package, create new
+examples and improve the usability. Please see the
+`contributing`{.interpreted-text role="ref"} page for more information.
+
+## Citing OpenML-Python
+
+If you use OpenML-Python in a scientific publication, we would
+appreciate a reference to the following paper:
+
+| Matthias Feurer, Jan N. van Rijn, Arlind Kadra, Pieter Gijsbers,
+ Neeratyoy Mallik, Sahithya Ravi, Andreas Müller, Joaquin Vanschoren,
+ Frank Hutter
+| **OpenML-Python: an extensible Python API for OpenML**
+| Journal of Machine Learning Research, 22(100):1−5, 2021
+| <https://www.jmlr.org/papers/v22/19-920.html>
+
+> Bibtex entry:
+>
+> @article{JMLR:v22:19-920,
+> author = {Matthias Feurer and Jan N. van Rijn and Arlind Kadra and Pieter Gijsbers and Neeratyoy Mallik and Sahithya Ravi and Andreas Müller and Joaquin Vanschoren and Frank Hutter},
+> title = {OpenML-Python: an extensible Python API for OpenML},
+> journal = {Journal of Machine Learning Research},
+> year = {2021},
+> volume = {22},
+> number = {100},
+> pages = {1--5},
+> url = {http://jmlr.org/papers/v22/19-920.html}
+> }
diff --git a/docs/progress.md b/docs/progress.md
new file mode 100644
index 000000000..cc046a55b
--- /dev/null
+++ b/docs/progress.md
@@ -0,0 +1,493 @@
+orphan
+
+:
+
+# Changelog {#progress}
+
+## next
+
+> - MAINT #1340: Add Numpy 2.0 support. Update tests to work with
+> scikit-learn \<= 1.5.
+> - ADD #1342: Add HTTP header to requests to indicate they are from
+> openml-python.
+
+## 0.14.2
+
+> - MAINT #1280: Use the server-provided `parquet_url` instead of
+> `minio_url` to determine the location of the parquet file.
+> - ADD #716: add documentation for remaining attributes of classes
+> and functions.
+> - ADD #1261: more annotations for type hints.
+> - MAINT #1294: update tests to new tag specification.
+> - FIX #1314: Update fetching a bucket from MinIO.
+> - FIX #1315: Make class label retrieval more lenient.
+> - ADD #1316: add feature descriptions ontologies support.
+> - MAINT #1310/#1307: switch to ruff and resolve all mypy errors.
+
+## 0.14.1
+
+> - FIX: Fallback on downloading ARFF when failing to download parquet
+> from MinIO due to a ServerError.
+
+## 0.14.0
+
+**IMPORTANT:** This release paves the way towards a breaking update of
+OpenML-Python. From version 0.15, functions that had the option to
+return a pandas DataFrame will return a pandas DataFrame by default.
+This version (0.14) emits a warning if you still use the old access
+functionality. More concretely:
+
+- In 0.15 we will drop the ability to return dictionaries in listing
+ calls and only provide pandas DataFrames. To disable warnings in
+ 0.14 you have to request a pandas DataFrame (using
+ `output_format="dataframe"`).
+- In 0.15 we will drop the ability to return datasets as numpy arrays
+ and only provide pandas DataFrames. To disable warnings in 0.14 you
+ have to request a pandas DataFrame (using
+ `dataset_format="dataframe"`).
+
+Furthermore, from version 0.15, OpenML-Python will no longer download
+datasets and dataset metadata by default. This version (0.14) emits a
+warning if you don\'t explicitly specifiy the desired behavior.
+
+Please see the pull requests #1258 and #1260 for further information.
+
+- ADD #1081: New flag that allows disabling downloading dataset
+ features.
+- ADD #1132: New flag that forces a redownload of cached data.
+- FIX #1244: Fixes a rare bug where task listing could fail when the
+ server returned invalid data.
+- DOC #1229: Fixes a comment string for the main example.
+- DOC #1241: Fixes a comment in an example.
+- MAINT #1124: Improve naming of helper functions that govern the
+ cache directories.
+- MAINT #1223, #1250: Update tools used in pre-commit to the latest
+ versions (`black==23.30`, `mypy==1.3.0`, `flake8==6.0.0`).
+- MAINT #1253: Update the citation request to the JMLR paper.
+- MAINT #1246: Add a warning that warns the user that checking for
+ duplicate runs on the server cannot be done without an API key.
+
+## 0.13.1
+
+- ADD #1081 #1132: Add additional options for (not) downloading
+ datasets `openml.datasets.get_dataset` and cache management.
+- ADD #1028: Add functions to delete runs, flows, datasets, and tasks
+ (e.g., `openml.datasets.delete_dataset`).
+- ADD #1144: Add locally computed results to the `OpenMLRun` object\'s
+ representation if the run was created locally and not downloaded
+ from the server.
+- ADD #1180: Improve the error message when the checksum of a
+ downloaded dataset does not match the checksum provided by the API.
+- ADD #1201: Make `OpenMLTraceIteration` a dataclass.
+- DOC #1069: Add argument documentation for the `OpenMLRun` class.
+- DOC #1241 #1229 #1231: Minor documentation fixes and resolve
+ documentation examples not working.
+- FIX #1197 #559 #1131: Fix the order of ground truth and predictions
+ in the `OpenMLRun` object and in `format_prediction`.
+- FIX #1198: Support numpy 1.24 and higher.
+- FIX #1216: Allow unknown task types on the server. This is only
+ relevant when new task types are added to the test server.
+- FIX #1223: Fix mypy errors for implicit optional typing.
+- MAINT #1155: Add dependabot github action to automatically update
+ other github actions.
+- MAINT #1199: Obtain pre-commit\'s flake8 from github.com instead of
+ gitlab.com.
+- MAINT #1215: Support latest numpy version.
+- MAINT #1218: Test Python3.6 on Ubuntu 20.04 instead of the latest
+ Ubuntu (which is 22.04).
+- MAINT #1221 #1212 #1206 #1211: Update github actions to the latest
+ versions.
+
+## 0.13.0
+
+> - FIX #1030: `pre-commit` hooks now no longer should issue a
+> warning.
+> - FIX #1058, #1100: Avoid `NoneType` error when printing task
+> without `class_labels` attribute.
+> - FIX #1110: Make arguments to `create_study` and `create_suite`
+> that are defined as optional by the OpenML XSD actually optional.
+> - FIX #1147: `openml.flow.flow_exists` no longer requires an API
+> key.
+> - FIX #1184: Automatically resolve proxies when downloading from
+> minio. Turn this off by setting environment variable
+> `no_proxy="*"`.
+> - MAINT #1088: Do CI for Windows on Github Actions instead of
+> Appveyor.
+> - MAINT #1104: Fix outdated docstring for `list_task`.
+> - MAINT #1146: Update the pre-commit dependencies.
+> - ADD #1103: Add a `predictions` property to OpenMLRun for easy
+> accessibility of prediction data.
+> - ADD #1188: EXPERIMENTAL. Allow downloading all files from a minio
+> bucket with `download_all_files=True` for `get_dataset`.
+
+## 0.12.2
+
+- ADD #1065: Add a `retry_policy` configuration option that determines
+ the frequency and number of times to attempt to retry server
+ requests.
+- ADD #1075: A docker image is now automatically built on a push to
+ develop. It can be used to build docs or run tests in an isolated
+ environment.
+- ADD: You can now avoid downloading \'qualities\' meta-data when
+ downloading a task with the `download_qualities` parameter of
+ `openml.tasks.get_task[s]` functions.
+- DOC: Fixes a few broken links in the documentation.
+- DOC #1061: Improve examples to always show a warning when they
+ switch to the test server.
+- DOC #1067: Improve documentation on the scikit-learn extension
+ interface.
+- DOC #1068: Create dedicated extensions page.
+- FIX #1075: Correctly convert [y]{.title-ref} to a pandas series when
+ downloading sparse data.
+- MAINT: Rename [master]{.title-ref} brach to [ main]{.title-ref}
+ branch.
+- MAINT/DOC: Automatically check for broken external links when
+ building the documentation.
+- MAINT/DOC: Fail documentation building on warnings. This will make
+ the documentation building fail if a reference cannot be found (i.e.
+ an internal link is broken).
+
+## 0.12.1
+
+- ADD #895/#1038: Measure runtimes of scikit-learn runs also for
+ models which are parallelized via the joblib.
+- DOC #1050: Refer to the webpage instead of the XML file in the main
+ example.
+- DOC #1051: Document existing extensions to OpenML-Python besides the
+ shipped scikit-learn extension.
+- FIX #1035: Render class attributes and methods again.
+- ADD #1049: Add a command line tool for configuration openml-python.
+- FIX #1042: Fixes a rare concurrency issue with OpenML-Python and
+ joblib which caused the joblib worker pool to fail.
+- FIX #1053: Fixes a bug which could prevent importing the package in
+ a docker container.
+
+## 0.12.0
+
+- ADD #964: Validate `ignore_attribute`, `default_target_attribute`,
+ `row_id_attribute` are set to attributes that exist on the dataset
+ when calling `create_dataset`.
+- ADD #979: Dataset features and qualities are now also cached in
+ pickle format.
+- ADD #982: Add helper functions for column transformers.
+- ADD #989: `run_model_on_task` will now warn the user the the model
+ passed has already been fitted.
+- ADD #1009 : Give possibility to not download the dataset qualities.
+ The cached version is used even so download attribute is false.
+- ADD #1016: Add scikit-learn 0.24 support.
+- ADD #1020: Add option to parallelize evaluation of tasks with
+ joblib.
+- ADD #1022: Allow minimum version of dependencies to be listed for a
+ flow, use more accurate minimum versions for scikit-learn
+ dependencies.
+- ADD #1023: Add admin-only calls for adding topics to datasets.
+- ADD #1029: Add support for fetching dataset from a minio server in
+ parquet format.
+- ADD #1031: Generally improve runtime measurements, add them for some
+ previously unsupported flows (e.g. BaseSearchCV derived flows).
+- DOC #973 : Change the task used in the welcome page example so it no
+ longer fails using numerical dataset.
+- MAINT #671: Improved the performance of `check_datasets_active` by
+ only querying the given list of datasets in contrast to querying all
+ datasets. Modified the corresponding unit test.
+- MAINT #891: Changed the way that numerical features are stored.
+ Numerical features that range from 0 to 255 are now stored as uint8,
+ which reduces the storage space required as well as storing and
+ loading times.
+- MAINT #975, #988: Add CI through Github Actions.
+- MAINT #977: Allow `short` and `long` scenarios for unit tests.
+ Reduce the workload for some unit tests.
+- MAINT #985, #1000: Improve unit test stability and output
+ readability, and adds load balancing.
+- MAINT #1018: Refactor data loading and storage. Data is now
+ compressed on the first call to [get_data]{.title-ref}.
+- MAINT #1024: Remove flaky decorator for study unit test.
+- FIX #883 #884 #906 #972: Various improvements to the caching system.
+- FIX #980: Speed up `check_datasets_active`.
+- FIX #984: Add a retry mechanism when the server encounters a
+ database issue.
+- FIX #1004: Fixed an issue that prevented installation on some
+ systems (e.g. Ubuntu).
+- FIX #1013: Fixes a bug where `OpenMLRun.setup_string` was not
+ uploaded to the server, prepares for `run_details` being sent from
+ the server.
+- FIX #1021: Fixes an issue that could occur when running unit tests
+ and openml-python was not in PATH.
+- FIX #1037: Fixes a bug where a dataset could not be loaded if a
+ categorical value had listed nan-like as a possible category.
+
+## 0.11.0
+
+- ADD #753: Allows uploading custom flows to OpenML via OpenML-Python.
+- ADD #777: Allows running a flow on pandas dataframes (in addition to
+ numpy arrays).
+- ADD #888: Allow passing a [task_id]{.title-ref} to
+ [run_model_on_task]{.title-ref}.
+- ADD #894: Support caching of datasets using feather format as an
+ option.
+- ADD #929: Add `edit_dataset` and `fork_dataset` to allow editing and
+ forking of uploaded datasets.
+- ADD #866, #943: Add support for scikit-learn\'s
+ [passthrough]{.title-ref} and [drop]{.title-ref} when uploading
+ flows to OpenML.
+- ADD #879: Add support for scikit-learn\'s MLP hyperparameter
+ [layer_sizes]{.title-ref}.
+- ADD #894: Support caching of datasets using feather format as an
+ option.
+- ADD #945: PEP 561 compliance for distributing Type information.
+- DOC #660: Remove nonexistent argument from docstring.
+- DOC #901: The API reference now documents the config file and its
+ options.
+- DOC #912: API reference now shows [create_task]{.title-ref}.
+- DOC #954: Remove TODO text from documentation.
+- DOC #960: document how to upload multiple ignore attributes.
+- FIX #873: Fixes an issue which resulted in incorrect URLs when
+ printing OpenML objects after switching the server.
+- FIX #885: Logger no longer registered by default. Added utility
+ functions to easily register logging to console and file.
+- FIX #890: Correct the scaling of data in the SVM example.
+- MAINT #371: `list_evaluations` default `size` changed from `None` to
+ `10_000`.
+- MAINT #767: Source distribution installation is now unit-tested.
+- MAINT #781: Add pre-commit and automated code formatting with black.
+- MAINT #804: Rename arguments of list_evaluations to indicate they
+ expect lists of ids.
+- MAINT #836: OpenML supports only pandas version 1.0.0 or above.
+- MAINT #865: OpenML no longer bundles test files in the source
+ distribution.
+- MAINT #881: Improve the error message for too-long URIs.
+- MAINT #897: Dropping support for Python 3.5.
+- MAINT #916: Adding support for Python 3.8.
+- MAINT #920: Improve error messages for dataset upload.
+- MAINT #921: Improve hangling of the OpenML server URL in the config
+ file.
+- MAINT #925: Improve error handling and error message when loading
+ datasets.
+- MAINT #928: Restructures the contributing documentation.
+- MAINT #936: Adding support for scikit-learn 0.23.X.
+- MAINT #945: Make OpenML-Python PEP562 compliant.
+- MAINT #951: Converts TaskType class to a TaskType enum.
+
+## 0.10.2
+
+- ADD #857: Adds task type ID to list_runs
+- DOC #862: Added license BSD 3-Clause to each of the source files.
+
+## 0.10.1
+
+- ADD #175: Automatically adds the docstring of scikit-learn objects
+ to flow and its parameters.
+- ADD #737: New evaluation listing call that includes the
+ hyperparameter settings.
+- ADD #744: It is now possible to only issue a warning and not raise
+ an exception if the package versions for a flow are not met when
+ deserializing it.
+- ADD #783: The URL to download the predictions for a run is now
+ stored in the run object.
+- ADD #790: Adds the uploader name and id as new filtering options for
+ `list_evaluations`.
+- ADD #792: New convenience function `openml.flow.get_flow_id`.
+- ADD #861: Debug-level log information now being written to a file in
+ the cache directory (at most 2 MB).
+- DOC #778: Introduces instructions on how to publish an extension to
+ support other libraries than scikit-learn.
+- DOC #785: The examples section is completely restructured into
+ simple simple examples, advanced examples and examples showcasing
+ the use of OpenML-Python to reproduce papers which were done with
+ OpenML-Python.
+- DOC #788: New example on manually iterating through the split of a
+ task.
+- DOC #789: Improve the usage of dataframes in the examples.
+- DOC #791: New example for the paper *Efficient and Robust Automated
+ Machine Learning* by Feurer et al. (2015).
+- DOC #803: New example for the paper *Don't Rule Out Simple Models
+ Prematurely: A Large Scale Benchmark Comparing Linear and Non-linear
+ Classifiers in OpenML* by Benjamin Strang et al. (2018).
+- DOC #808: New example demonstrating basic use cases of a dataset.
+- DOC #810: New example demonstrating the use of benchmarking studies
+ and suites.
+- DOC #832: New example for the paper *Scalable Hyperparameter
+ Transfer Learning* by Valerio Perrone et al. (2019)
+- DOC #834: New example showing how to plot the loss surface for a
+ support vector machine.
+- FIX #305: Do not require the external version in the flow XML when
+ loading an object.
+- FIX #734: Better handling of *\"old\"* flows.
+- FIX #736: Attach a StreamHandler to the openml logger instead of the
+ root logger.
+- FIX #758: Fixes an error which made the client API crash when
+ loading a sparse data with categorical variables.
+- FIX #779: Do not fail on corrupt pickle
+- FIX #782: Assign the study id to the correct class attribute.
+- FIX #819: Automatically convert column names to type string when
+ uploading a dataset.
+- FIX #820: Make `__repr__` work for datasets which do not have an id.
+- MAINT #796: Rename an argument to make the function
+ `list_evaluations` more consistent.
+- MAINT #811: Print the full error message given by the server.
+- MAINT #828: Create base class for OpenML entity classes.
+- MAINT #829: Reduce the number of data conversion warnings.
+- MAINT #831: Warn if there\'s an empty flow description when
+ publishing a flow.
+- MAINT #837: Also print the flow XML if a flow fails to validate.
+- FIX #838: Fix list_evaluations_setups to work when evaluations are
+ not a 100 multiple.
+- FIX #847: Fixes an issue where the client API would crash when
+ trying to download a dataset when there are no qualities available
+ on the server.
+- MAINT #849: Move logic of most different `publish` functions into
+ the base class.
+- MAINt #850: Remove outdated test code.
+
+## 0.10.0
+
+- ADD #737: Add list_evaluations_setups to return hyperparameters
+ along with list of evaluations.
+- FIX #261: Test server is cleared of all files uploaded during unit
+ testing.
+- FIX #447: All files created by unit tests no longer persist in
+ local.
+- FIX #608: Fixing dataset_id referenced before assignment error in
+ get_run function.
+- FIX #447: All files created by unit tests are deleted after the
+ completion of all unit tests.
+- FIX #589: Fixing a bug that did not successfully upload the columns
+ to ignore when creating and publishing a dataset.
+- FIX #608: Fixing dataset_id referenced before assignment error in
+ get_run function.
+- DOC #639: More descriptive documention for function to convert array
+ format.
+- DOC #719: Add documentation on uploading tasks.
+- ADD #687: Adds a function to retrieve the list of evaluation
+ measures available.
+- ADD #695: A function to retrieve all the data quality measures
+ available.
+- ADD #412: Add a function to trim flow names for scikit-learn flows.
+- ADD #715: [list_evaluations]{.title-ref} now has an option to sort
+ evaluations by score (value).
+- ADD #722: Automatic reinstantiation of flow in
+ [run_model_on_task]{.title-ref}. Clearer errors if that\'s not
+ possible.
+- ADD #412: The scikit-learn extension populates the short name field
+ for flows.
+- MAINT #726: Update examples to remove deprecation warnings from
+ scikit-learn
+- MAINT #752: Update OpenML-Python to be compatible with sklearn 0.21
+- ADD #790: Add user ID and name to list_evaluations
+
+## 0.9.0
+
+- ADD #560: OpenML-Python can now handle regression tasks as well.
+- ADD #620, #628, #632, #649, #682: Full support for studies and
+ distinguishes suites from studies.
+- ADD #607: Tasks can now be created and uploaded.
+- ADD #647, #673: Introduced the extension interface. This provides an
+ easy way to create a hook for machine learning packages to perform
+ e.g. automated runs.
+- ADD #548, #646, #676: Support for Pandas DataFrame and
+ SparseDataFrame
+- ADD #662: Results of listing functions can now be returned as
+ pandas.DataFrame.
+- ADD #59: Datasets can now also be retrieved by name.
+- ADD #672: Add timing measurements for runs, when possible.
+- ADD #661: Upload time and error messages now displayed with
+ [list_runs]{.title-ref}.
+- ADD #644: Datasets can now be downloaded \'lazily\', retrieving only
+ metadata at first, and the full dataset only when necessary.
+- ADD #659: Lazy loading of task splits.
+- ADD #516: [run_flow_on_task]{.title-ref} flow uploading is now
+ optional.
+- ADD #680: Adds
+ [openml.config.start_using_configuration_for_example]{.title-ref}
+ (and resp. stop) to easily connect to the test server.
+- ADD #75, #653: Adds a pretty print for objects of the top-level
+ classes.
+- FIX #642: [check_datasets_active]{.title-ref} now correctly also
+ returns active status of deactivated datasets.
+- FIX #304, #636: Allow serialization of numpy datatypes and list of
+ lists of more types (e.g. bools, ints) for flows.
+- FIX #651: Fixed a bug that would prevent openml-python from finding
+ the user\'s config file.
+- FIX #693: OpenML-Python uses liac-arff instead of scipy.io for
+ loading task splits now.
+- DOC #678: Better color scheme for code examples in documentation.
+- DOC #681: Small improvements and removing list of missing functions.
+- DOC #684: Add notice to examples that connect to the test server.
+- DOC #688: Add new example on retrieving evaluations.
+- DOC #691: Update contributing guidelines to use Github draft feature
+ instead of tags in title.
+- DOC #692: All functions are documented now.
+- MAINT #184: Dropping Python2 support.
+- MAINT #596: Fewer dependencies for regular pip install.
+- MAINT #652: Numpy and Scipy are no longer required before
+ installation.
+- MAINT #655: Lazy loading is now preferred in unit tests.
+- MAINT #667: Different tag functions now share code.
+- MAINT #666: More descriptive error message for
+ [TypeError]{.title-ref} in [list_runs]{.title-ref}.
+- MAINT #668: Fix some type hints.
+- MAINT #677: [dataset.get_data]{.title-ref} now has consistent
+ behavior in its return type.
+- MAINT #686: Adds ignore directives for several [mypy]{.title-ref}
+ folders.
+- MAINT #629, #630: Code now adheres to single PEP8 standard.
+
+## 0.8.0
+
+- ADD #440: Improved dataset upload.
+- ADD #545, #583: Allow uploading a dataset from a pandas DataFrame.
+- ADD #528: New functions to update the status of a dataset.
+- ADD #523: Support for scikit-learn 0.20\'s new ColumnTransformer.
+- ADD #459: Enhanced support to store runs on disk prior to uploading
+ them to OpenML.
+- ADD #564: New helpers to access the structure of a flow (and find
+ its subflows).
+- ADD #618: The software will from now on retry to connect to the
+ server if a connection failed. The number of retries can be
+ configured.
+- FIX #538: Support loading clustering tasks.
+- FIX #464: Fixes a bug related to listing functions (returns correct
+ listing size).
+- FIX #580: Listing function now works properly when there are less
+ results than requested.
+- FIX #571: Fixes an issue where tasks could not be downloaded in
+ parallel.
+- FIX #536: Flows can now be printed when the flow name is None.
+- FIX #504: Better support for hierarchical hyperparameters when
+ uploading scikit-learn\'s grid and random search.
+- FIX #569: Less strict checking of flow dependencies when loading
+ flows.
+- FIX #431: Pickle of task splits are no longer cached.
+- DOC #540: More examples for dataset uploading.
+- DOC #554: Remove the doubled progress entry from the docs.
+- MAINT #613: Utilize the latest updates in OpenML evaluation
+ listings.
+- MAINT #482: Cleaner interface for handling search traces.
+- MAINT #557: Continuous integration works for scikit-learn 0.18-0.20.
+- MAINT #542: Continuous integration now runs python3.7 as well.
+- MAINT #535: Continuous integration now enforces PEP8 compliance for
+ new code.
+- MAINT #527: Replace deprecated nose by pytest.
+- MAINT #510: Documentation is now built by travis-ci instead of
+ circle-ci.
+- MAINT: Completely re-designed documentation built on sphinx gallery.
+- MAINT #462: Appveyor CI support.
+- MAINT #477: Improve error handling for issue
+ [#479](https://github.com/openml/openml-python/pull/479): the OpenML
+ connector fails earlier and with a better error message when failing
+ to create a flow from the OpenML description.
+- MAINT #561: Improve documentation on running specific unit tests.
+
+## 0.4.-0.7
+
+There is no changelog for these versions.
+
+## 0.3.0
+
+- Add this changelog
+- 2nd example notebook PyOpenML.ipynb
+- Pagination support for list datasets and list tasks
+
+## Prior
+
+There is no changelog for prior versions.
diff --git a/docs/usage.md b/docs/usage.md
new file mode 100644
index 000000000..6a9a55be3
--- /dev/null
+++ b/docs/usage.md
@@ -0,0 +1,206 @@
+# User Guide
+
+This document will guide you through the most important use cases,
+functions and classes in the OpenML Python API. Throughout this
+document, we will use [pandas](https://pandas.pydata.org/) to format and
+filter tables.
+
+## Installation
+
+The OpenML Python package is a connector to
+[OpenML](https://www.openml.org/). It allows you to use and share
+datasets and tasks, run machine learning algorithms on them and then
+share the results online.
+
+The following tutorial gives a short introduction on how to install and
+set up the OpenML Python connector, followed up by a simple example.
+
+- `sphx_glr_examples_20_basic_introduction_tutorial.py`{.interpreted-text
+ role="ref"}
+
+## Configuration
+
+The configuration file resides in a directory `.config/openml` in the
+home directory of the user and is called config (More specifically, it
+resides in the [configuration directory specified by the XDGB Base
+Directory
+Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html)).
+It consists of `key = value` pairs which are separated by newlines. The
+following keys are defined:
+
+-
+
+ apikey:
+
+ : - required to access the server. The
+ `sphx_glr_examples_20_basic_introduction_tutorial.py`{.interpreted-text
+ role="ref"} describes how to obtain an API key.
+
+-
+
+ server:
+
+ : - default: `http://www.openml.org`. Alternatively, use
+ `test.openml.org` for the test server.
+
+-
+
+ cachedir:
+
+ : - if not given, will default to `~/.openml/cache`
+
+-
+
+ avoid_duplicate_runs:
+
+ : - if set to `True`, when `run_flow_on_task` or similar methods
+ are called a lookup is performed to see if there already
+ exists such a run on the server. If so, download those
+ results instead.
+ - if not given, will default to `True`.
+
+-
+
+ retry_policy:
+
+ : - Defines how to react when the server is unavailable or
+ experiencing high load. It determines both how often to
+ attempt to reconnect and how quickly to do so. Please don\'t
+ use `human` in an automated script that you run more than
+ one instance of, it might increase the time to complete your
+ jobs and that of others.
+ - human (default): For people running openml in interactive
+ fashion. Try only a few times, but in quick succession.
+ - robot: For people using openml in an automated fashion. Keep
+ trying to reconnect for a longer time, quickly increasing
+ the time between retries.
+
+-
+
+ connection_n_retries:
+
+ : - number of connection retries
+ - default depends on retry_policy (5 for `human`, 50 for
+ `robot`)
+
+-
+
+ verbosity:
+
+ : - 0: normal output
+ - 1: info output
+ - 2: debug output
+
+This file is easily configurable by the `openml` command line interface.
+To see where the file is stored, and what its values are, use [openml
+configure none]{.title-ref}. Set any field with `openml configure FIELD`
+or even all fields with just `openml configure`.
+
+## Docker
+
+It is also possible to try out the latest development version of
+`openml-python` with docker:
+
+``` bash
+docker run -it openml/openml-python
+```
+
+See the [openml-python docker
+documentation](https://github.com/openml/openml-python/blob/main/docker/readme.md)
+for more information.
+
+## Key concepts
+
+OpenML contains several key concepts which it needs to make machine
+learning research shareable. A machine learning experiment consists of
+one or several **runs**, which describe the performance of an algorithm
+(called a **flow** in OpenML), its hyperparameter settings (called a
+**setup**) on a **task**. A **Task** is the combination of a
+**dataset**, a split and an evaluation metric. In this user guide we
+will go through listing and exploring existing **tasks** to actually
+running machine learning algorithms on them. In a further user guide we
+will examine how to search through **datasets** in order to curate a
+list of **tasks**.
+
+A further explanation is given in the [OpenML user
+guide](https://openml.github.io/OpenML/#concepts).
+
+## Working with tasks
+
+You can think of a task as an experimentation protocol, describing how
+to apply a machine learning model to a dataset in a way that is
+comparable with the results of others (more on how to do that further
+down). Tasks are containers, defining which dataset to use, what kind of
+task we\'re solving (regression, classification, clustering, etc\...)
+and which column to predict. Furthermore, it also describes how to split
+the dataset into a train and test set, whether to use several disjoint
+train and test splits (cross-validation) and whether this should be
+repeated several times. Also, the task defines a target metric for which
+a flow should be optimized.
+
+Below you can find our tutorial regarding tasks and if you want to know
+more you can read the [OpenML guide](https://docs.openml.org/#tasks):
+
+- `sphx_glr_examples_30_extended_tasks_tutorial.py`{.interpreted-text
+ role="ref"}
+
+## Running machine learning algorithms and uploading results
+
+In order to upload and share results of running a machine learning
+algorithm on a task, we need to create an
+`~openml.OpenMLRun`{.interpreted-text role="class"}. A run object can be
+created by running a `~openml.OpenMLFlow`{.interpreted-text
+role="class"} or a scikit-learn compatible model on a task. We will
+focus on the simpler example of running a scikit-learn model.
+
+Flows are descriptions of something runable which does the machine
+learning. A flow contains all information to set up the necessary
+machine learning library and its dependencies as well as all possible
+parameters.
+
+A run is the outcome of running a flow on a task. It contains all
+parameter settings for the flow, a setup string (most likely a command
+line call) and all predictions of that run. When a run is uploaded to
+the server, the server automatically calculates several metrics which
+can be used to compare the performance of different flows to each other.
+
+So far, the OpenML Python connector works only with estimator objects
+following the [scikit-learn estimator
+API](https://scikit-learn.org/stable/developers/develop.html#apis-of-scikit-learn-objects).
+Those can be directly run on a task, and a flow will automatically be
+created or downloaded from the server if it already exists.
+
+The next tutorial covers how to train different machine learning models,
+how to run machine learning models on OpenML data and how to share the
+results:
+
+- `sphx_glr_examples_20_basic_simple_flows_and_runs_tutorial.py`{.interpreted-text
+ role="ref"}
+
+## Datasets
+
+OpenML provides a large collection of datasets and the benchmark
+\"[OpenML100](https://docs.openml.org/benchmark/)\" which consists of a
+curated list of datasets.
+
+You can find the dataset that best fits your requirements by making use
+of the available metadata. The tutorial which follows explains how to
+get a list of datasets, how to filter the list to find the dataset that
+suits your requirements and how to download a dataset:
+
+- `sphx_glr_examples_30_extended_datasets_tutorial.py`{.interpreted-text
+ role="ref"}
+
+OpenML is about sharing machine learning results and the datasets they
+were obtained on. Learn how to share your datasets in the following
+tutorial:
+
+- `sphx_glr_examples_30_extended_create_upload_tutorial.py`{.interpreted-text
+ role="ref"}
+
+# Extending OpenML-Python
+
+OpenML-Python provides an extension interface to connect machine
+learning libraries directly to the API and ships a `scikit-learn`
+extension. You can find more information in the Section
+`extensions`{.interpreted-text role="ref"}\'
diff --git a/mkdocs.yml b/mkdocs.yml
new file mode 100644
index 000000000..e283b675e
--- /dev/null
+++ b/mkdocs.yml
@@ -0,0 +1,13 @@
+site_name: openml-python
+theme:
+ name: material
+
+markdown_extensions:
+ - pymdownx.highlight:
+ anchor_linenums: true
+ - pymdownx.superfences
+ - attr_list
+
+plugins:
+ - search
+ - autorefs
diff --git a/pyproject.toml b/pyproject.toml
index 99ff2b804..fa8816e1a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -88,6 +88,8 @@ docs=[
"sphinx-gallery",
"sphinx_bootstrap_theme",
"numpydoc",
+ "mkdocs-material",
+ "mkdocs-autorefs"
]
[project.urls]
From ff29c99217978947bb4f949d2514b17945cfd6b9 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 29 Jul 2024 12:01:03 +0200
Subject: [PATCH 02/25] Fix reference, avoids duplicate installation headers
---
docs/index.md | 2 +-
docs/usage.md | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/docs/index.md b/docs/index.md
index ddf28196b..4fb82443e 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -44,7 +44,7 @@ python -m pip install openml
```
For more advanced installation information, please see the
-["Installation"][installation] section.
+["Installation"][installation-guide] section.
## Content
diff --git a/docs/usage.md b/docs/usage.md
index 6a9a55be3..eefbb0532 100644
--- a/docs/usage.md
+++ b/docs/usage.md
@@ -5,6 +5,7 @@ functions and classes in the OpenML Python API. Throughout this
document, we will use [pandas](https://pandas.pydata.org/) to format and
filter tables.
+[](){#installation-guide}
## Installation
The OpenML Python package is a connector to
From 81a0a7bd9b4fec34a79faf2ca81195abdd7d0c99 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 29 Jul 2024 12:18:55 +0200
Subject: [PATCH 03/25] Improve citation section, fix last broken link
---
docs/contributing.md | 4 ---
docs/examples/index.md | 1 +
docs/index.md | 63 +++++++++++++++++-------------------------
mkdocs.yml | 4 +++
4 files changed, 31 insertions(+), 41 deletions(-)
create mode 100644 docs/examples/index.md
diff --git a/docs/contributing.md b/docs/contributing.md
index 93cbbe9d9..7329e2345 100644
--- a/docs/contributing.md
+++ b/docs/contributing.md
@@ -1,7 +1,3 @@
-orphan
-
-:
-
# Contributing
Contribution to the OpenML package is highly appreciated in all forms.
diff --git a/docs/examples/index.md b/docs/examples/index.md
new file mode 100644
index 000000000..4650e144c
--- /dev/null
+++ b/docs/examples/index.md
@@ -0,0 +1 @@
+# Example Gallery
\ No newline at end of file
diff --git a/docs/index.md b/docs/index.md
index 4fb82443e..d9247ed54 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -32,8 +32,7 @@ run.publish()
print(f'View the run online: {run.openml_url}')
```
-You can find more examples in our `examples-index`{.interpreted-text
-role="ref"}.
+Find more examples in our ["Example Gallery"][example-gallery].
## How to get OpenML for python
@@ -46,15 +45,6 @@ python -m pip install openml
For more advanced installation information, please see the
["Installation"][installation-guide] section.
-## Content
-
-- `usage`{.interpreted-text role="ref"}
-- `api`{.interpreted-text role="ref"}
-- `examples-index`{.interpreted-text role="ref"}
-- `extensions`{.interpreted-text role="ref"}
-- `contributing`{.interpreted-text role="ref"}
-- `progress`{.interpreted-text role="ref"}
-
## Further information
- [OpenML documentation](https://docs.openml.org/)
@@ -67,33 +57,32 @@ For more advanced installation information, please see the
## Contributing
-Contribution to the OpenML package is highly appreciated. The OpenML
-package currently has a 1/4 position for the development and all help
-possible is needed to extend and maintain the package, create new
-examples and improve the usability. Please see the
-`contributing`{.interpreted-text role="ref"} page for more information.
+Contribution to the OpenML package is highly appreciated. Please see the
+["Contributing"][contributing] page for more information.
## Citing OpenML-Python
If you use OpenML-Python in a scientific publication, we would
-appreciate a reference to the following paper:
-
-| Matthias Feurer, Jan N. van Rijn, Arlind Kadra, Pieter Gijsbers,
- Neeratyoy Mallik, Sahithya Ravi, Andreas Müller, Joaquin Vanschoren,
- Frank Hutter
-| **OpenML-Python: an extensible Python API for OpenML**
-| Journal of Machine Learning Research, 22(100):1−5, 2021
-| <https://www.jmlr.org/papers/v22/19-920.html>
-
-> Bibtex entry:
->
-> @article{JMLR:v22:19-920,
-> author = {Matthias Feurer and Jan N. van Rijn and Arlind Kadra and Pieter Gijsbers and Neeratyoy Mallik and Sahithya Ravi and Andreas Müller and Joaquin Vanschoren and Frank Hutter},
-> title = {OpenML-Python: an extensible Python API for OpenML},
-> journal = {Journal of Machine Learning Research},
-> year = {2021},
-> volume = {22},
-> number = {100},
-> pages = {1--5},
-> url = {http://jmlr.org/papers/v22/19-920.html}
-> }
+appreciate a reference to our JMLR-MLOSS paper
+["OpenML-Python: an extensible Python API for OpenML"](https://www.jmlr.org/papers/v22/19-920.html):
+
+=== "MLA"
+
+ Feurer, Matthias, et al.
+ "OpenML-Python: an extensible Python API for OpenML."
+ _Journal of Machine Learning Research_ 22.100 (2021):1−5.
+
+=== "Bibtex"
+
+ ```bibtex
+ @article{JMLR:v22:19-920,
+ author = {Matthias Feurer and Jan N. van Rijn and Arlind Kadra and Pieter Gijsbers and Neeratyoy Mallik and Sahithya Ravi and Andreas Müller and Joaquin Vanschoren and Frank Hutter},
+ title = {OpenML-Python: an extensible Python API for OpenML},
+ journal = {Journal of Machine Learning Research},
+ year = {2021},
+ volume = {22},
+ number = {100},
+ pages = {1--5},
+ url = {http://jmlr.org/papers/v22/19-920.html}
+ }
+ ```
\ No newline at end of file
diff --git a/mkdocs.yml b/mkdocs.yml
index e283b675e..2a5239f2a 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -1,12 +1,16 @@
site_name: openml-python
theme:
name: material
+ features:
+ - content.code.copy
markdown_extensions:
- pymdownx.highlight:
anchor_linenums: true
- pymdownx.superfences
- attr_list
+ - pymdownx.tabbed:
+ alternate_style: true
plugins:
- search
From 5cb04dd3cc11387f4d9fadf63ef9a7df3b2352b8 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 29 Jul 2024 12:20:59 +0200
Subject: [PATCH 04/25] Remove files which are to be generated
---
.../openml.datasets.OpenMLDataFeature.rst.md | 13 -------------
docs/generated/openml.datasets.OpenMLDataset.rst.md | 13 -------------
.../openml.datasets.attributes_arff_from_df.rst.md | 11 -----------
.../openml.datasets.check_datasets_active.rst.md | 11 -----------
.../generated/openml.datasets.create_dataset.rst.md | 11 -----------
.../generated/openml.datasets.delete_dataset.rst.md | 11 -----------
docs/generated/openml.datasets.edit_dataset.rst.md | 11 -----------
docs/generated/openml.datasets.fork_dataset.rst.md | 11 -----------
docs/generated/openml.datasets.get_dataset.rst.md | 11 -----------
docs/generated/openml.datasets.get_datasets.rst.md | 11 -----------
docs/generated/openml.datasets.list_datasets.rst.md | 11 -----------
.../generated/openml.datasets.list_qualities.rst.md | 11 -----------
docs/generated/openml.datasets.status_update.rst.md | 11 -----------
.../openml.evaluations.OpenMLEvaluation.rst.md | 13 -------------
...enml.evaluations.list_evaluation_measures.rst.md | 11 -----------
.../openml.evaluations.list_evaluations.rst.md | 11 -----------
...penml.evaluations.list_evaluations_setups.rst.md | 11 -----------
docs/generated/openml.extensions.Extension.rst.md | 13 -------------
.../openml.extensions.get_extension_by_flow.rst.md | 11 -----------
.../openml.extensions.get_extension_by_model.rst.md | 11 -----------
.../openml.extensions.register_extension.rst.md | 11 -----------
...penml.extensions.sklearn.SklearnExtension.rst.md | 13 -------------
docs/generated/openml.flows.OpenMLFlow.rst.md | 13 -------------
.../openml.flows.assert_flows_equal.rst.md | 11 -----------
docs/generated/openml.flows.delete_flow.rst.md | 11 -----------
docs/generated/openml.flows.flow_exists.rst.md | 11 -----------
docs/generated/openml.flows.get_flow.rst.md | 11 -----------
docs/generated/openml.flows.list_flows.rst.md | 11 -----------
docs/generated/openml.runs.OpenMLRun.rst.md | 13 -------------
docs/generated/openml.runs.delete_run.rst.md | 11 -----------
docs/generated/openml.runs.get_run.rst.md | 11 -----------
docs/generated/openml.runs.get_run_trace.rst.md | 11 -----------
docs/generated/openml.runs.get_runs.rst.md | 11 -----------
.../openml.runs.initialize_model_from_run.rst.md | 11 -----------
.../openml.runs.initialize_model_from_trace.rst.md | 11 -----------
docs/generated/openml.runs.list_runs.rst.md | 11 -----------
docs/generated/openml.runs.run_exists.rst.md | 11 -----------
docs/generated/openml.runs.run_flow_on_task.rst.md | 11 -----------
docs/generated/openml.runs.run_model_on_task.rst.md | 11 -----------
docs/generated/openml.setups.OpenMLParameter.rst.md | 13 -------------
docs/generated/openml.setups.OpenMLSetup.rst.md | 13 -------------
docs/generated/openml.setups.get_setup.rst.md | 11 -----------
.../generated/openml.setups.initialize_model.rst.md | 11 -----------
docs/generated/openml.setups.list_setups.rst.md | 11 -----------
docs/generated/openml.setups.setup_exists.rst.md | 11 -----------
.../openml.study.OpenMLBenchmarkSuite.rst.md | 13 -------------
docs/generated/openml.study.OpenMLStudy.rst.md | 13 -------------
docs/generated/openml.study.attach_to_study.rst.md | 11 -----------
docs/generated/openml.study.attach_to_suite.rst.md | 11 -----------
.../openml.study.create_benchmark_suite.rst.md | 11 -----------
docs/generated/openml.study.create_study.rst.md | 11 -----------
docs/generated/openml.study.delete_study.rst.md | 11 -----------
docs/generated/openml.study.delete_suite.rst.md | 11 -----------
.../generated/openml.study.detach_from_study.rst.md | 11 -----------
.../generated/openml.study.detach_from_suite.rst.md | 11 -----------
docs/generated/openml.study.get_study.rst.md | 11 -----------
docs/generated/openml.study.get_suite.rst.md | 11 -----------
docs/generated/openml.study.list_studies.rst.md | 11 -----------
docs/generated/openml.study.list_suites.rst.md | 11 -----------
.../openml.study.update_study_status.rst.md | 11 -----------
.../openml.study.update_suite_status.rst.md | 11 -----------
.../openml.tasks.OpenMLClassificationTask.rst.md | 13 -------------
.../openml.tasks.OpenMLClusteringTask.rst.md | 13 -------------
.../openml.tasks.OpenMLLearningCurveTask.rst.md | 13 -------------
.../openml.tasks.OpenMLRegressionTask.rst.md | 13 -------------
docs/generated/openml.tasks.OpenMLSplit.rst.md | 13 -------------
.../openml.tasks.OpenMLSupervisedTask.rst.md | 13 -------------
docs/generated/openml.tasks.OpenMLTask.rst.md | 13 -------------
docs/generated/openml.tasks.TaskType.rst.md | 13 -------------
docs/generated/openml.tasks.create_task.rst.md | 11 -----------
docs/generated/openml.tasks.delete_task.rst.md | 11 -----------
docs/generated/openml.tasks.get_task.rst.md | 11 -----------
docs/generated/openml.tasks.get_tasks.rst.md | 11 -----------
docs/generated/openml.tasks.list_tasks.rst.md | 11 -----------
74 files changed, 852 deletions(-)
delete mode 100644 docs/generated/openml.datasets.OpenMLDataFeature.rst.md
delete mode 100644 docs/generated/openml.datasets.OpenMLDataset.rst.md
delete mode 100644 docs/generated/openml.datasets.attributes_arff_from_df.rst.md
delete mode 100644 docs/generated/openml.datasets.check_datasets_active.rst.md
delete mode 100644 docs/generated/openml.datasets.create_dataset.rst.md
delete mode 100644 docs/generated/openml.datasets.delete_dataset.rst.md
delete mode 100644 docs/generated/openml.datasets.edit_dataset.rst.md
delete mode 100644 docs/generated/openml.datasets.fork_dataset.rst.md
delete mode 100644 docs/generated/openml.datasets.get_dataset.rst.md
delete mode 100644 docs/generated/openml.datasets.get_datasets.rst.md
delete mode 100644 docs/generated/openml.datasets.list_datasets.rst.md
delete mode 100644 docs/generated/openml.datasets.list_qualities.rst.md
delete mode 100644 docs/generated/openml.datasets.status_update.rst.md
delete mode 100644 docs/generated/openml.evaluations.OpenMLEvaluation.rst.md
delete mode 100644 docs/generated/openml.evaluations.list_evaluation_measures.rst.md
delete mode 100644 docs/generated/openml.evaluations.list_evaluations.rst.md
delete mode 100644 docs/generated/openml.evaluations.list_evaluations_setups.rst.md
delete mode 100644 docs/generated/openml.extensions.Extension.rst.md
delete mode 100644 docs/generated/openml.extensions.get_extension_by_flow.rst.md
delete mode 100644 docs/generated/openml.extensions.get_extension_by_model.rst.md
delete mode 100644 docs/generated/openml.extensions.register_extension.rst.md
delete mode 100644 docs/generated/openml.extensions.sklearn.SklearnExtension.rst.md
delete mode 100644 docs/generated/openml.flows.OpenMLFlow.rst.md
delete mode 100644 docs/generated/openml.flows.assert_flows_equal.rst.md
delete mode 100644 docs/generated/openml.flows.delete_flow.rst.md
delete mode 100644 docs/generated/openml.flows.flow_exists.rst.md
delete mode 100644 docs/generated/openml.flows.get_flow.rst.md
delete mode 100644 docs/generated/openml.flows.list_flows.rst.md
delete mode 100644 docs/generated/openml.runs.OpenMLRun.rst.md
delete mode 100644 docs/generated/openml.runs.delete_run.rst.md
delete mode 100644 docs/generated/openml.runs.get_run.rst.md
delete mode 100644 docs/generated/openml.runs.get_run_trace.rst.md
delete mode 100644 docs/generated/openml.runs.get_runs.rst.md
delete mode 100644 docs/generated/openml.runs.initialize_model_from_run.rst.md
delete mode 100644 docs/generated/openml.runs.initialize_model_from_trace.rst.md
delete mode 100644 docs/generated/openml.runs.list_runs.rst.md
delete mode 100644 docs/generated/openml.runs.run_exists.rst.md
delete mode 100644 docs/generated/openml.runs.run_flow_on_task.rst.md
delete mode 100644 docs/generated/openml.runs.run_model_on_task.rst.md
delete mode 100644 docs/generated/openml.setups.OpenMLParameter.rst.md
delete mode 100644 docs/generated/openml.setups.OpenMLSetup.rst.md
delete mode 100644 docs/generated/openml.setups.get_setup.rst.md
delete mode 100644 docs/generated/openml.setups.initialize_model.rst.md
delete mode 100644 docs/generated/openml.setups.list_setups.rst.md
delete mode 100644 docs/generated/openml.setups.setup_exists.rst.md
delete mode 100644 docs/generated/openml.study.OpenMLBenchmarkSuite.rst.md
delete mode 100644 docs/generated/openml.study.OpenMLStudy.rst.md
delete mode 100644 docs/generated/openml.study.attach_to_study.rst.md
delete mode 100644 docs/generated/openml.study.attach_to_suite.rst.md
delete mode 100644 docs/generated/openml.study.create_benchmark_suite.rst.md
delete mode 100644 docs/generated/openml.study.create_study.rst.md
delete mode 100644 docs/generated/openml.study.delete_study.rst.md
delete mode 100644 docs/generated/openml.study.delete_suite.rst.md
delete mode 100644 docs/generated/openml.study.detach_from_study.rst.md
delete mode 100644 docs/generated/openml.study.detach_from_suite.rst.md
delete mode 100644 docs/generated/openml.study.get_study.rst.md
delete mode 100644 docs/generated/openml.study.get_suite.rst.md
delete mode 100644 docs/generated/openml.study.list_studies.rst.md
delete mode 100644 docs/generated/openml.study.list_suites.rst.md
delete mode 100644 docs/generated/openml.study.update_study_status.rst.md
delete mode 100644 docs/generated/openml.study.update_suite_status.rst.md
delete mode 100644 docs/generated/openml.tasks.OpenMLClassificationTask.rst.md
delete mode 100644 docs/generated/openml.tasks.OpenMLClusteringTask.rst.md
delete mode 100644 docs/generated/openml.tasks.OpenMLLearningCurveTask.rst.md
delete mode 100644 docs/generated/openml.tasks.OpenMLRegressionTask.rst.md
delete mode 100644 docs/generated/openml.tasks.OpenMLSplit.rst.md
delete mode 100644 docs/generated/openml.tasks.OpenMLSupervisedTask.rst.md
delete mode 100644 docs/generated/openml.tasks.OpenMLTask.rst.md
delete mode 100644 docs/generated/openml.tasks.TaskType.rst.md
delete mode 100644 docs/generated/openml.tasks.create_task.rst.md
delete mode 100644 docs/generated/openml.tasks.delete_task.rst.md
delete mode 100644 docs/generated/openml.tasks.get_task.rst.md
delete mode 100644 docs/generated/openml.tasks.get_tasks.rst.md
delete mode 100644 docs/generated/openml.tasks.list_tasks.rst.md
diff --git a/docs/generated/openml.datasets.OpenMLDataFeature.rst.md b/docs/generated/openml.datasets.OpenMLDataFeature.rst.md
deleted file mode 100644
index 5e049b341..000000000
--- a/docs/generated/openml.datasets.OpenMLDataFeature.rst.md
+++ /dev/null
@@ -1,13 +0,0 @@
-orphan
-
-:
-
-# `openml.datasets`{.interpreted-text role="mod"}.OpenMLDataFeature
-
-::: currentmodule
-openml.datasets
-:::
-
-::: autoclass
-OpenMLDataFeature
-:::
diff --git a/docs/generated/openml.datasets.OpenMLDataset.rst.md b/docs/generated/openml.datasets.OpenMLDataset.rst.md
deleted file mode 100644
index 63b34cad3..000000000
--- a/docs/generated/openml.datasets.OpenMLDataset.rst.md
+++ /dev/null
@@ -1,13 +0,0 @@
-orphan
-
-:
-
-# `openml.datasets`{.interpreted-text role="mod"}.OpenMLDataset
-
-::: currentmodule
-openml.datasets
-:::
-
-::: autoclass
-OpenMLDataset
-:::
diff --git a/docs/generated/openml.datasets.attributes_arff_from_df.rst.md b/docs/generated/openml.datasets.attributes_arff_from_df.rst.md
deleted file mode 100644
index f923a447c..000000000
--- a/docs/generated/openml.datasets.attributes_arff_from_df.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.datasets`{.interpreted-text role="mod"}.attributes_arff_from_df
-
-::: currentmodule
-openml.datasets
-:::
-
-::: autofunction
-attributes_arff_from_df
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.datasets.check_datasets_active.rst.md b/docs/generated/openml.datasets.check_datasets_active.rst.md
deleted file mode 100644
index 9b12e9b7c..000000000
--- a/docs/generated/openml.datasets.check_datasets_active.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.datasets`{.interpreted-text role="mod"}.check_datasets_active
-
-::: currentmodule
-openml.datasets
-:::
-
-::: autofunction
-check_datasets_active
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.datasets.create_dataset.rst.md b/docs/generated/openml.datasets.create_dataset.rst.md
deleted file mode 100644
index d0a7d9e13..000000000
--- a/docs/generated/openml.datasets.create_dataset.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.datasets`{.interpreted-text role="mod"}.create_dataset
-
-::: currentmodule
-openml.datasets
-:::
-
-::: autofunction
-create_dataset
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.datasets.delete_dataset.rst.md b/docs/generated/openml.datasets.delete_dataset.rst.md
deleted file mode 100644
index fd90d06c5..000000000
--- a/docs/generated/openml.datasets.delete_dataset.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.datasets`{.interpreted-text role="mod"}.delete_dataset
-
-::: currentmodule
-openml.datasets
-:::
-
-::: autofunction
-delete_dataset
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.datasets.edit_dataset.rst.md b/docs/generated/openml.datasets.edit_dataset.rst.md
deleted file mode 100644
index 251a54ba0..000000000
--- a/docs/generated/openml.datasets.edit_dataset.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.datasets`{.interpreted-text role="mod"}.edit_dataset
-
-::: currentmodule
-openml.datasets
-:::
-
-::: autofunction
-edit_dataset
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.datasets.fork_dataset.rst.md b/docs/generated/openml.datasets.fork_dataset.rst.md
deleted file mode 100644
index 43e36c964..000000000
--- a/docs/generated/openml.datasets.fork_dataset.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.datasets`{.interpreted-text role="mod"}.fork_dataset
-
-::: currentmodule
-openml.datasets
-:::
-
-::: autofunction
-fork_dataset
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.datasets.get_dataset.rst.md b/docs/generated/openml.datasets.get_dataset.rst.md
deleted file mode 100644
index a8b5001b2..000000000
--- a/docs/generated/openml.datasets.get_dataset.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.datasets`{.interpreted-text role="mod"}.get_dataset
-
-::: currentmodule
-openml.datasets
-:::
-
-::: autofunction
-get_dataset
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.datasets.get_datasets.rst.md b/docs/generated/openml.datasets.get_datasets.rst.md
deleted file mode 100644
index 2585288a5..000000000
--- a/docs/generated/openml.datasets.get_datasets.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.datasets`{.interpreted-text role="mod"}.get_datasets
-
-::: currentmodule
-openml.datasets
-:::
-
-::: autofunction
-get_datasets
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.datasets.list_datasets.rst.md b/docs/generated/openml.datasets.list_datasets.rst.md
deleted file mode 100644
index 98832708b..000000000
--- a/docs/generated/openml.datasets.list_datasets.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.datasets`{.interpreted-text role="mod"}.list_datasets
-
-::: currentmodule
-openml.datasets
-:::
-
-::: autofunction
-list_datasets
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.datasets.list_qualities.rst.md b/docs/generated/openml.datasets.list_qualities.rst.md
deleted file mode 100644
index 659f91410..000000000
--- a/docs/generated/openml.datasets.list_qualities.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.datasets`{.interpreted-text role="mod"}.list_qualities
-
-::: currentmodule
-openml.datasets
-:::
-
-::: autofunction
-list_qualities
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.datasets.status_update.rst.md b/docs/generated/openml.datasets.status_update.rst.md
deleted file mode 100644
index edb8df9a6..000000000
--- a/docs/generated/openml.datasets.status_update.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.datasets`{.interpreted-text role="mod"}.status_update
-
-::: currentmodule
-openml.datasets
-:::
-
-::: autofunction
-status_update
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.evaluations.OpenMLEvaluation.rst.md b/docs/generated/openml.evaluations.OpenMLEvaluation.rst.md
deleted file mode 100644
index e953137eb..000000000
--- a/docs/generated/openml.evaluations.OpenMLEvaluation.rst.md
+++ /dev/null
@@ -1,13 +0,0 @@
-orphan
-
-:
-
-# `openml.evaluations`{.interpreted-text role="mod"}.OpenMLEvaluation
-
-::: currentmodule
-openml.evaluations
-:::
-
-::: autoclass
-OpenMLEvaluation
-:::
diff --git a/docs/generated/openml.evaluations.list_evaluation_measures.rst.md b/docs/generated/openml.evaluations.list_evaluation_measures.rst.md
deleted file mode 100644
index 81a25dd54..000000000
--- a/docs/generated/openml.evaluations.list_evaluation_measures.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.evaluations`{.interpreted-text role="mod"}.list_evaluation_measures
-
-::: currentmodule
-openml.evaluations
-:::
-
-::: autofunction
-list_evaluation_measures
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.evaluations.list_evaluations.rst.md b/docs/generated/openml.evaluations.list_evaluations.rst.md
deleted file mode 100644
index d826310a9..000000000
--- a/docs/generated/openml.evaluations.list_evaluations.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.evaluations`{.interpreted-text role="mod"}.list_evaluations
-
-::: currentmodule
-openml.evaluations
-:::
-
-::: autofunction
-list_evaluations
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.evaluations.list_evaluations_setups.rst.md b/docs/generated/openml.evaluations.list_evaluations_setups.rst.md
deleted file mode 100644
index a2f37e5a9..000000000
--- a/docs/generated/openml.evaluations.list_evaluations_setups.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.evaluations`{.interpreted-text role="mod"}.list_evaluations_setups
-
-::: currentmodule
-openml.evaluations
-:::
-
-::: autofunction
-list_evaluations_setups
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.extensions.Extension.rst.md b/docs/generated/openml.extensions.Extension.rst.md
deleted file mode 100644
index 8ea18ecf4..000000000
--- a/docs/generated/openml.extensions.Extension.rst.md
+++ /dev/null
@@ -1,13 +0,0 @@
-orphan
-
-:
-
-# `openml.extensions`{.interpreted-text role="mod"}.Extension
-
-::: currentmodule
-openml.extensions
-:::
-
-::: autoclass
-Extension
-:::
diff --git a/docs/generated/openml.extensions.get_extension_by_flow.rst.md b/docs/generated/openml.extensions.get_extension_by_flow.rst.md
deleted file mode 100644
index f86230751..000000000
--- a/docs/generated/openml.extensions.get_extension_by_flow.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.extensions`{.interpreted-text role="mod"}.get_extension_by_flow
-
-::: currentmodule
-openml.extensions
-:::
-
-::: autofunction
-get_extension_by_flow
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.extensions.get_extension_by_model.rst.md b/docs/generated/openml.extensions.get_extension_by_model.rst.md
deleted file mode 100644
index 057a0812e..000000000
--- a/docs/generated/openml.extensions.get_extension_by_model.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.extensions`{.interpreted-text role="mod"}.get_extension_by_model
-
-::: currentmodule
-openml.extensions
-:::
-
-::: autofunction
-get_extension_by_model
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.extensions.register_extension.rst.md b/docs/generated/openml.extensions.register_extension.rst.md
deleted file mode 100644
index 8304875c0..000000000
--- a/docs/generated/openml.extensions.register_extension.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.extensions`{.interpreted-text role="mod"}.register_extension
-
-::: currentmodule
-openml.extensions
-:::
-
-::: autofunction
-register_extension
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.extensions.sklearn.SklearnExtension.rst.md b/docs/generated/openml.extensions.sklearn.SklearnExtension.rst.md
deleted file mode 100644
index 8cd1c9714..000000000
--- a/docs/generated/openml.extensions.sklearn.SklearnExtension.rst.md
+++ /dev/null
@@ -1,13 +0,0 @@
-orphan
-
-:
-
-# `openml.extensions.sklearn`{.interpreted-text role="mod"}.SklearnExtension
-
-::: currentmodule
-openml.extensions.sklearn
-:::
-
-::: autoclass
-SklearnExtension
-:::
diff --git a/docs/generated/openml.flows.OpenMLFlow.rst.md b/docs/generated/openml.flows.OpenMLFlow.rst.md
deleted file mode 100644
index c3ba4f80f..000000000
--- a/docs/generated/openml.flows.OpenMLFlow.rst.md
+++ /dev/null
@@ -1,13 +0,0 @@
-orphan
-
-:
-
-# `openml.flows`{.interpreted-text role="mod"}.OpenMLFlow
-
-::: currentmodule
-openml.flows
-:::
-
-::: autoclass
-OpenMLFlow
-:::
diff --git a/docs/generated/openml.flows.assert_flows_equal.rst.md b/docs/generated/openml.flows.assert_flows_equal.rst.md
deleted file mode 100644
index 0e55a16ef..000000000
--- a/docs/generated/openml.flows.assert_flows_equal.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.flows`{.interpreted-text role="mod"}.assert_flows_equal
-
-::: currentmodule
-openml.flows
-:::
-
-::: autofunction
-assert_flows_equal
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.flows.delete_flow.rst.md b/docs/generated/openml.flows.delete_flow.rst.md
deleted file mode 100644
index d3abe3503..000000000
--- a/docs/generated/openml.flows.delete_flow.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.flows`{.interpreted-text role="mod"}.delete_flow
-
-::: currentmodule
-openml.flows
-:::
-
-::: autofunction
-delete_flow
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.flows.flow_exists.rst.md b/docs/generated/openml.flows.flow_exists.rst.md
deleted file mode 100644
index bf766b1ac..000000000
--- a/docs/generated/openml.flows.flow_exists.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.flows`{.interpreted-text role="mod"}.flow_exists
-
-::: currentmodule
-openml.flows
-:::
-
-::: autofunction
-flow_exists
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.flows.get_flow.rst.md b/docs/generated/openml.flows.get_flow.rst.md
deleted file mode 100644
index 10b800c80..000000000
--- a/docs/generated/openml.flows.get_flow.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.flows`{.interpreted-text role="mod"}.get_flow
-
-::: currentmodule
-openml.flows
-:::
-
-::: autofunction
-get_flow
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.flows.list_flows.rst.md b/docs/generated/openml.flows.list_flows.rst.md
deleted file mode 100644
index d557b7029..000000000
--- a/docs/generated/openml.flows.list_flows.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.flows`{.interpreted-text role="mod"}.list_flows
-
-::: currentmodule
-openml.flows
-:::
-
-::: autofunction
-list_flows
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.runs.OpenMLRun.rst.md b/docs/generated/openml.runs.OpenMLRun.rst.md
deleted file mode 100644
index 36230c66c..000000000
--- a/docs/generated/openml.runs.OpenMLRun.rst.md
+++ /dev/null
@@ -1,13 +0,0 @@
-orphan
-
-:
-
-# `openml.runs`{.interpreted-text role="mod"}.OpenMLRun
-
-::: currentmodule
-openml.runs
-:::
-
-::: autoclass
-OpenMLRun
-:::
diff --git a/docs/generated/openml.runs.delete_run.rst.md b/docs/generated/openml.runs.delete_run.rst.md
deleted file mode 100644
index 7a1603c1b..000000000
--- a/docs/generated/openml.runs.delete_run.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.runs`{.interpreted-text role="mod"}.delete_run
-
-::: currentmodule
-openml.runs
-:::
-
-::: autofunction
-delete_run
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.runs.get_run.rst.md b/docs/generated/openml.runs.get_run.rst.md
deleted file mode 100644
index 8957d4dc8..000000000
--- a/docs/generated/openml.runs.get_run.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.runs`{.interpreted-text role="mod"}.get_run
-
-::: currentmodule
-openml.runs
-:::
-
-::: autofunction
-get_run
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.runs.get_run_trace.rst.md b/docs/generated/openml.runs.get_run_trace.rst.md
deleted file mode 100644
index 67f4a31f6..000000000
--- a/docs/generated/openml.runs.get_run_trace.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.runs`{.interpreted-text role="mod"}.get_run_trace
-
-::: currentmodule
-openml.runs
-:::
-
-::: autofunction
-get_run_trace
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.runs.get_runs.rst.md b/docs/generated/openml.runs.get_runs.rst.md
deleted file mode 100644
index 7150a72f5..000000000
--- a/docs/generated/openml.runs.get_runs.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.runs`{.interpreted-text role="mod"}.get_runs
-
-::: currentmodule
-openml.runs
-:::
-
-::: autofunction
-get_runs
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.runs.initialize_model_from_run.rst.md b/docs/generated/openml.runs.initialize_model_from_run.rst.md
deleted file mode 100644
index 6eed93268..000000000
--- a/docs/generated/openml.runs.initialize_model_from_run.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.runs`{.interpreted-text role="mod"}.initialize_model_from_run
-
-::: currentmodule
-openml.runs
-:::
-
-::: autofunction
-initialize_model_from_run
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.runs.initialize_model_from_trace.rst.md b/docs/generated/openml.runs.initialize_model_from_trace.rst.md
deleted file mode 100644
index 0edd6f439..000000000
--- a/docs/generated/openml.runs.initialize_model_from_trace.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.runs`{.interpreted-text role="mod"}.initialize_model_from_trace
-
-::: currentmodule
-openml.runs
-:::
-
-::: autofunction
-initialize_model_from_trace
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.runs.list_runs.rst.md b/docs/generated/openml.runs.list_runs.rst.md
deleted file mode 100644
index 51a556b62..000000000
--- a/docs/generated/openml.runs.list_runs.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.runs`{.interpreted-text role="mod"}.list_runs
-
-::: currentmodule
-openml.runs
-:::
-
-::: autofunction
-list_runs
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.runs.run_exists.rst.md b/docs/generated/openml.runs.run_exists.rst.md
deleted file mode 100644
index 9d2b8b9e7..000000000
--- a/docs/generated/openml.runs.run_exists.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.runs`{.interpreted-text role="mod"}.run_exists
-
-::: currentmodule
-openml.runs
-:::
-
-::: autofunction
-run_exists
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.runs.run_flow_on_task.rst.md b/docs/generated/openml.runs.run_flow_on_task.rst.md
deleted file mode 100644
index 081b550a8..000000000
--- a/docs/generated/openml.runs.run_flow_on_task.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.runs`{.interpreted-text role="mod"}.run_flow_on_task
-
-::: currentmodule
-openml.runs
-:::
-
-::: autofunction
-run_flow_on_task
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.runs.run_model_on_task.rst.md b/docs/generated/openml.runs.run_model_on_task.rst.md
deleted file mode 100644
index bda77fc1f..000000000
--- a/docs/generated/openml.runs.run_model_on_task.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.runs`{.interpreted-text role="mod"}.run_model_on_task
-
-::: currentmodule
-openml.runs
-:::
-
-::: autofunction
-run_model_on_task
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.setups.OpenMLParameter.rst.md b/docs/generated/openml.setups.OpenMLParameter.rst.md
deleted file mode 100644
index 6c5468b1a..000000000
--- a/docs/generated/openml.setups.OpenMLParameter.rst.md
+++ /dev/null
@@ -1,13 +0,0 @@
-orphan
-
-:
-
-# `openml.setups`{.interpreted-text role="mod"}.OpenMLParameter
-
-::: currentmodule
-openml.setups
-:::
-
-::: autoclass
-OpenMLParameter
-:::
diff --git a/docs/generated/openml.setups.OpenMLSetup.rst.md b/docs/generated/openml.setups.OpenMLSetup.rst.md
deleted file mode 100644
index 6d372fac0..000000000
--- a/docs/generated/openml.setups.OpenMLSetup.rst.md
+++ /dev/null
@@ -1,13 +0,0 @@
-orphan
-
-:
-
-# `openml.setups`{.interpreted-text role="mod"}.OpenMLSetup
-
-::: currentmodule
-openml.setups
-:::
-
-::: autoclass
-OpenMLSetup
-:::
diff --git a/docs/generated/openml.setups.get_setup.rst.md b/docs/generated/openml.setups.get_setup.rst.md
deleted file mode 100644
index bb65d1edf..000000000
--- a/docs/generated/openml.setups.get_setup.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.setups`{.interpreted-text role="mod"}.get_setup
-
-::: currentmodule
-openml.setups
-:::
-
-::: autofunction
-get_setup
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.setups.initialize_model.rst.md b/docs/generated/openml.setups.initialize_model.rst.md
deleted file mode 100644
index 4dd66f52c..000000000
--- a/docs/generated/openml.setups.initialize_model.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.setups`{.interpreted-text role="mod"}.initialize_model
-
-::: currentmodule
-openml.setups
-:::
-
-::: autofunction
-initialize_model
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.setups.list_setups.rst.md b/docs/generated/openml.setups.list_setups.rst.md
deleted file mode 100644
index 4be3cebe0..000000000
--- a/docs/generated/openml.setups.list_setups.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.setups`{.interpreted-text role="mod"}.list_setups
-
-::: currentmodule
-openml.setups
-:::
-
-::: autofunction
-list_setups
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.setups.setup_exists.rst.md b/docs/generated/openml.setups.setup_exists.rst.md
deleted file mode 100644
index 1027524e4..000000000
--- a/docs/generated/openml.setups.setup_exists.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.setups`{.interpreted-text role="mod"}.setup_exists
-
-::: currentmodule
-openml.setups
-:::
-
-::: autofunction
-setup_exists
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.study.OpenMLBenchmarkSuite.rst.md b/docs/generated/openml.study.OpenMLBenchmarkSuite.rst.md
deleted file mode 100644
index 2240023ce..000000000
--- a/docs/generated/openml.study.OpenMLBenchmarkSuite.rst.md
+++ /dev/null
@@ -1,13 +0,0 @@
-orphan
-
-:
-
-# `openml.study`{.interpreted-text role="mod"}.OpenMLBenchmarkSuite
-
-::: currentmodule
-openml.study
-:::
-
-::: autoclass
-OpenMLBenchmarkSuite
-:::
diff --git a/docs/generated/openml.study.OpenMLStudy.rst.md b/docs/generated/openml.study.OpenMLStudy.rst.md
deleted file mode 100644
index 3f9fb44dc..000000000
--- a/docs/generated/openml.study.OpenMLStudy.rst.md
+++ /dev/null
@@ -1,13 +0,0 @@
-orphan
-
-:
-
-# `openml.study`{.interpreted-text role="mod"}.OpenMLStudy
-
-::: currentmodule
-openml.study
-:::
-
-::: autoclass
-OpenMLStudy
-:::
diff --git a/docs/generated/openml.study.attach_to_study.rst.md b/docs/generated/openml.study.attach_to_study.rst.md
deleted file mode 100644
index 87d02d70c..000000000
--- a/docs/generated/openml.study.attach_to_study.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.study`{.interpreted-text role="mod"}.attach_to_study
-
-::: currentmodule
-openml.study
-:::
-
-::: autofunction
-attach_to_study
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.study.attach_to_suite.rst.md b/docs/generated/openml.study.attach_to_suite.rst.md
deleted file mode 100644
index 752f0c438..000000000
--- a/docs/generated/openml.study.attach_to_suite.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.study`{.interpreted-text role="mod"}.attach_to_suite
-
-::: currentmodule
-openml.study
-:::
-
-::: autofunction
-attach_to_suite
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.study.create_benchmark_suite.rst.md b/docs/generated/openml.study.create_benchmark_suite.rst.md
deleted file mode 100644
index 2c2b22c6a..000000000
--- a/docs/generated/openml.study.create_benchmark_suite.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.study`{.interpreted-text role="mod"}.create_benchmark_suite
-
-::: currentmodule
-openml.study
-:::
-
-::: autofunction
-create_benchmark_suite
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.study.create_study.rst.md b/docs/generated/openml.study.create_study.rst.md
deleted file mode 100644
index dc7ed4595..000000000
--- a/docs/generated/openml.study.create_study.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.study`{.interpreted-text role="mod"}.create_study
-
-::: currentmodule
-openml.study
-:::
-
-::: autofunction
-create_study
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.study.delete_study.rst.md b/docs/generated/openml.study.delete_study.rst.md
deleted file mode 100644
index c1fed242b..000000000
--- a/docs/generated/openml.study.delete_study.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.study`{.interpreted-text role="mod"}.delete_study
-
-::: currentmodule
-openml.study
-:::
-
-::: autofunction
-delete_study
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.study.delete_suite.rst.md b/docs/generated/openml.study.delete_suite.rst.md
deleted file mode 100644
index f65e310a7..000000000
--- a/docs/generated/openml.study.delete_suite.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.study`{.interpreted-text role="mod"}.delete_suite
-
-::: currentmodule
-openml.study
-:::
-
-::: autofunction
-delete_suite
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.study.detach_from_study.rst.md b/docs/generated/openml.study.detach_from_study.rst.md
deleted file mode 100644
index 25b4a4e62..000000000
--- a/docs/generated/openml.study.detach_from_study.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.study`{.interpreted-text role="mod"}.detach_from_study
-
-::: currentmodule
-openml.study
-:::
-
-::: autofunction
-detach_from_study
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.study.detach_from_suite.rst.md b/docs/generated/openml.study.detach_from_suite.rst.md
deleted file mode 100644
index 44f6751ae..000000000
--- a/docs/generated/openml.study.detach_from_suite.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.study`{.interpreted-text role="mod"}.detach_from_suite
-
-::: currentmodule
-openml.study
-:::
-
-::: autofunction
-detach_from_suite
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.study.get_study.rst.md b/docs/generated/openml.study.get_study.rst.md
deleted file mode 100644
index 34e385e7e..000000000
--- a/docs/generated/openml.study.get_study.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.study`{.interpreted-text role="mod"}.get_study
-
-::: currentmodule
-openml.study
-:::
-
-::: autofunction
-get_study
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.study.get_suite.rst.md b/docs/generated/openml.study.get_suite.rst.md
deleted file mode 100644
index a3c0a6008..000000000
--- a/docs/generated/openml.study.get_suite.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.study`{.interpreted-text role="mod"}.get_suite
-
-::: currentmodule
-openml.study
-:::
-
-::: autofunction
-get_suite
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.study.list_studies.rst.md b/docs/generated/openml.study.list_studies.rst.md
deleted file mode 100644
index f98fd0410..000000000
--- a/docs/generated/openml.study.list_studies.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.study`{.interpreted-text role="mod"}.list_studies
-
-::: currentmodule
-openml.study
-:::
-
-::: autofunction
-list_studies
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.study.list_suites.rst.md b/docs/generated/openml.study.list_suites.rst.md
deleted file mode 100644
index 13ee56751..000000000
--- a/docs/generated/openml.study.list_suites.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.study`{.interpreted-text role="mod"}.list_suites
-
-::: currentmodule
-openml.study
-:::
-
-::: autofunction
-list_suites
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.study.update_study_status.rst.md b/docs/generated/openml.study.update_study_status.rst.md
deleted file mode 100644
index 19c4a01a1..000000000
--- a/docs/generated/openml.study.update_study_status.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.study`{.interpreted-text role="mod"}.update_study_status
-
-::: currentmodule
-openml.study
-:::
-
-::: autofunction
-update_study_status
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.study.update_suite_status.rst.md b/docs/generated/openml.study.update_suite_status.rst.md
deleted file mode 100644
index 36698c599..000000000
--- a/docs/generated/openml.study.update_suite_status.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.study`{.interpreted-text role="mod"}.update_suite_status
-
-::: currentmodule
-openml.study
-:::
-
-::: autofunction
-update_suite_status
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.tasks.OpenMLClassificationTask.rst.md b/docs/generated/openml.tasks.OpenMLClassificationTask.rst.md
deleted file mode 100644
index 4dd7ad30b..000000000
--- a/docs/generated/openml.tasks.OpenMLClassificationTask.rst.md
+++ /dev/null
@@ -1,13 +0,0 @@
-orphan
-
-:
-
-# `openml.tasks`{.interpreted-text role="mod"}.OpenMLClassificationTask
-
-::: currentmodule
-openml.tasks
-:::
-
-::: autoclass
-OpenMLClassificationTask
-:::
diff --git a/docs/generated/openml.tasks.OpenMLClusteringTask.rst.md b/docs/generated/openml.tasks.OpenMLClusteringTask.rst.md
deleted file mode 100644
index 98d21cec1..000000000
--- a/docs/generated/openml.tasks.OpenMLClusteringTask.rst.md
+++ /dev/null
@@ -1,13 +0,0 @@
-orphan
-
-:
-
-# `openml.tasks`{.interpreted-text role="mod"}.OpenMLClusteringTask
-
-::: currentmodule
-openml.tasks
-:::
-
-::: autoclass
-OpenMLClusteringTask
-:::
diff --git a/docs/generated/openml.tasks.OpenMLLearningCurveTask.rst.md b/docs/generated/openml.tasks.OpenMLLearningCurveTask.rst.md
deleted file mode 100644
index bf7e68d57..000000000
--- a/docs/generated/openml.tasks.OpenMLLearningCurveTask.rst.md
+++ /dev/null
@@ -1,13 +0,0 @@
-orphan
-
-:
-
-# `openml.tasks`{.interpreted-text role="mod"}.OpenMLLearningCurveTask
-
-::: currentmodule
-openml.tasks
-:::
-
-::: autoclass
-OpenMLLearningCurveTask
-:::
diff --git a/docs/generated/openml.tasks.OpenMLRegressionTask.rst.md b/docs/generated/openml.tasks.OpenMLRegressionTask.rst.md
deleted file mode 100644
index edea0fa16..000000000
--- a/docs/generated/openml.tasks.OpenMLRegressionTask.rst.md
+++ /dev/null
@@ -1,13 +0,0 @@
-orphan
-
-:
-
-# `openml.tasks`{.interpreted-text role="mod"}.OpenMLRegressionTask
-
-::: currentmodule
-openml.tasks
-:::
-
-::: autoclass
-OpenMLRegressionTask
-:::
diff --git a/docs/generated/openml.tasks.OpenMLSplit.rst.md b/docs/generated/openml.tasks.OpenMLSplit.rst.md
deleted file mode 100644
index 893fddb49..000000000
--- a/docs/generated/openml.tasks.OpenMLSplit.rst.md
+++ /dev/null
@@ -1,13 +0,0 @@
-orphan
-
-:
-
-# `openml.tasks`{.interpreted-text role="mod"}.OpenMLSplit
-
-::: currentmodule
-openml.tasks
-:::
-
-::: autoclass
-OpenMLSplit
-:::
diff --git a/docs/generated/openml.tasks.OpenMLSupervisedTask.rst.md b/docs/generated/openml.tasks.OpenMLSupervisedTask.rst.md
deleted file mode 100644
index c8e327eea..000000000
--- a/docs/generated/openml.tasks.OpenMLSupervisedTask.rst.md
+++ /dev/null
@@ -1,13 +0,0 @@
-orphan
-
-:
-
-# `openml.tasks`{.interpreted-text role="mod"}.OpenMLSupervisedTask
-
-::: currentmodule
-openml.tasks
-:::
-
-::: autoclass
-OpenMLSupervisedTask
-:::
diff --git a/docs/generated/openml.tasks.OpenMLTask.rst.md b/docs/generated/openml.tasks.OpenMLTask.rst.md
deleted file mode 100644
index 467633e83..000000000
--- a/docs/generated/openml.tasks.OpenMLTask.rst.md
+++ /dev/null
@@ -1,13 +0,0 @@
-orphan
-
-:
-
-# `openml.tasks`{.interpreted-text role="mod"}.OpenMLTask
-
-::: currentmodule
-openml.tasks
-:::
-
-::: autoclass
-OpenMLTask
-:::
diff --git a/docs/generated/openml.tasks.TaskType.rst.md b/docs/generated/openml.tasks.TaskType.rst.md
deleted file mode 100644
index 5692454db..000000000
--- a/docs/generated/openml.tasks.TaskType.rst.md
+++ /dev/null
@@ -1,13 +0,0 @@
-orphan
-
-:
-
-# `openml.tasks`{.interpreted-text role="mod"}.TaskType
-
-::: currentmodule
-openml.tasks
-:::
-
-::: autoclass
-TaskType
-:::
diff --git a/docs/generated/openml.tasks.create_task.rst.md b/docs/generated/openml.tasks.create_task.rst.md
deleted file mode 100644
index c049a4e7c..000000000
--- a/docs/generated/openml.tasks.create_task.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.tasks`{.interpreted-text role="mod"}.create_task
-
-::: currentmodule
-openml.tasks
-:::
-
-::: autofunction
-create_task
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.tasks.delete_task.rst.md b/docs/generated/openml.tasks.delete_task.rst.md
deleted file mode 100644
index eb810dcb8..000000000
--- a/docs/generated/openml.tasks.delete_task.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.tasks`{.interpreted-text role="mod"}.delete_task
-
-::: currentmodule
-openml.tasks
-:::
-
-::: autofunction
-delete_task
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.tasks.get_task.rst.md b/docs/generated/openml.tasks.get_task.rst.md
deleted file mode 100644
index 985711a33..000000000
--- a/docs/generated/openml.tasks.get_task.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.tasks`{.interpreted-text role="mod"}.get_task
-
-::: currentmodule
-openml.tasks
-:::
-
-::: autofunction
-get_task
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.tasks.get_tasks.rst.md b/docs/generated/openml.tasks.get_tasks.rst.md
deleted file mode 100644
index 396be2131..000000000
--- a/docs/generated/openml.tasks.get_tasks.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.tasks`{.interpreted-text role="mod"}.get_tasks
-
-::: currentmodule
-openml.tasks
-:::
-
-::: autofunction
-get_tasks
-:::
-
-<div class="clearer"></div>
diff --git a/docs/generated/openml.tasks.list_tasks.rst.md b/docs/generated/openml.tasks.list_tasks.rst.md
deleted file mode 100644
index 1e24c7b9a..000000000
--- a/docs/generated/openml.tasks.list_tasks.rst.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# `openml.tasks`{.interpreted-text role="mod"}.list_tasks
-
-::: currentmodule
-openml.tasks
-:::
-
-::: autofunction
-list_tasks
-:::
-
-<div class="clearer"></div>
From 94d30249981520df08c85c8d6e7244fd40a31193 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 29 Jul 2024 12:22:08 +0200
Subject: [PATCH 05/25] remove orphan statements
---
docs/api.md | 4 ----
docs/extensions.md | 4 ----
docs/progress.md | 4 ----
3 files changed, 12 deletions(-)
diff --git a/docs/api.md b/docs/api.md
index 5ce9448e6..a55450279 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -1,7 +1,3 @@
-orphan
-
-:
-
# API
## Modules
diff --git a/docs/extensions.md b/docs/extensions.md
index dbe7b1940..f2aa230f5 100644
--- a/docs/extensions.md
+++ b/docs/extensions.md
@@ -1,7 +1,3 @@
-orphan
-
-:
-
# Extensions
OpenML-Python provides an extension interface to connect other machine
diff --git a/docs/progress.md b/docs/progress.md
index cc046a55b..c2923576b 100644
--- a/docs/progress.md
+++ b/docs/progress.md
@@ -1,7 +1,3 @@
-orphan
-
-:
-
# Changelog {#progress}
## next
From 6dcb538ebe4f0eb7c5f829ffa59826baf4000dd9 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 29 Jul 2024 13:49:48 +0200
Subject: [PATCH 06/25] Add plugin for apidoc generation
---
docs/api.md | 300 +++++++++++++++++++++++++------------------------
mkdocs.yml | 5 +
pyproject.toml | 3 +-
3 files changed, 158 insertions(+), 150 deletions(-)
diff --git a/docs/api.md b/docs/api.md
index a55450279..2b9f62374 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -2,219 +2,221 @@
## Modules
-### `openml.datasets`{.interpreted-text role="mod"}
+### `openml.datasets`
-::: {.automodule no-members="" no-inherited-members=""}
-openml.datasets
-:::
+::: openml.datasets.OpenMLDataset
-#### Dataset Classes
+[//]: # (::: {.automodule no-members="" no-inherited-members=""})
+[//]: # (openml.datasets)
+[//]: # (:::)
-::: currentmodule
-openml.datasets
-:::
+#### Dataset Classes
-::: {.autosummary toctree="generated/" template="class.rst"}
-OpenMLDataFeature OpenMLDataset
-:::
+[//]: # (::: currentmodule)
+[//]: # (openml.datasets)
+[//]: # (:::)
+[//]: # ()
+[//]: # (::: {.autosummary toctree="generated/" template="class.rst"})
+[//]: # (OpenMLDataFeature OpenMLDataset)
+[//]: # (:::)
#### Dataset Functions
-::: currentmodule
-openml.datasets
-:::
-
-::: {.autosummary toctree="generated/" template="function.rst"}
-attributes_arff_from_df check_datasets_active create_dataset
-delete_dataset get_dataset get_datasets list_datasets list_qualities
-status_update edit_dataset fork_dataset
-:::
+[//]: # (::: currentmodule)
+[//]: # (openml.datasets)
+[//]: # (:::)
+[//]: # ()
+[//]: # (::: {.autosummary toctree="generated/" template="function.rst"})
+[//]: # (attributes_arff_from_df check_datasets_active create_dataset)
+[//]: # (delete_dataset get_dataset get_datasets list_datasets list_qualities)
+[//]: # (status_update edit_dataset fork_dataset)
+[//]: # (:::)
### `openml.evaluations`{.interpreted-text role="mod"}
-::: {.automodule no-members="" no-inherited-members=""}
-openml.evaluations
-:::
+[//]: # (::: {.automodule no-members="" no-inherited-members=""})
+[//]: # (openml.evaluations)
+[//]: # (:::)
#### Evaluations Classes
-::: currentmodule
-openml.evaluations
-:::
-
-::: {.autosummary toctree="generated/" template="class.rst"}
-OpenMLEvaluation
-:::
-
+[//]: # (::: currentmodule)
+[//]: # (openml.evaluations)
+[//]: # (:::)
+[//]: # ()
+[//]: # (::: {.autosummary toctree="generated/" template="class.rst"})
+[//]: # (OpenMLEvaluation)
+[//]: # (:::)
+[//]: # ()
#### Evaluations Functions
-::: currentmodule
-openml.evaluations
-:::
-
-::: {.autosummary toctree="generated/" template="function.rst"}
-list_evaluations list_evaluation_measures list_evaluations_setups
-:::
+[//]: # (::: currentmodule)
+[//]: # (openml.evaluations)
+[//]: # (:::)
+[//]: # ()
+[//]: # (::: {.autosummary toctree="generated/" template="function.rst"})
+[//]: # (list_evaluations list_evaluation_measures list_evaluations_setups)
+[//]: # (:::)
### `openml.flows`{.interpreted-text role="mod"}: Flow Functions
-::: {.automodule no-members="" no-inherited-members=""}
-openml.flows
-:::
+[//]: # (::: {.automodule no-members="" no-inherited-members=""})
+[//]: # (openml.flows)
+[//]: # (:::)
#### Flow Classes
-::: currentmodule
-openml.flows
-:::
-
-::: {.autosummary toctree="generated/" template="class.rst"}
-OpenMLFlow
-:::
-
+[//]: # (::: currentmodule)
+[//]: # (openml.flows)
+[//]: # (:::)
+[//]: # ()
+[//]: # (::: {.autosummary toctree="generated/" template="class.rst"})
+[//]: # (OpenMLFlow)
+[//]: # (:::)
+[//]: # ()
#### Flow Functions
-::: currentmodule
-openml.flows
-:::
-
-::: {.autosummary toctree="generated/" template="function.rst"}
-assert_flows_equal delete_flow flow_exists get_flow list_flows
-:::
+[//]: # (::: currentmodule)
+[//]: # (openml.flows)
+[//]: # (:::)
+[//]: # ()
+[//]: # (::: {.autosummary toctree="generated/" template="function.rst"})
+[//]: # (assert_flows_equal delete_flow flow_exists get_flow list_flows)
+[//]: # (:::)
### `openml.runs`{.interpreted-text role="mod"}: Run Functions
-::: {.automodule no-members="" no-inherited-members=""}
-openml.runs
-:::
+[//]: # (::: {.automodule no-members="" no-inherited-members=""})
+[//]: # (openml.runs)
+[//]: # (:::)
#### Run Classes
-::: currentmodule
-openml.runs
-:::
-
-::: {.autosummary toctree="generated/" template="class.rst"}
-OpenMLRun
-:::
+[//]: # (::: currentmodule)
+[//]: # (openml.runs)
+[//]: # (:::)
+[//]: # ()
+[//]: # (::: {.autosummary toctree="generated/" template="class.rst"})
+[//]: # (OpenMLRun)
+[//]: # (:::)
#### Run Functions
-::: currentmodule
-openml.runs
-:::
-
-::: {.autosummary toctree="generated/" template="function.rst"}
-delete_run get_run get_runs get_run_trace initialize_model_from_run
-initialize_model_from_trace list_runs run_model_on_task run_flow_on_task
-run_exists
-:::
+[//]: # (::: currentmodule)
+[//]: # (openml.runs)
+[//]: # (:::)
+[//]: # ()
+[//]: # (::: {.autosummary toctree="generated/" template="function.rst"})
+[//]: # (delete_run get_run get_runs get_run_trace initialize_model_from_run)
+[//]: # (initialize_model_from_trace list_runs run_model_on_task run_flow_on_task)
+[//]: # (run_exists)
+[//]: # (:::)
### `openml.setups`{.interpreted-text role="mod"}: Setup Functions
-::: {.automodule no-members="" no-inherited-members=""}
-openml.setups
-:::
+[//]: # (::: {.automodule no-members="" no-inherited-members=""})
+[//]: # (openml.setups)
+[//]: # (:::)
#### Setup Classes
-::: currentmodule
-openml.setups
-:::
-
-::: {.autosummary toctree="generated/" template="class.rst"}
-OpenMLParameter OpenMLSetup
-:::
+[//]: # (::: currentmodule)
+[//]: # (openml.setups)
+[//]: # (:::)
+[//]: # ()
+[//]: # (::: {.autosummary toctree="generated/" template="class.rst"})
+[//]: # (OpenMLParameter OpenMLSetup)
+[//]: # (:::)
#### Setup Functions
-::: currentmodule
-openml.setups
-:::
-
-::: {.autosummary toctree="generated/" template="function.rst"}
-get_setup initialize_model list_setups setup_exists
-:::
+[//]: # (::: currentmodule)
+[//]: # (openml.setups)
+[//]: # (:::)
+[//]: # ()
+[//]: # (::: {.autosummary toctree="generated/" template="function.rst"})
+[//]: # (get_setup initialize_model list_setups setup_exists)
+[//]: # (:::)
### `openml.study`{.interpreted-text role="mod"}: Study Functions
-::: {.automodule no-members="" no-inherited-members=""}
-openml.study
-:::
+[//]: # (::: {.automodule no-members="" no-inherited-members=""})
+[//]: # (openml.study)
+[//]: # (:::)
#### Study Classes
-::: currentmodule
-openml.study
-:::
-
-::: {.autosummary toctree="generated/" template="class.rst"}
-OpenMLBenchmarkSuite OpenMLStudy
-:::
+[//]: # (::: currentmodule)
+[//]: # (openml.study)
+[//]: # (:::)
+[//]: # ()
+[//]: # (::: {.autosummary toctree="generated/" template="class.rst"})
+[//]: # (OpenMLBenchmarkSuite OpenMLStudy)
+[//]: # (:::)
#### Study Functions
-::: currentmodule
-openml.study
-:::
-
-::: {.autosummary toctree="generated/" template="function.rst"}
-attach_to_study attach_to_suite create_benchmark_suite create_study
-delete_study delete_suite detach_from_study detach_from_suite get_study
-get_suite list_studies list_suites update_study_status
-update_suite_status
-:::
-
+[//]: # (::: currentmodule)
+[//]: # (openml.study)
+[//]: # (:::)
+[//]: # ()
+[//]: # (::: {.autosummary toctree="generated/" template="function.rst"})
+[//]: # (attach_to_study attach_to_suite create_benchmark_suite create_study)
+[//]: # (delete_study delete_suite detach_from_study detach_from_suite get_study)
+[//]: # (get_suite list_studies list_suites update_study_status)
+[//]: # (update_suite_status)
+[//]: # (:::)
+[//]: # ()
### `openml.tasks`{.interpreted-text role="mod"}: Task Functions
-::: {.automodule no-members="" no-inherited-members=""}
-openml.tasks
-:::
+[//]: # (::: {.automodule no-members="" no-inherited-members=""})
+[//]: # (openml.tasks)
+[//]: # (:::)
#### Task Classes
-::: currentmodule
-openml.tasks
-:::
+[//]: # (::: currentmodule)
+[//]: # (openml.tasks)
+[//]: # (:::)
-::: {.autosummary toctree="generated/" template="class.rst"}
-OpenMLClassificationTask OpenMLClusteringTask OpenMLLearningCurveTask
-OpenMLRegressionTask OpenMLSplit OpenMLSupervisedTask OpenMLTask
-TaskType
-:::
+[//]: # (::: {.autosummary toctree="generated/" template="class.rst"})
+[//]: # (OpenMLClassificationTask OpenMLClusteringTask OpenMLLearningCurveTask)
+[//]: # (OpenMLRegressionTask OpenMLSplit OpenMLSupervisedTask OpenMLTask)
+[//]: # (TaskType)
+[//]: # (:::)
#### Task Functions
-::: currentmodule
-openml.tasks
-:::
-
-::: {.autosummary toctree="generated/" template="function.rst"}
-create_task delete_task get_task get_tasks list_tasks
-:::
+[//]: # (::: currentmodule)
+[//]: # (openml.tasks)
+[//]: # (:::)
+[//]: # ()
+[//]: # (::: {.autosummary toctree="generated/" template="function.rst"})
+[//]: # (create_task delete_task get_task get_tasks list_tasks)
+[//]: # (:::)
## Extensions {#api_extensions}
-::: {.automodule no-members="" no-inherited-members=""}
-openml.extensions
-:::
+[//]: # (::: {.automodule no-members="" no-inherited-members=""})
+[//]: # (openml.extensions)
+[//]: # (:::)
### Extension Classes
-::: currentmodule
-openml.extensions
-:::
-
-::: {.autosummary toctree="generated/" template="class.rst"}
-Extension sklearn.SklearnExtension
-:::
-
+[//]: # (::: currentmodule)
+[//]: # (openml.extensions)
+[//]: # (:::)
+[//]: # ()
+[//]: # (::: {.autosummary toctree="generated/" template="class.rst"})
+[//]: # (Extension sklearn.SklearnExtension)
+[//]: # (:::)
+[//]: # ()
### Extension Functions
-::: currentmodule
-openml.extensions
-:::
-
-::: {.autosummary toctree="generated/" template="function.rst"}
-get_extension_by_flow get_extension_by_model register_extension
-:::
+[//]: # (::: currentmodule)
+[//]: # (openml.extensions)
+[//]: # (:::)
+[//]: # ()
+[//]: # (::: {.autosummary toctree="generated/" template="function.rst"})
+[//]: # (get_extension_by_flow get_extension_by_model register_extension)
+[//]: # (:::)
diff --git a/mkdocs.yml b/mkdocs.yml
index 2a5239f2a..bed70b9db 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -15,3 +15,8 @@ markdown_extensions:
plugins:
- search
- autorefs
+ - mkdocstrings:
+ handlers:
+ python:
+ options:
+ docstring_style: numpy
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index fa8816e1a..2ecdcfd7c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -89,7 +89,8 @@ docs=[
"sphinx_bootstrap_theme",
"numpydoc",
"mkdocs-material",
- "mkdocs-autorefs"
+ "mkdocs-autorefs",
+ "mkdocstrings[python]",
]
[project.urls]
From 0ef96eefd5582ebc4e3781444090a4f0b7eb52a7 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 29 Jul 2024 14:17:15 +0200
Subject: [PATCH 07/25] Generate API documentation pages
---
mkdocs.yml | 5 ++++-
pyproject.toml | 1 +
scripts/gen_ref_pages.py | 26 ++++++++++++++++++++++++++
3 files changed, 31 insertions(+), 1 deletion(-)
create mode 100644 scripts/gen_ref_pages.py
diff --git a/mkdocs.yml b/mkdocs.yml
index bed70b9db..888fc2104 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -19,4 +19,7 @@ plugins:
handlers:
python:
options:
- docstring_style: numpy
\ No newline at end of file
+ docstring_style: numpy
+ - gen-files:
+ scripts:
+ - scripts/gen_ref_pages.py
diff --git a/pyproject.toml b/pyproject.toml
index 2ecdcfd7c..3f2b306d3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -91,6 +91,7 @@ docs=[
"mkdocs-material",
"mkdocs-autorefs",
"mkdocstrings[python]",
+ "mdocs-gen-files",
]
[project.urls]
diff --git a/scripts/gen_ref_pages.py b/scripts/gen_ref_pages.py
new file mode 100644
index 000000000..9664bfb51
--- /dev/null
+++ b/scripts/gen_ref_pages.py
@@ -0,0 +1,26 @@
+"""Generate the code reference pages."""
+
+from pathlib import Path
+
+import mkdocs_gen_files
+
+root = Path(__file__).parent.parent
+src = root / "openml"
+
+for path in sorted(src.rglob("*.py")):
+ module_path = path.relative_to(root).with_suffix("")
+ doc_path = path.relative_to(src).with_suffix(".md")
+ full_doc_path = Path("reference", doc_path)
+
+ parts = tuple(module_path.parts)
+
+ if parts[-1] == "__init__":
+ parts = parts[:-1]
+ elif parts[-1] == "__main__":
+ continue
+
+ with mkdocs_gen_files.open(full_doc_path, "w") as fd:
+ identifier = ".".join(parts)
+ print("::: " + identifier, file=fd)
+
+ mkdocs_gen_files.set_edit_path(full_doc_path, path.relative_to(root))
From f9dd346f939bc5965c896db579cf8ef4bad672dc Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 29 Jul 2024 14:38:36 +0200
Subject: [PATCH 08/25] Add API pages to nav, use section-index for init
---
mkdocs.yml | 7 +++++++
pyproject.toml | 4 +++-
scripts/gen_ref_pages.py | 16 +++++++++++++++-
3 files changed, 25 insertions(+), 2 deletions(-)
diff --git a/mkdocs.yml b/mkdocs.yml
index 888fc2104..4355b5bb2 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -4,6 +4,10 @@ theme:
features:
- content.code.copy
+nav:
+ - Code Reference: reference/
+
+
markdown_extensions:
- pymdownx.highlight:
anchor_linenums: true
@@ -15,6 +19,7 @@ markdown_extensions:
plugins:
- search
- autorefs
+ - section-index
- mkdocstrings:
handlers:
python:
@@ -23,3 +28,5 @@ plugins:
- gen-files:
scripts:
- scripts/gen_ref_pages.py
+ - literate-nav:
+ nav_file: SUMMARY.md
diff --git a/pyproject.toml b/pyproject.toml
index 3f2b306d3..7dedcdb01 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -91,7 +91,9 @@ docs=[
"mkdocs-material",
"mkdocs-autorefs",
"mkdocstrings[python]",
- "mdocs-gen-files",
+ "mkdocs-gen-files",
+ "mkdocs-literate-nav",
+ "mkdocs-section-index",
]
[project.urls]
diff --git a/scripts/gen_ref_pages.py b/scripts/gen_ref_pages.py
index 9664bfb51..608bb4993 100644
--- a/scripts/gen_ref_pages.py
+++ b/scripts/gen_ref_pages.py
@@ -1,9 +1,16 @@
-"""Generate the code reference pages."""
+"""Generate the code reference pages.
+
+based on https://github.com/mkdocstrings/mkdocstrings/blob/33aa573efb17b13e7b9da77e29aeccb3fbddd8e8/docs/recipes.md
+but modified for lack of "src/" file structure.
+
+"""
from pathlib import Path
import mkdocs_gen_files
+nav = mkdocs_gen_files.Nav()
+
root = Path(__file__).parent.parent
src = root / "openml"
@@ -16,11 +23,18 @@
if parts[-1] == "__init__":
parts = parts[:-1]
+ doc_path = doc_path.with_name("index.md")
+ full_doc_path = full_doc_path.with_name("index.md")
elif parts[-1] == "__main__":
continue
+ nav[parts] = doc_path.as_posix()
+
with mkdocs_gen_files.open(full_doc_path, "w") as fd:
identifier = ".".join(parts)
print("::: " + identifier, file=fd)
mkdocs_gen_files.set_edit_path(full_doc_path, path.relative_to(root))
+
+ with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as nav_file:
+ nav_file.writelines(nav.build_literate_nav())
From 12a5066cdfe079f2402e71f5e6ef0d014dfca866 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 29 Jul 2024 14:42:19 +0200
Subject: [PATCH 09/25] Remove examples doc as they are generated
---
docs/examples/20_basic/index.rst.md | 65 -
.../20_basic/introduction_tutorial.rst.md | 229 ---
.../20_basic/sg_execution_times.rst.md | 21 -
.../20_basic/simple_datasets_tutorial.rst.md | 173 ---
.../simple_flows_and_runs_tutorial.rst.md | 174 ---
.../20_basic/simple_suites_tutorial.rst.md | 188 ---
.../30_extended/configure_logging.rst.md | 107 --
.../30_extended/create_upload_tutorial.rst.md | 460 ------
docs/examples/30_extended/custom_flow_.rst.md | 294 ----
.../30_extended/datasets_tutorial.rst.md | 1040 -------------
.../fetch_evaluations_tutorial.rst.md | 360 -----
.../fetch_runtimes_tutorial.rst.md | 1135 ---------------
.../30_extended/flow_id_tutorial.rst.md | 208 ---
.../flows_and_runs_tutorial.rst.md | 604 --------
docs/examples/30_extended/index.rst.md | 205 ---
.../plot_svm_hyperparameters_tutorial.rst.md | 235 ---
.../30_extended/run_setup_tutorial.rst.md | 215 ---
.../30_extended/sg_execution_times.rst.md | 51 -
.../30_extended/study_tutorial.rst.md | 402 -----
.../30_extended/suites_tutorial.rst.md | 291 ----
.../task_manual_iteration_tutorial.rst.md | 522 -------
.../30_extended/tasks_tutorial.rst.md | 1292 -----------------
.../2015_neurips_feurer_example.rst.md | 167 ---
.../40_paper/2018_ida_strang_example.rst.md | 191 ---
docs/examples/index.md | 1 -
25 files changed, 8630 deletions(-)
delete mode 100644 docs/examples/20_basic/index.rst.md
delete mode 100644 docs/examples/20_basic/introduction_tutorial.rst.md
delete mode 100644 docs/examples/20_basic/sg_execution_times.rst.md
delete mode 100644 docs/examples/20_basic/simple_datasets_tutorial.rst.md
delete mode 100644 docs/examples/20_basic/simple_flows_and_runs_tutorial.rst.md
delete mode 100644 docs/examples/20_basic/simple_suites_tutorial.rst.md
delete mode 100644 docs/examples/30_extended/configure_logging.rst.md
delete mode 100644 docs/examples/30_extended/create_upload_tutorial.rst.md
delete mode 100644 docs/examples/30_extended/custom_flow_.rst.md
delete mode 100644 docs/examples/30_extended/datasets_tutorial.rst.md
delete mode 100644 docs/examples/30_extended/fetch_evaluations_tutorial.rst.md
delete mode 100644 docs/examples/30_extended/fetch_runtimes_tutorial.rst.md
delete mode 100644 docs/examples/30_extended/flow_id_tutorial.rst.md
delete mode 100644 docs/examples/30_extended/flows_and_runs_tutorial.rst.md
delete mode 100644 docs/examples/30_extended/index.rst.md
delete mode 100644 docs/examples/30_extended/plot_svm_hyperparameters_tutorial.rst.md
delete mode 100644 docs/examples/30_extended/run_setup_tutorial.rst.md
delete mode 100644 docs/examples/30_extended/sg_execution_times.rst.md
delete mode 100644 docs/examples/30_extended/study_tutorial.rst.md
delete mode 100644 docs/examples/30_extended/suites_tutorial.rst.md
delete mode 100644 docs/examples/30_extended/task_manual_iteration_tutorial.rst.md
delete mode 100644 docs/examples/30_extended/tasks_tutorial.rst.md
delete mode 100644 docs/examples/40_paper/2015_neurips_feurer_example.rst.md
delete mode 100644 docs/examples/40_paper/2018_ida_strang_example.rst.md
delete mode 100644 docs/examples/index.md
diff --git a/docs/examples/20_basic/index.rst.md b/docs/examples/20_basic/index.rst.md
deleted file mode 100644
index 2f29baa0f..000000000
--- a/docs/examples/20_basic/index.rst.md
+++ /dev/null
@@ -1,65 +0,0 @@
-# Introductory Examples {#sphx_glr_examples_20_basic}
-
-Introductory examples to the usage of the OpenML python connector.
-
-<div class="sphx-glr-thumbnails">
-<div class="sphx-glr-thumbcontainer" tooltip="This is a brief showcase of OpenML benchmark suites, which were introduced by Bischl et al. (20...">
-
-::: only
-html
-
-
-
-`sphx_glr_examples_20_basic_simple_suites_tutorial.py`{.interpreted-text
-role="ref"}
-:::
-
-<div class="sphx-glr-thumbnail-title">Benchmark suites</div>
-</div>
-<div class="sphx-glr-thumbcontainer" tooltip="A simple tutorial on how to train/run a model and how to upload the results.">
-
-::: only
-html
-
-
-
-`sphx_glr_examples_20_basic_simple_flows_and_runs_tutorial.py`{.interpreted-text
-role="ref"}
-:::
-
-<div class="sphx-glr-thumbnail-title">Flows and Runs</div>
-</div>
-<div class="sphx-glr-thumbcontainer" tooltip="An example how to set up OpenML-Python followed up by a simple example.">
-
-::: only
-html
-
-
-
-`sphx_glr_examples_20_basic_introduction_tutorial.py`{.interpreted-text
-role="ref"}
-:::
-
-<div class="sphx-glr-thumbnail-title">Introduction tutorial & Setup</div>
-</div>
-<div class="sphx-glr-thumbcontainer" tooltip="A basic tutorial on how to list, load and visualize datasets.">
-
-::: only
-html
-
-
-
-`sphx_glr_examples_20_basic_simple_datasets_tutorial.py`{.interpreted-text
-role="ref"}
-:::
-
-<div class="sphx-glr-thumbnail-title">Datasets</div>
-</div>
-</div>
-
-::: {.toctree hidden=""}
-/examples/20_basic/simple_suites_tutorial
-/examples/20_basic/simple_flows_and_runs_tutorial
-/examples/20_basic/introduction_tutorial
-/examples/20_basic/simple_datasets_tutorial
-:::
diff --git a/docs/examples/20_basic/introduction_tutorial.rst.md b/docs/examples/20_basic/introduction_tutorial.rst.md
deleted file mode 100644
index 56903b1f0..000000000
--- a/docs/examples/20_basic/introduction_tutorial.rst.md
+++ /dev/null
@@ -1,229 +0,0 @@
-::::: only
-html
-
-:::: {.note .sphx-glr-download-link-note}
-::: title
-Note
-:::
-
-`Go to the end <sphx_glr_download_examples_20_basic_introduction_tutorial.py>`{.interpreted-text
-role="ref"} to download the full example code
-::::
-:::::
-
-::: rst-class
-sphx-glr-example-title
-:::
-
-# Introduction tutorial & Setup {#sphx_glr_examples_20_basic_introduction_tutorial.py}
-
-An example how to set up OpenML-Python followed up by a simple example.
-
-OpenML is an online collaboration platform for machine learning which
-allows you to:
-
-- Find or share interesting, well-documented datasets
-- Define research / modelling goals (tasks)
-- Explore large amounts of machine learning algorithms, with APIs in
- Java, R, Python
-- Log and share reproducible experiments, models, results
-- Works seamlessly with scikit-learn and other libraries
-- Large scale benchmarking, compare to state of the art
-
-## Installation
-
-Installation is done via `pip`:
-
-``` bash
-pip install openml
-```
-
-For further information, please check out the installation guide at
-`installation`{.interpreted-text role="ref"}.
-
-## Authentication
-
-The OpenML server can only be accessed by users who have signed up on
-the OpenML platform. If you don't have an account yet, sign up now. You
-will receive an API key, which will authenticate you to the server and
-allow you to download and upload datasets, tasks, runs and flows.
-
-- Create an OpenML account (free) on <https://www.openml.org>.
-- After logging in, open your account page (avatar on the top right)
-- Open \'Account Settings\', then \'API authentication\' to find your
- API key.
-
-There are two ways to permanently authenticate:
-
-- Use the `openml` CLI tool with `openml configure apikey MYKEY`,
- replacing **MYKEY** with your API key.
-- Create a plain text file **\~/.openml/config** with the line
- **\'apikey=MYKEY\'**, replacing **MYKEY** with your API key. The
- config file must be in the directory \~/.openml/config and exist
- prior to importing the openml module.
-
-Alternatively, by running the code below and replacing \'YOURKEY\' with
-your API key, you authenticate for the duration of the python process.
-
-``` default
-# License: BSD 3-Clause
-
-import openml
-from sklearn import neighbors
-```
-
-:::: warning
-::: title
-Warning
-:::
-
-.. include:: ../../test_server_usage_warning.txt
-::::
-
-``` default
-openml.config.start_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!
- warnings.warn(
-```
-:::
-
-When using the main server instead, make sure your apikey is configured.
-This can be done with the following line of code (uncomment it!). Never
-share your apikey with others.
-
-``` default
-# openml.config.apikey = 'YOURKEY'
-```
-
-## Caching
-
-When downloading datasets, tasks, runs and flows, they will be cached to
-retrieve them without calling the server later. As with the API key, the
-cache directory can be either specified through the config file or
-through the API:
-
-- Add the line **cachedir = \'MYDIR\'** to the config file, replacing
- \'MYDIR\' with the path to the cache directory. By default, OpenML
- will use **\~/.openml/cache** as the cache directory.
-- Run the code below, replacing \'YOURDIR\' with the path to the cache
- directory.
-
-``` default
-# Uncomment and set your OpenML cache directory
-# import os
-# openml.config.cache_directory = os.path.expanduser('YOURDIR')
-```
-
-## Simple Example
-
-Download the OpenML task for the eeg-eye-state.
-
-``` default
-task = openml.tasks.get_task(403)
-data = openml.datasets.get_dataset(task.dataset_id)
-clf = neighbors.KNeighborsClassifier(n_neighbors=5)
-run = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False)
-# Publish the experiment on OpenML (optional, requires an API key).
-# For this tutorial, our configuration publishes to the test server
-# as to not crowd the main server with runs created by examples.
-myrun = run.publish()
-print(f"kNN on {data.name}: {myrun.openml_url}")
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-kNN on eeg-eye-state: https://test.openml.org/r/7327
-```
-:::
-
-``` default
-openml.config.stop_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-timing
-
-**Total running time of the script:** ( 0 minutes 8.000 seconds)
-:::
-
-::::::: {#sphx_glr_download_examples_20_basic_introduction_tutorial.py}
-:::::: only
-html
-
-::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
-::: {.container .sphx-glr-download .sphx-glr-download-python}
-`Download Python source code: introduction_tutorial.py <introduction_tutorial.py>`{.interpreted-text
-role="download"}
-:::
-
-::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
-`Download Jupyter notebook: introduction_tutorial.ipynb <introduction_tutorial.ipynb>`{.interpreted-text
-role="download"}
-:::
-:::::
-::::::
-:::::::
-
-:::: only
-html
-
-::: rst-class
-sphx-glr-signature
-
-[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
-:::
-::::
diff --git a/docs/examples/20_basic/sg_execution_times.rst.md b/docs/examples/20_basic/sg_execution_times.rst.md
deleted file mode 100644
index 5017d8cee..000000000
--- a/docs/examples/20_basic/sg_execution_times.rst.md
+++ /dev/null
@@ -1,21 +0,0 @@
-orphan
-
-:
-
-# Computation times {#sphx_glr_examples_20_basic_sg_execution_times}
-
-**00:27.928** total execution time for **examples_20_basic** files:
-
- ---------------------------------------------------------------------------------- ------------ -----
- `sphx_glr_examples_20_basic_simple_suites_tutorial.py`{.interpreted-text 00:09.728 0.0
- role="ref"} (`simple_suites_tutorial.py`) MB
-
- `sphx_glr_examples_20_basic_introduction_tutorial.py`{.interpreted-text 00:07.1000 0.0
- role="ref"} (`introduction_tutorial.py`) MB
-
- `sphx_glr_examples_20_basic_simple_datasets_tutorial.py`{.interpreted-text 00:06.402 0.0
- role="ref"} (`simple_datasets_tutorial.py`) MB
-
- `sphx_glr_examples_20_basic_simple_flows_and_runs_tutorial.py`{.interpreted-text 00:03.799 0.0
- role="ref"} (`simple_flows_and_runs_tutorial.py`) MB
- ---------------------------------------------------------------------------------- ------------ -----
diff --git a/docs/examples/20_basic/simple_datasets_tutorial.rst.md b/docs/examples/20_basic/simple_datasets_tutorial.rst.md
deleted file mode 100644
index 326ca1f2d..000000000
--- a/docs/examples/20_basic/simple_datasets_tutorial.rst.md
+++ /dev/null
@@ -1,173 +0,0 @@
-::::: only
-html
-
-:::: {.note .sphx-glr-download-link-note}
-::: title
-Note
-:::
-
-`Go to the end <sphx_glr_download_examples_20_basic_simple_datasets_tutorial.py>`{.interpreted-text
-role="ref"} to download the full example code
-::::
-:::::
-
-::: rst-class
-sphx-glr-example-title
-:::
-
-# Datasets {#sphx_glr_examples_20_basic_simple_datasets_tutorial.py}
-
-A basic tutorial on how to list, load and visualize datasets.
-
-In general, we recommend working with tasks, so that the results can be
-easily reproduced. Furthermore, the results can be compared to existing
-results at OpenML. However, for the purposes of this tutorial, we are
-going to work with the datasets directly.
-
-``` default
-# License: BSD 3-Clause
-
-import openml
-```
-
-## List datasets
-
-``` default
-datasets_df = openml.datasets.list_datasets(output_format="dataframe")
-print(datasets_df.head(n=10))
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-did name ... NumberOfNumericFeatures NumberOfSymbolicFeatures
-2 2 anneal ... 6.0 33.0
-3 3 kr-vs-kp ... 0.0 37.0
-4 4 labor ... 8.0 9.0
-5 5 arrhythmia ... 206.0 74.0
-6 6 letter ... 16.0 1.0
-7 7 audiology ... 0.0 70.0
-8 8 liver-disorders ... 6.0 0.0
-9 9 autos ... 15.0 11.0
-10 10 lymph ... 3.0 16.0
-11 11 balance-scale ... 4.0 1.0
-
-[10 rows x 16 columns]
-```
-:::
-
-## Download a dataset
-
-``` default
-# Iris dataset https://www.openml.org/d/61
-dataset = openml.datasets.get_dataset(61)
-
-# Print a summary
-print(
- f"This is dataset '{dataset.name}', the target feature is "
- f"'{dataset.default_target_attribute}'"
-)
-print(f"URL: {dataset.url}")
-print(dataset.description[:500])
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-This is dataset 'iris', the target feature is 'class'
-URL: https://api.openml.org/data/v1/download/61/iris.arff
-**Author**: R.A. Fisher
-**Source**: [UCI](https://archive.ics.uci.edu/ml/datasets/Iris) - 1936 - Donated by Michael Marshall
-**Please cite**:
-
-**Iris Plants Database**
-This is perhaps the best known database to be found in the pattern recognition literature. Fisher's paper is a classic in the field and is referenced frequently to this day. (See Duda & Hart, for example.) The data set contains 3 classes of 50 instances each, where each class refers to a type of iris plant. One class
-```
-:::
-
-## Load a dataset
-
-``` default
-# X - An array/dataframe where each row represents one example with
-# the corresponding feature values.
-# y - the classes for each example
-# categorical_indicator - an array that indicates which feature is categorical
-# attribute_names - the names of the features for the examples (X) and
-# target feature (y)
-X, y, categorical_indicator, attribute_names = dataset.get_data(
- dataset_format="dataframe", target=dataset.default_target_attribute
-)
-```
-
-## Visualize the dataset
-
-``` default
-import pandas as pd
-import seaborn as sns
-import matplotlib.pyplot as plt
-
-sns.set_style("darkgrid")
-
-
-def hide_current_axis(*args, **kwds):
- plt.gca().set_visible(False)
-
-
-# We combine all the data so that we can map the different
-# examples to different colors according to the classes.
-combined_data = pd.concat([X, y], axis=1)
-iris_plot = sns.pairplot(combined_data, hue="class")
-iris_plot.map_upper(hide_current_axis)
-plt.show()
-```
-
-::: {.image-sg .sphx-glr-single-img alt="simple datasets tutorial" srcset="/examples/20_basic/images/sphx_glr_simple_datasets_tutorial_001.png"}
-/examples/20_basic/images/sphx_glr_simple_datasets_tutorial_001.png
-:::
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/openml/venv/lib/python3.10/site-packages/seaborn/axisgrid.py:118: UserWarning: The figure layout has changed to tight
- self._figure.tight_layout(*args, **kwargs)
-```
-:::
-
-::: rst-class
-sphx-glr-timing
-
-**Total running time of the script:** ( 0 minutes 6.402 seconds)
-:::
-
-::::::: {#sphx_glr_download_examples_20_basic_simple_datasets_tutorial.py}
-:::::: only
-html
-
-::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
-::: {.container .sphx-glr-download .sphx-glr-download-python}
-`Download Python source code: simple_datasets_tutorial.py <simple_datasets_tutorial.py>`{.interpreted-text
-role="download"}
-:::
-
-::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
-`Download Jupyter notebook: simple_datasets_tutorial.ipynb <simple_datasets_tutorial.ipynb>`{.interpreted-text
-role="download"}
-:::
-:::::
-::::::
-:::::::
-
-:::: only
-html
-
-::: rst-class
-sphx-glr-signature
-
-[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
-:::
-::::
diff --git a/docs/examples/20_basic/simple_flows_and_runs_tutorial.rst.md b/docs/examples/20_basic/simple_flows_and_runs_tutorial.rst.md
deleted file mode 100644
index 960d65e37..000000000
--- a/docs/examples/20_basic/simple_flows_and_runs_tutorial.rst.md
+++ /dev/null
@@ -1,174 +0,0 @@
-::::: only
-html
-
-:::: {.note .sphx-glr-download-link-note}
-::: title
-Note
-:::
-
-`Go to the end <sphx_glr_download_examples_20_basic_simple_flows_and_runs_tutorial.py>`{.interpreted-text
-role="ref"} to download the full example code
-::::
-:::::
-
-::: rst-class
-sphx-glr-example-title
-:::
-
-# Flows and Runs {#sphx_glr_examples_20_basic_simple_flows_and_runs_tutorial.py}
-
-A simple tutorial on how to train/run a model and how to upload the
-results.
-
-``` default
-# License: BSD 3-Clause
-
-import openml
-from sklearn import ensemble, neighbors
-```
-
-:::: warning
-::: title
-Warning
-:::
-
-.. include:: ../../test_server_usage_warning.txt
-::::
-
-``` default
-openml.config.start_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!
- warnings.warn(
-```
-:::
-
-# Train a machine learning model
-
-``` default
-# NOTE: We are using dataset 20 from the test server: https://test.openml.org/d/20
-dataset = openml.datasets.get_dataset(20)
-X, y, categorical_indicator, attribute_names = dataset.get_data(
- target=dataset.default_target_attribute
-)
-clf = neighbors.KNeighborsClassifier(n_neighbors=3)
-clf.fit(X, y)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-```
-:::
-
-<div class="output_subarea output_html rendered_html output_result">
-<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: "▸";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: "▾";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: "";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: "";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: "";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id="sk-container-id-1" class="sk-top-container"><div class="sk-text-repr-fallback"><pre>KNeighborsClassifier(n_neighbors=3)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class="sk-container" hidden><div class="sk-item"><div class="sk-estimator sk-toggleable"><input class="sk-toggleable__control sk-hidden--visually" id="sk-estimator-id-1" type="checkbox" checked><label for="sk-estimator-id-1" class="sk-toggleable__label sk-toggleable__label-arrow">KNeighborsClassifier</label><div class="sk-toggleable__content"><pre>KNeighborsClassifier(n_neighbors=3)</pre></div></div></div></div></div>
-</div>
-<br />
-<br />
-
-# Running a model on a task
-
-``` default
-task = openml.tasks.get_task(119)
-clf = ensemble.RandomForestClassifier()
-run = openml.runs.run_model_on_task(clf, task)
-print(run)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-OpenML Run
-==========
-Uploader Name...................: None
-Metric..........................: None
-Local Result - Accuracy (+- STD): 0.7510 +- 0.0000
-Local Runtime - ms (+- STD).....: 95.9373 +- 0.0000
-Run ID..........................: None
-Task ID.........................: 119
-Task Type.......................: None
-Task URL........................: https://test.openml.org/t/119
-Flow ID.........................: 4417
-Flow Name.......................: sklearn.ensemble._forest.RandomForestClassifier
-Flow URL........................: https://test.openml.org/f/4417
-Setup ID........................: None
-Setup String....................: Python_3.10.12. Sklearn_1.3.0. NumPy_1.25.1. SciPy_1.11.1.
-Dataset ID......................: 20
-Dataset URL.....................: https://test.openml.org/d/20
-```
-:::
-
-# Publishing the run
-
-``` default
-myrun = run.publish()
-print(f"Run was uploaded to {myrun.openml_url}")
-print(f"The flow can be found at {myrun.flow.openml_url}")
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-Run was uploaded to https://test.openml.org/r/7325
-The flow can be found at https://test.openml.org/f/4417
-```
-:::
-
-``` default
-openml.config.stop_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-timing
-
-**Total running time of the script:** ( 0 minutes 3.799 seconds)
-:::
-
-::::::: {#sphx_glr_download_examples_20_basic_simple_flows_and_runs_tutorial.py}
-:::::: only
-html
-
-::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
-::: {.container .sphx-glr-download .sphx-glr-download-python}
-`Download Python source code: simple_flows_and_runs_tutorial.py <simple_flows_and_runs_tutorial.py>`{.interpreted-text
-role="download"}
-:::
-
-::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
-`Download Jupyter notebook: simple_flows_and_runs_tutorial.ipynb <simple_flows_and_runs_tutorial.ipynb>`{.interpreted-text
-role="download"}
-:::
-:::::
-::::::
-:::::::
-
-:::: only
-html
-
-::: rst-class
-sphx-glr-signature
-
-[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
-:::
-::::
diff --git a/docs/examples/20_basic/simple_suites_tutorial.rst.md b/docs/examples/20_basic/simple_suites_tutorial.rst.md
deleted file mode 100644
index e636a3a38..000000000
--- a/docs/examples/20_basic/simple_suites_tutorial.rst.md
+++ /dev/null
@@ -1,188 +0,0 @@
-::::: only
-html
-
-:::: {.note .sphx-glr-download-link-note}
-::: title
-Note
-:::
-
-`Go to the end <sphx_glr_download_examples_20_basic_simple_suites_tutorial.py>`{.interpreted-text
-role="ref"} to download the full example code
-::::
-:::::
-
-::: rst-class
-sphx-glr-example-title
-:::
-
-# Benchmark suites {#sphx_glr_examples_20_basic_simple_suites_tutorial.py}
-
-This is a brief showcase of OpenML benchmark suites, which were
-introduced by [Bischl et al.
-(2019)](https://arxiv.org/abs/1708.03731v2). Benchmark suites
-standardize the datasets and splits to be used in an experiment or
-paper. They are fully integrated into OpenML and simplify both the
-sharing of the setup and the results.
-
-``` default
-# License: BSD 3-Clause
-
-import openml
-```
-
-## OpenML-CC18
-
-As an example we have a look at the OpenML-CC18, which is a suite of 72
-classification datasets from OpenML which were carefully selected to be
-usable by many algorithms and also represent datasets commonly used in
-machine learning research. These are all datasets from mid-2018 that
-satisfy a large set of clear requirements for thorough yet practical
-benchmarking:
-
-1. the number of observations are between 500 and 100,000 to focus on
- medium-sized datasets,
-2. the number of features does not exceed 5,000 features to keep the
- runtime of the algorithms low
-3. the target attribute has at least two classes with no class having
- less than 20 observations
-4. the ratio of the minority class and the majority class is above 0.05
- (to eliminate highly imbalanced datasets which require special
- treatment for both algorithms and evaluation measures).
-
-A full description can be found in the [OpenML benchmarking
-docs](https://docs.openml.org/benchmark/#openml-cc18).
-
-In this example we\'ll focus on how to use benchmark suites in practice.
-
-## Downloading benchmark suites
-
-``` default
-suite = openml.study.get_suite(99)
-print(suite)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-OpenML Benchmark Suite
-======================
-ID..............: 99
-Name............: OpenML-CC18 Curated Classification benchmark
-Status..........: active
-Main Entity Type: task
-Study URL.......: https://www.openml.org/s/99
-# of Data.......: 72
-# of Tasks......: 72
-Creator.........: https://www.openml.org/u/1
-Upload Time.....: 2019-02-21 18:47:13
-```
-:::
-
-The benchmark suite does not download the included tasks and datasets
-itself, but only contains a list of which tasks constitute the study.
-
-Tasks can then be accessed via
-
-``` default
-tasks = suite.tasks
-print(tasks)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-[3, 6, 11, 12, 14, 15, 16, 18, 22, 23, 28, 29, 31, 32, 37, 43, 45, 49, 53, 219, 2074, 2079, 3021, 3022, 3481, 3549, 3560, 3573, 3902, 3903, 3904, 3913, 3917, 3918, 7592, 9910, 9946, 9952, 9957, 9960, 9964, 9971, 9976, 9977, 9978, 9981, 9985, 10093, 10101, 14952, 14954, 14965, 14969, 14970, 125920, 125922, 146195, 146800, 146817, 146819, 146820, 146821, 146822, 146824, 146825, 167119, 167120, 167121, 167124, 167125, 167140, 167141]
-```
-:::
-
-and iterated over for benchmarking. For speed reasons we only iterate
-over the first three tasks:
-
-``` default
-for task_id in tasks[:3]:
- task = openml.tasks.get_task(task_id)
- print(task)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-OpenML Classification Task
-==========================
-Task Type Description: https://www.openml.org/tt/TaskType.SUPERVISED_CLASSIFICATION
-Task ID..............: 3
-Task URL.............: https://www.openml.org/t/3
-Estimation Procedure.: crossvalidation
-Target Feature.......: class
-# of Classes.........: 2
-Cost Matrix..........: Available
-OpenML Classification Task
-==========================
-Task Type Description: https://www.openml.org/tt/TaskType.SUPERVISED_CLASSIFICATION
-Task ID..............: 6
-Task URL.............: https://www.openml.org/t/6
-Estimation Procedure.: crossvalidation
-Target Feature.......: class
-# of Classes.........: 26
-Cost Matrix..........: Available
-OpenML Classification Task
-==========================
-Task Type Description: https://www.openml.org/tt/TaskType.SUPERVISED_CLASSIFICATION
-Task ID..............: 11
-Task URL.............: https://www.openml.org/t/11
-Estimation Procedure.: crossvalidation
-Target Feature.......: class
-# of Classes.........: 3
-Cost Matrix..........: Available
-```
-:::
-
-## Further examples
-
-- `sphx_glr_examples_30_extended_suites_tutorial.py`{.interpreted-text
- role="ref"}
-- `sphx_glr_examples_30_extended_study_tutorial.py`{.interpreted-text
- role="ref"}
-- `sphx_glr_examples_40_paper_2018_ida_strang_example.py`{.interpreted-text
- role="ref"}
-
-::: rst-class
-sphx-glr-timing
-
-**Total running time of the script:** ( 0 minutes 9.728 seconds)
-:::
-
-::::::: {#sphx_glr_download_examples_20_basic_simple_suites_tutorial.py}
-:::::: only
-html
-
-::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
-::: {.container .sphx-glr-download .sphx-glr-download-python}
-`Download Python source code: simple_suites_tutorial.py <simple_suites_tutorial.py>`{.interpreted-text
-role="download"}
-:::
-
-::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
-`Download Jupyter notebook: simple_suites_tutorial.ipynb <simple_suites_tutorial.ipynb>`{.interpreted-text
-role="download"}
-:::
-:::::
-::::::
-:::::::
-
-:::: only
-html
-
-::: rst-class
-sphx-glr-signature
-
-[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
-:::
-::::
diff --git a/docs/examples/30_extended/configure_logging.rst.md b/docs/examples/30_extended/configure_logging.rst.md
deleted file mode 100644
index 3ba0d4ab5..000000000
--- a/docs/examples/30_extended/configure_logging.rst.md
+++ /dev/null
@@ -1,107 +0,0 @@
-::::: only
-html
-
-:::: {.note .sphx-glr-download-link-note}
-::: title
-Note
-:::
-
-`Go to the end <sphx_glr_download_examples_30_extended_configure_logging.py>`{.interpreted-text
-role="ref"} to download the full example code
-::::
-:::::
-
-::: rst-class
-sphx-glr-example-title
-:::
-
-# Logging {#sphx_glr_examples_30_extended_configure_logging.py}
-
-Explains openml-python logging, and shows how to configure it.
-
-Openml-python uses the [Python logging
-module](https://docs.python.org/3/library/logging.html) to provide users
-with log messages. Each log message is assigned a level of importance,
-see the table in Python\'s logging tutorial
-[here](https://docs.python.org/3/howto/logging.html#when-to-use-logging).
-
-By default, openml-python will print log messages of level
-[WARNING]{.title-ref} and above to console. All log messages (including
-[DEBUG]{.title-ref} and [INFO]{.title-ref}) are also saved in a file,
-which can be found in your cache directory (see also the
-`sphx_glr_examples_20_basic_introduction_tutorial.py`{.interpreted-text
-role="ref"}). These file logs are automatically deleted if needed, and
-use at most 2MB of space.
-
-It is possible to configure what log levels to send to console and file.
-When downloading a dataset from OpenML, a [DEBUG]{.title-ref}-level
-message is written:
-
-``` default
-# License: BSD 3-Clause
-
-import openml
-
-openml.datasets.get_dataset("iris")
-
-# With default configuration, the above example will show no output to console.
-# However, in your cache directory you should find a file named 'openml_python.log',
-# which has a DEBUG message written to it. It should be either like
-# "[DEBUG] [10:46:19:openml.datasets.dataset] Saved dataset 61: iris to file ..."
-# or like
-# "[DEBUG] [10:49:38:openml.datasets.dataset] Data pickle file already exists and is up to date."
-# , depending on whether or not you had downloaded iris before.
-# The processed log levels can be configured programmatically:
-
-import logging
-
-openml.config.set_console_log_level(logging.DEBUG)
-openml.config.set_file_log_level(logging.WARNING)
-openml.datasets.get_dataset("iris")
-
-# Now the log level that was previously written to file should also be shown in the console.
-# The message is now no longer written to file as the `file_log` was set to level `WARNING`.
-#
-# It is also possible to specify the desired log levels through the configuration file.
-# This way you will not need to set them on each script separately.
-# Add the line **verbosity = NUMBER** and/or **file_verbosity = NUMBER** to the config file,
-# where 'NUMBER' should be one of:
-#
-# * 0: `logging.WARNING` and up.
-# * 1: `logging.INFO` and up.
-# * 2: `logging.DEBUG` and up (i.e. all messages).
-```
-
-::: rst-class
-sphx-glr-timing
-
-**Total running time of the script:** ( 0 minutes 0.000 seconds)
-:::
-
-::::::: {#sphx_glr_download_examples_30_extended_configure_logging.py}
-:::::: only
-html
-
-::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
-::: {.container .sphx-glr-download .sphx-glr-download-python}
-`Download Python source code: configure_logging.py <configure_logging.py>`{.interpreted-text
-role="download"}
-:::
-
-::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
-`Download Jupyter notebook: configure_logging.ipynb <configure_logging.ipynb>`{.interpreted-text
-role="download"}
-:::
-:::::
-::::::
-:::::::
-
-:::: only
-html
-
-::: rst-class
-sphx-glr-signature
-
-[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
-:::
-::::
diff --git a/docs/examples/30_extended/create_upload_tutorial.rst.md b/docs/examples/30_extended/create_upload_tutorial.rst.md
deleted file mode 100644
index 83bfe0301..000000000
--- a/docs/examples/30_extended/create_upload_tutorial.rst.md
+++ /dev/null
@@ -1,460 +0,0 @@
-::::: only
-html
-
-:::: {.note .sphx-glr-download-link-note}
-::: title
-Note
-:::
-
-`Go to the end <sphx_glr_download_examples_30_extended_create_upload_tutorial.py>`{.interpreted-text
-role="ref"} to download the full example code
-::::
-:::::
-
-::: rst-class
-sphx-glr-example-title
-:::
-
-# Dataset upload tutorial {#sphx_glr_examples_30_extended_create_upload_tutorial.py}
-
-A tutorial on how to create and upload a dataset to OpenML.
-
-``` default
-# License: BSD 3-Clause
-
-import numpy as np
-import pandas as pd
-import sklearn.datasets
-from scipy.sparse import coo_matrix
-
-import openml
-from openml.datasets.functions import create_dataset
-```
-
-:::: warning
-::: title
-Warning
-:::
-
-.. include:: ../../test_server_usage_warning.txt
-::::
-
-``` default
-openml.config.start_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!
- warnings.warn(
-```
-:::
-
-Below we will cover the following cases of the dataset object:
-
-- A numpy array
-- A list
-- A pandas dataframe
-- A sparse matrix
-- A pandas sparse dataframe
-
-# Dataset is a numpy array
-
-A numpy array can contain lists in the case of dense data or it can
-contain OrderedDicts in the case of sparse data.
-
-## Prepare dataset
-
-Load an example dataset from scikit-learn which we will upload to
-OpenML.org via the API.
-
-``` default
-diabetes = sklearn.datasets.load_diabetes()
-name = "Diabetes(scikit-learn)"
-X = diabetes.data
-y = diabetes.target
-attribute_names = diabetes.feature_names
-description = diabetes.DESCR
-```
-
-OpenML does not distinguish between the attributes and targets on the
-data level and stores all data in a single matrix.
-
-The target feature is indicated as meta-data of the dataset (and tasks
-on that data).
-
-``` default
-data = np.concatenate((X, y.reshape((-1, 1))), axis=1)
-attribute_names = list(attribute_names)
-attributes = [(attribute_name, "REAL") for attribute_name in attribute_names] + [
- ("class", "INTEGER")
-]
-citation = (
- "Bradley Efron, Trevor Hastie, Iain Johnstone and "
- "Robert Tibshirani (2004) (Least Angle Regression) "
- "Annals of Statistics (with discussion), 407-499"
-)
-paper_url = "https://web.stanford.edu/~hastie/Papers/LARS/LeastAngle_2002.pdf"
-```
-
-## Create the dataset object
-
-The definition of all fields can be found in the XSD files describing
-the expected format:
-
-<https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/openml.data.upload.xsd>
-
-``` default
-diabetes_dataset = create_dataset(
- # The name of the dataset (needs to be unique).
- # Must not be longer than 128 characters and only contain
- # a-z, A-Z, 0-9 and the following special characters: _\-\.(),
- name=name,
- # Textual description of the dataset.
- description=description,
- # The person who created the dataset.
- creator="Bradley Efron, Trevor Hastie, Iain Johnstone and Robert Tibshirani",
- # People who contributed to the current version of the dataset.
- contributor=None,
- # The date the data was originally collected, given by the uploader.
- collection_date="09-01-2012",
- # Language in which the data is represented.
- # Starts with 1 upper case letter, rest lower case, e.g. 'English'.
- language="English",
- # License under which the data is/will be distributed.
- licence="BSD (from scikit-learn)",
- # Name of the target. Can also have multiple values (comma-separated).
- default_target_attribute="class",
- # The attribute that represents the row-id column, if present in the
- # dataset.
- row_id_attribute=None,
- # Attribute or list of attributes that should be excluded in modelling, such as
- # identifiers and indexes. E.g. "feat1" or ["feat1","feat2"]
- ignore_attribute=None,
- # How to cite the paper.
- citation=citation,
- # Attributes of the data
- attributes=attributes,
- data=data,
- # A version label which is provided by the user.
- version_label="test",
- original_data_url="https://www4.stat.ncsu.edu/~boos/var.select/diabetes.html",
- paper_url=paper_url,
-)
-```
-
-``` default
-diabetes_dataset.publish()
-print(f"URL for dataset: {diabetes_dataset.openml_url}")
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-URL for dataset: https://test.openml.org/d/1322
-```
-:::
-
-# Dataset is a list
-
-A list can contain lists in the case of dense data or it can contain
-OrderedDicts in the case of sparse data.
-
-Weather dataset:
-<https://storm.cis.fordham.edu/~gweiss/data-mining/datasets.html>
-
-``` default
-data = [
- ["sunny", 85, 85, "FALSE", "no"],
- ["sunny", 80, 90, "TRUE", "no"],
- ["overcast", 83, 86, "FALSE", "yes"],
- ["rainy", 70, 96, "FALSE", "yes"],
- ["rainy", 68, 80, "FALSE", "yes"],
- ["rainy", 65, 70, "TRUE", "no"],
- ["overcast", 64, 65, "TRUE", "yes"],
- ["sunny", 72, 95, "FALSE", "no"],
- ["sunny", 69, 70, "FALSE", "yes"],
- ["rainy", 75, 80, "FALSE", "yes"],
- ["sunny", 75, 70, "TRUE", "yes"],
- ["overcast", 72, 90, "TRUE", "yes"],
- ["overcast", 81, 75, "FALSE", "yes"],
- ["rainy", 71, 91, "TRUE", "no"],
-]
-
-attribute_names = [
- ("outlook", ["sunny", "overcast", "rainy"]),
- ("temperature", "REAL"),
- ("humidity", "REAL"),
- ("windy", ["TRUE", "FALSE"]),
- ("play", ["yes", "no"]),
-]
-
-description = (
- "The weather problem is a tiny dataset that we will use repeatedly"
- " to illustrate machine learning methods. Entirely fictitious, it "
- "supposedly concerns the conditions that are suitable for playing "
- "some unspecified game. In general, instances in a dataset are "
- "characterized by the values of features, or attributes, that measure "
- "different aspects of the instance. In this case there are four "
- "attributes: outlook, temperature, humidity, and windy. "
- "The outcome is whether to play or not."
-)
-
-citation = (
- "I. H. Witten, E. Frank, M. A. Hall, and ITPro,"
- "Data mining practical machine learning tools and techniques, "
- "third edition. Burlington, Mass.: Morgan Kaufmann Publishers, 2011"
-)
-
-weather_dataset = create_dataset(
- name="Weather",
- description=description,
- creator="I. H. Witten, E. Frank, M. A. Hall, and ITPro",
- contributor=None,
- collection_date="01-01-2011",
- language="English",
- licence=None,
- default_target_attribute="play",
- row_id_attribute=None,
- ignore_attribute=None,
- citation=citation,
- attributes=attribute_names,
- data=data,
- version_label="example",
-)
-```
-
-``` default
-weather_dataset.publish()
-print(f"URL for dataset: {weather_dataset.openml_url}")
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-URL for dataset: https://test.openml.org/d/1323
-```
-:::
-
-# Dataset is a pandas DataFrame
-
-It might happen that your dataset is made of heterogeneous data which
-can usually be stored as a Pandas DataFrame. DataFrames offer the
-advantage of storing the type of data for each column as well as the
-attribute names. Therefore, when providing a Pandas DataFrame, OpenML
-can infer this information without needing to explicitly provide it when
-calling the function `openml.datasets.create_dataset`{.interpreted-text
-role="func"}. In this regard, you only need to pass `'auto'` to the
-`attributes` parameter.
-
-``` default
-df = pd.DataFrame(data, columns=[col_name for col_name, _ in attribute_names])
-# enforce the categorical column to have a categorical dtype
-df["outlook"] = df["outlook"].astype("category")
-df["windy"] = df["windy"].astype("bool")
-df["play"] = df["play"].astype("category")
-print(df.info())
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-<class 'pandas.core.frame.DataFrame'>
-RangeIndex: 14 entries, 0 to 13
-Data columns (total 5 columns):
- # Column Non-Null Count Dtype
---- ------ -------------- -----
- 0 outlook 14 non-null category
- 1 temperature 14 non-null int64
- 2 humidity 14 non-null int64
- 3 windy 14 non-null bool
- 4 play 14 non-null category
-dtypes: bool(1), category(2), int64(2)
-memory usage: 650.0 bytes
-None
-```
-:::
-
-We enforce the column \'outlook\' and \'play\' to be a categorical dtype
-while the column \'windy\' is kept as a boolean column. \'temperature\'
-and \'humidity\' are kept as numeric columns. Then, we can call
-`openml.datasets.create_dataset`{.interpreted-text role="func"} by
-passing the dataframe and fixing the parameter `attributes` to `'auto'`.
-
-``` default
-weather_dataset = create_dataset(
- name="Weather",
- description=description,
- creator="I. H. Witten, E. Frank, M. A. Hall, and ITPro",
- contributor=None,
- collection_date="01-01-2011",
- language="English",
- licence=None,
- default_target_attribute="play",
- row_id_attribute=None,
- ignore_attribute=None,
- citation=citation,
- attributes="auto",
- data=df,
- version_label="example",
-)
-```
-
-``` default
-weather_dataset.publish()
-print(f"URL for dataset: {weather_dataset.openml_url}")
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-URL for dataset: https://test.openml.org/d/1324
-```
-:::
-
-# Dataset is a sparse matrix
-
-``` default
-sparse_data = coo_matrix(
- ([0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], ([0, 1, 1, 2, 2, 3, 3], [0, 1, 2, 0, 2, 0, 1]))
-)
-
-column_names = [
- ("input1", "REAL"),
- ("input2", "REAL"),
- ("y", "REAL"),
-]
-
-xor_dataset = create_dataset(
- name="XOR",
- description="Dataset representing the XOR operation",
- creator=None,
- contributor=None,
- collection_date=None,
- language="English",
- licence=None,
- default_target_attribute="y",
- row_id_attribute=None,
- ignore_attribute=None,
- citation=None,
- attributes=column_names,
- data=sparse_data,
- version_label="example",
-)
-```
-
-``` default
-xor_dataset.publish()
-print(f"URL for dataset: {xor_dataset.openml_url}")
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-URL for dataset: https://test.openml.org/d/1325
-```
-:::
-
-# Dataset is a pandas dataframe with sparse columns
-
-``` default
-sparse_data = coo_matrix(
- ([1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0], ([0, 1, 1, 2, 2, 3, 3], [0, 1, 2, 0, 2, 0, 1]))
-)
-column_names = ["input1", "input2", "y"]
-df = pd.DataFrame.sparse.from_spmatrix(sparse_data, columns=column_names)
-print(df.info())
-
-xor_dataset = create_dataset(
- name="XOR",
- description="Dataset representing the XOR operation",
- creator=None,
- contributor=None,
- collection_date=None,
- language="English",
- licence=None,
- default_target_attribute="y",
- row_id_attribute=None,
- ignore_attribute=None,
- citation=None,
- attributes="auto",
- data=df,
- version_label="example",
-)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-<class 'pandas.core.frame.DataFrame'>
-RangeIndex: 4 entries, 0 to 3
-Data columns (total 3 columns):
- # Column Non-Null Count Dtype
---- ------ -------------- -----
- 0 input1 4 non-null Sparse[float64, 0]
- 1 input2 4 non-null Sparse[float64, 0]
- 2 y 4 non-null Sparse[float64, 0]
-dtypes: Sparse[float64, 0](3)
-memory usage: 212.0 bytes
-None
-```
-:::
-
-``` default
-xor_dataset.publish()
-print(f"URL for dataset: {xor_dataset.openml_url}")
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-URL for dataset: https://test.openml.org/d/1326
-```
-:::
-
-``` default
-openml.config.stop_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-timing
-
-**Total running time of the script:** ( 0 minutes 2.833 seconds)
-:::
-
-::::::: {#sphx_glr_download_examples_30_extended_create_upload_tutorial.py}
-:::::: only
-html
-
-::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
-::: {.container .sphx-glr-download .sphx-glr-download-python}
-`Download Python source code: create_upload_tutorial.py <create_upload_tutorial.py>`{.interpreted-text
-role="download"}
-:::
-
-::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
-`Download Jupyter notebook: create_upload_tutorial.ipynb <create_upload_tutorial.ipynb>`{.interpreted-text
-role="download"}
-:::
-:::::
-::::::
-:::::::
-
-:::: only
-html
-
-::: rst-class
-sphx-glr-signature
-
-[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
-:::
-::::
diff --git a/docs/examples/30_extended/custom_flow_.rst.md b/docs/examples/30_extended/custom_flow_.rst.md
deleted file mode 100644
index 0bf45015a..000000000
--- a/docs/examples/30_extended/custom_flow_.rst.md
+++ /dev/null
@@ -1,294 +0,0 @@
-::::: only
-html
-
-:::: {.note .sphx-glr-download-link-note}
-::: title
-Note
-:::
-
-`Go to the end <sphx_glr_download_examples_30_extended_custom_flow_.py>`{.interpreted-text
-role="ref"} to download the full example code
-::::
-:::::
-
-::: rst-class
-sphx-glr-example-title
-:::
-
-# Creating and Using a Custom Flow {#sphx_glr_examples_30_extended_custom_flow_.py}
-
-The most convenient way to create a flow for your machine learning
-workflow is to generate it automatically as described in the
-`sphx_glr_examples_30_extended_flow_id_tutorial.py`{.interpreted-text
-role="ref"} tutorial. However, there are scenarios where this is not
-possible, such as when the flow uses a framework without an extension or
-when the flow is described by a script.
-
-In those cases you can still create a custom flow by following the steps
-of this tutorial. As an example we will use the flows generated for the
-[AutoML Benchmark](https://openml.github.io/automlbenchmark/), and also
-show how to link runs to the custom flow.
-
-``` default
-# License: BSD 3-Clause
-
-from collections import OrderedDict
-import numpy as np
-
-import openml
-from openml import OpenMLClassificationTask
-from openml.runs.functions import format_prediction
-```
-
-:::: warning
-::: title
-Warning
-:::
-
-.. include:: ../../test_server_usage_warning.txt
-::::
-
-``` default
-openml.config.start_using_configuration_for_example()
-```
-
-## 1. Defining the flow
-
-The first step is to define all the hyperparameters of your flow. The
-API pages feature a descriptions of each variable of the
-`openml.flows.OpenMLFlow`{.interpreted-text role="class"}. Note that
-[external version]{.title-ref} and [name]{.title-ref} together uniquely
-identify a flow.
-
-The AutoML Benchmark runs AutoML systems across a range of tasks. OpenML
-stores Flows for each AutoML system. However, the AutoML benchmark adds
-preprocessing to the flow, so should be described in a new flow.
-
-We will break down the flow arguments into several groups, for the
-tutorial. First we will define the name and version information. Make
-sure to leave enough information so others can determine exactly which
-version of the package/script is used. Use tags so users can find your
-flow easily.
-
-``` default
-general = dict(
- name="automlbenchmark_autosklearn",
- description=(
- "Auto-sklearn as set up by the AutoML Benchmark"
- "Source: https://github.com/openml/automlbenchmark/releases/tag/v0.9"
- ),
- external_version="amlb==0.9",
- language="English",
- tags=["amlb", "benchmark", "study_218"],
- dependencies="amlb==0.9",
-)
-```
-
-Next we define the flow hyperparameters. We define their name and
-default value in [parameters]{.title-ref}, and provide meta-data for
-each hyperparameter through [parameters_meta_info]{.title-ref}. Note
-that even though the argument name is [parameters]{.title-ref} they
-describe the hyperparameters. The use of ordered dicts is required.
-
-``` default
-flow_hyperparameters = dict(
- parameters=OrderedDict(time="240", memory="32", cores="8"),
- parameters_meta_info=OrderedDict(
- cores=OrderedDict(description="number of available cores", data_type="int"),
- memory=OrderedDict(description="memory in gigabytes", data_type="int"),
- time=OrderedDict(description="time in minutes", data_type="int"),
- ),
-)
-```
-
-It is possible to build a flow which uses other flows. For example, the
-Random Forest Classifier is a flow, but you could also construct a flow
-which uses a Random Forest Classifier in a ML pipeline. When
-constructing the pipeline flow, you can use the Random Forest Classifier
-flow as a *subflow*. It allows for all hyperparameters of the Random
-Classifier Flow to also be specified in your pipeline flow.
-
-Note: you can currently only specific one subflow as part of the
-components.
-
-In this example, the auto-sklearn flow is a subflow: the auto-sklearn
-flow is entirely executed as part of this flow. This allows people to
-specify auto-sklearn hyperparameters used in this flow. In general,
-using a subflow is not required.
-
-Note: flow 9313 is not actually the right flow on the test server, but
-that does not matter for this demonstration.
-
-``` default
-autosklearn_flow = openml.flows.get_flow(9313) # auto-sklearn 0.5.1
-subflow = dict(
- components=OrderedDict(automl_tool=autosklearn_flow),
- # If you do not want to reference a subflow, you can use the following:
- # components=OrderedDict(),
-)
-```
-
-With all parameters of the flow defined, we can now initialize the
-OpenMLFlow and publish. Because we provided all the details already, we
-do not need to provide a [model]{.title-ref} to the flow.
-
-In our case, we don\'t even have a model. It is possible to have a model
-but still require to follow these steps when the model (python object)
-does not have an extensions from which to automatically extract the
-hyperparameters. So whether you have a model with no extension or no
-model at all, explicitly set the model of the flow to
-[None]{.title-ref}.
-
-``` default
-autosklearn_amlb_flow = openml.flows.OpenMLFlow(
- **general,
- **flow_hyperparameters,
- **subflow,
- model=None,
-)
-autosklearn_amlb_flow.publish()
-print(f"autosklearn flow created: {autosklearn_amlb_flow.flow_id}")
-```
-
-## 2. Using the flow
-
-This Section will show how to upload run data for your custom flow. Take
-care to change the values of parameters as well as the task id, to
-reflect the actual run. Task and parameter values in the example are
-fictional.
-
-``` default
-flow_id = autosklearn_amlb_flow.flow_id
-
-parameters = [
- OrderedDict([("oml:name", "cores"), ("oml:value", 4), ("oml:component", flow_id)]),
- OrderedDict([("oml:name", "memory"), ("oml:value", 16), ("oml:component", flow_id)]),
- OrderedDict([("oml:name", "time"), ("oml:value", 120), ("oml:component", flow_id)]),
-]
-
-task_id = 1200 # Iris Task
-task = openml.tasks.get_task(task_id)
-dataset_id = task.get_dataset().dataset_id
-```
-
-The last bit of information for the run we need are the predicted
-values. The exact format of the predictions will depend on the task.
-
-The predictions should always be a list of lists, each list should
-contain:
-
-- the repeat number: for repeated evaluation strategies. (e.g.
- repeated cross-validation)
-- the fold number: for cross-validation. (what should this be for
- holdout?)
-- 0: this field is for backward compatibility.
-- index: the row (of the original dataset) for which the prediction
- was made.
-- p_1, \..., p_c: for each class the predicted probability of the
- sample belonging to that class. (no elements for regression tasks)
- Make sure the order of these elements follows the order of
- [task.class_labels]{.title-ref}.
-- the predicted class/value for the sample
-- the true class/value for the sample
-
-When using openml-python extensions (such as through
-[run_model_on_task]{.title-ref}), all of this formatting is automatic.
-Unfortunately we can not automate this procedure for custom flows, which
-means a little additional effort is required.
-
-Here we generated some random predictions in place. You can ignore this
-code, or use it to better understand the formatting of the predictions.
-
-Find the repeats/folds for this task:
-
-``` default
-n_repeats, n_folds, _ = task.get_split_dimensions()
-all_test_indices = [
- (repeat, fold, index)
- for repeat in range(n_repeats)
- for fold in range(n_folds)
- for index in task.get_train_test_split_indices(fold, repeat)[1]
-]
-
-# random class probabilities (Iris has 150 samples and 3 classes):
-r = np.random.rand(150 * n_repeats, 3)
-# scale the random values so that the probabilities of each sample sum to 1:
-y_proba = r / r.sum(axis=1).reshape(-1, 1)
-y_pred = y_proba.argmax(axis=1)
-
-class_map = dict(zip(range(3), task.class_labels))
-_, y_true = task.get_X_and_y()
-y_true = [class_map[y] for y in y_true]
-
-# We format the predictions with the utility function `format_prediction`.
-# It will organize the relevant data in the expected format/order.
-predictions = []
-for where, y, yp, proba in zip(all_test_indices, y_true, y_pred, y_proba):
- repeat, fold, index = where
-
- prediction = format_prediction(
- task=task,
- repeat=repeat,
- fold=fold,
- index=index,
- prediction=class_map[yp],
- truth=y,
- proba={c: pb for (c, pb) in zip(task.class_labels, proba)},
- )
- predictions.append(prediction)
-```
-
-Finally we can create the OpenMLRun object and upload. We use the
-argument setup_string because the used flow was a script.
-
-``` default
-benchmark_command = f"python3 runbenchmark.py auto-sklearn medium -m aws -t 119"
-my_run = openml.runs.OpenMLRun(
- task_id=task_id,
- flow_id=flow_id,
- dataset_id=dataset_id,
- parameter_settings=parameters,
- setup_string=benchmark_command,
- data_content=predictions,
- tags=["study_218"],
- description_text="Run generated by the Custom Flow tutorial.",
-)
-my_run.publish()
-print("run created:", my_run.run_id)
-
-openml.config.stop_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-timing
-
-**Total running time of the script:** ( 0 minutes 0.000 seconds)
-:::
-
-::::::: {#sphx_glr_download_examples_30_extended_custom_flow_.py}
-:::::: only
-html
-
-::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
-::: {.container .sphx-glr-download .sphx-glr-download-python}
-`Download Python source code: custom_flow_.py <custom_flow_.py>`{.interpreted-text
-role="download"}
-:::
-
-::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
-`Download Jupyter notebook: custom_flow_.ipynb <custom_flow_.ipynb>`{.interpreted-text
-role="download"}
-:::
-:::::
-::::::
-:::::::
-
-:::: only
-html
-
-::: rst-class
-sphx-glr-signature
-
-[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
-:::
-::::
diff --git a/docs/examples/30_extended/datasets_tutorial.rst.md b/docs/examples/30_extended/datasets_tutorial.rst.md
deleted file mode 100644
index ae02ecb31..000000000
--- a/docs/examples/30_extended/datasets_tutorial.rst.md
+++ /dev/null
@@ -1,1040 +0,0 @@
-::::: only
-html
-
-:::: {.note .sphx-glr-download-link-note}
-::: title
-Note
-:::
-
-`Go to the end <sphx_glr_download_examples_30_extended_datasets_tutorial.py>`{.interpreted-text
-role="ref"} to download the full example code
-::::
-:::::
-
-::: rst-class
-sphx-glr-example-title
-:::
-
-# Datasets {#sphx_glr_examples_30_extended_datasets_tutorial.py}
-
-How to list and download datasets.
-
-``` default
-# License: BSD 3-Clauses
-
-import openml
-import pandas as pd
-from openml.datasets import edit_dataset, fork_dataset, get_dataset
-```
-
-## Exercise 0
-
-- List datasets
- - Use the output_format parameter to select output type
- - Default gives \'dict\' (other option: \'dataframe\', see below)
-
-Note: list_datasets will return a pandas dataframe by default from 0.15.
-When using openml-python 0.14, [list_datasets]{.title-ref} will warn you
-to use output_format=\'dataframe\'.
-
-``` default
-datalist = openml.datasets.list_datasets(output_format="dataframe")
-datalist = datalist[["did", "name", "NumberOfInstances", "NumberOfFeatures", "NumberOfClasses"]]
-
-print(f"First 10 of {len(datalist)} datasets...")
-datalist.head(n=10)
-
-# The same can be done with lesser lines of code
-openml_df = openml.datasets.list_datasets(output_format="dataframe")
-openml_df.head(n=10)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-First 10 of 5308 datasets...
-```
-:::
-
-<div class="output_subarea output_html rendered_html output_result">
-<div>
-<style scoped>
- .dataframe tbody tr th:only-of-type {
- vertical-align: middle;
- }
-
- .dataframe tbody tr th {
- vertical-align: top;
- }
-
- .dataframe thead th {
- text-align: right;
- }
-</style>
-<table border="1" class="dataframe">
- <thead>
- <tr style="text-align: right;">
- <th></th>
- <th>did</th>
- <th>name</th>
- <th>version</th>
- <th>uploader</th>
- <th>status</th>
- <th>format</th>
- <th>MajorityClassSize</th>
- <th>MaxNominalAttDistinctValues</th>
- <th>MinorityClassSize</th>
- <th>NumberOfClasses</th>
- <th>NumberOfFeatures</th>
- <th>NumberOfInstances</th>
- <th>NumberOfInstancesWithMissingValues</th>
- <th>NumberOfMissingValues</th>
- <th>NumberOfNumericFeatures</th>
- <th>NumberOfSymbolicFeatures</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <th>2</th>
- <td>2</td>
- <td>anneal</td>
- <td>1</td>
- <td>1</td>
- <td>active</td>
- <td>ARFF</td>
- <td>684.0</td>
- <td>7.0</td>
- <td>8.0</td>
- <td>5.0</td>
- <td>39.0</td>
- <td>898.0</td>
- <td>898.0</td>
- <td>22175.0</td>
- <td>6.0</td>
- <td>33.0</td>
- </tr>
- <tr>
- <th>3</th>
- <td>3</td>
- <td>kr-vs-kp</td>
- <td>1</td>
- <td>1</td>
- <td>active</td>
- <td>ARFF</td>
- <td>1669.0</td>
- <td>3.0</td>
- <td>1527.0</td>
- <td>2.0</td>
- <td>37.0</td>
- <td>3196.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>37.0</td>
- </tr>
- <tr>
- <th>4</th>
- <td>4</td>
- <td>labor</td>
- <td>1</td>
- <td>1</td>
- <td>active</td>
- <td>ARFF</td>
- <td>37.0</td>
- <td>3.0</td>
- <td>20.0</td>
- <td>2.0</td>
- <td>17.0</td>
- <td>57.0</td>
- <td>56.0</td>
- <td>326.0</td>
- <td>8.0</td>
- <td>9.0</td>
- </tr>
- <tr>
- <th>5</th>
- <td>5</td>
- <td>arrhythmia</td>
- <td>1</td>
- <td>1</td>
- <td>active</td>
- <td>ARFF</td>
- <td>245.0</td>
- <td>13.0</td>
- <td>2.0</td>
- <td>13.0</td>
- <td>280.0</td>
- <td>452.0</td>
- <td>384.0</td>
- <td>408.0</td>
- <td>206.0</td>
- <td>74.0</td>
- </tr>
- <tr>
- <th>6</th>
- <td>6</td>
- <td>letter</td>
- <td>1</td>
- <td>1</td>
- <td>active</td>
- <td>ARFF</td>
- <td>813.0</td>
- <td>26.0</td>
- <td>734.0</td>
- <td>26.0</td>
- <td>17.0</td>
- <td>20000.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>16.0</td>
- <td>1.0</td>
- </tr>
- <tr>
- <th>7</th>
- <td>7</td>
- <td>audiology</td>
- <td>1</td>
- <td>1</td>
- <td>active</td>
- <td>ARFF</td>
- <td>57.0</td>
- <td>24.0</td>
- <td>1.0</td>
- <td>24.0</td>
- <td>70.0</td>
- <td>226.0</td>
- <td>222.0</td>
- <td>317.0</td>
- <td>0.0</td>
- <td>70.0</td>
- </tr>
- <tr>
- <th>8</th>
- <td>8</td>
- <td>liver-disorders</td>
- <td>1</td>
- <td>1</td>
- <td>active</td>
- <td>ARFF</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>0.0</td>
- <td>6.0</td>
- <td>345.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>6.0</td>
- <td>0.0</td>
- </tr>
- <tr>
- <th>9</th>
- <td>9</td>
- <td>autos</td>
- <td>1</td>
- <td>1</td>
- <td>active</td>
- <td>ARFF</td>
- <td>67.0</td>
- <td>22.0</td>
- <td>3.0</td>
- <td>6.0</td>
- <td>26.0</td>
- <td>205.0</td>
- <td>46.0</td>
- <td>59.0</td>
- <td>15.0</td>
- <td>11.0</td>
- </tr>
- <tr>
- <th>10</th>
- <td>10</td>
- <td>lymph</td>
- <td>1</td>
- <td>1</td>
- <td>active</td>
- <td>ARFF</td>
- <td>81.0</td>
- <td>8.0</td>
- <td>2.0</td>
- <td>4.0</td>
- <td>19.0</td>
- <td>148.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>3.0</td>
- <td>16.0</td>
- </tr>
- <tr>
- <th>11</th>
- <td>11</td>
- <td>balance-scale</td>
- <td>1</td>
- <td>1</td>
- <td>active</td>
- <td>ARFF</td>
- <td>288.0</td>
- <td>3.0</td>
- <td>49.0</td>
- <td>3.0</td>
- <td>5.0</td>
- <td>625.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>4.0</td>
- <td>1.0</td>
- </tr>
- </tbody>
-</table>
-</div>
-</div>
-<br />
-<br />
-
-## Exercise 1
-
-- Find datasets with more than 10000 examples.
-- Find a dataset called \'eeg_eye_state\'.
-- Find all datasets with more than 50 classes.
-
-``` default
-datalist[datalist.NumberOfInstances > 10000].sort_values(["NumberOfInstances"]).head(n=20)
-""
-datalist.query('name == "eeg-eye-state"')
-""
-datalist.query("NumberOfClasses > 50")
-```
-
-<div class="output_subarea output_html rendered_html output_result">
-<div>
-<style scoped>
- .dataframe tbody tr th:only-of-type {
- vertical-align: middle;
- }
-
- .dataframe tbody tr th {
- vertical-align: top;
- }
-
- .dataframe thead th {
- text-align: right;
- }
-</style>
-<table border="1" class="dataframe">
- <thead>
- <tr style="text-align: right;">
- <th></th>
- <th>did</th>
- <th>name</th>
- <th>NumberOfInstances</th>
- <th>NumberOfFeatures</th>
- <th>NumberOfClasses</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <th>1491</th>
- <td>1491</td>
- <td>one-hundred-plants-margin</td>
- <td>1600.0</td>
- <td>65.0</td>
- <td>100.0</td>
- </tr>
- <tr>
- <th>1492</th>
- <td>1492</td>
- <td>one-hundred-plants-shape</td>
- <td>1600.0</td>
- <td>65.0</td>
- <td>100.0</td>
- </tr>
- <tr>
- <th>1493</th>
- <td>1493</td>
- <td>one-hundred-plants-texture</td>
- <td>1599.0</td>
- <td>65.0</td>
- <td>100.0</td>
- </tr>
- <tr>
- <th>4552</th>
- <td>4552</td>
- <td>BachChoralHarmony</td>
- <td>5665.0</td>
- <td>17.0</td>
- <td>102.0</td>
- </tr>
- <tr>
- <th>41167</th>
- <td>41167</td>
- <td>dionis</td>
- <td>416188.0</td>
- <td>61.0</td>
- <td>355.0</td>
- </tr>
- <tr>
- <th>41169</th>
- <td>41169</td>
- <td>helena</td>
- <td>65196.0</td>
- <td>28.0</td>
- <td>100.0</td>
- </tr>
- <tr>
- <th>41960</th>
- <td>41960</td>
- <td>seattlecrime6</td>
- <td>523590.0</td>
- <td>8.0</td>
- <td>144.0</td>
- </tr>
- <tr>
- <th>41983</th>
- <td>41983</td>
- <td>CIFAR-100</td>
- <td>60000.0</td>
- <td>3073.0</td>
- <td>100.0</td>
- </tr>
- <tr>
- <th>42078</th>
- <td>42078</td>
- <td>beer_reviews</td>
- <td>1586614.0</td>
- <td>13.0</td>
- <td>104.0</td>
- </tr>
- <tr>
- <th>42087</th>
- <td>42087</td>
- <td>beer_reviews</td>
- <td>1586614.0</td>
- <td>13.0</td>
- <td>104.0</td>
- </tr>
- <tr>
- <th>42088</th>
- <td>42088</td>
- <td>beer_reviews</td>
- <td>1586614.0</td>
- <td>13.0</td>
- <td>104.0</td>
- </tr>
- <tr>
- <th>42089</th>
- <td>42089</td>
- <td>vancouver_employee</td>
- <td>1586614.0</td>
- <td>13.0</td>
- <td>104.0</td>
- </tr>
- <tr>
- <th>42123</th>
- <td>42123</td>
- <td>article_influence</td>
- <td>3615.0</td>
- <td>7.0</td>
- <td>3169.0</td>
- </tr>
- <tr>
- <th>42223</th>
- <td>42223</td>
- <td>dataset-autoHorse_fixed</td>
- <td>201.0</td>
- <td>69.0</td>
- <td>186.0</td>
- </tr>
- <tr>
- <th>42396</th>
- <td>42396</td>
- <td>aloi</td>
- <td>108000.0</td>
- <td>129.0</td>
- <td>1000.0</td>
- </tr>
- <tr>
- <th>43723</th>
- <td>43723</td>
- <td>Toronto-Apartment-Rental-Price</td>
- <td>1124.0</td>
- <td>7.0</td>
- <td>188.0</td>
- </tr>
- <tr>
- <th>44282</th>
- <td>44282</td>
- <td>Meta_Album_PLK_Mini</td>
- <td>3440.0</td>
- <td>3.0</td>
- <td>86.0</td>
- </tr>
- <tr>
- <th>44283</th>
- <td>44283</td>
- <td>Meta_Album_FLW_Mini</td>
- <td>4080.0</td>
- <td>3.0</td>
- <td>102.0</td>
- </tr>
- <tr>
- <th>44284</th>
- <td>44284</td>
- <td>Meta_Album_SPT_Mini</td>
- <td>2920.0</td>
- <td>3.0</td>
- <td>73.0</td>
- </tr>
- <tr>
- <th>44285</th>
- <td>44285</td>
- <td>Meta_Album_BRD_Mini</td>
- <td>12600.0</td>
- <td>3.0</td>
- <td>315.0</td>
- </tr>
- <tr>
- <th>44288</th>
- <td>44288</td>
- <td>Meta_Album_TEX_Mini</td>
- <td>2560.0</td>
- <td>3.0</td>
- <td>64.0</td>
- </tr>
- <tr>
- <th>44289</th>
- <td>44289</td>
- <td>Meta_Album_CRS_Mini</td>
- <td>7840.0</td>
- <td>3.0</td>
- <td>196.0</td>
- </tr>
- <tr>
- <th>44292</th>
- <td>44292</td>
- <td>Meta_Album_INS_2_Mini</td>
- <td>4080.0</td>
- <td>3.0</td>
- <td>102.0</td>
- </tr>
- <tr>
- <th>44298</th>
- <td>44298</td>
- <td>Meta_Album_DOG_Mini</td>
- <td>4800.0</td>
- <td>3.0</td>
- <td>120.0</td>
- </tr>
- <tr>
- <th>44304</th>
- <td>44304</td>
- <td>Meta_Album_TEX_ALOT_Mini</td>
- <td>10000.0</td>
- <td>3.0</td>
- <td>250.0</td>
- </tr>
- <tr>
- <th>44306</th>
- <td>44306</td>
- <td>Meta_Album_INS_Mini</td>
- <td>4160.0</td>
- <td>3.0</td>
- <td>104.0</td>
- </tr>
- <tr>
- <th>44317</th>
- <td>44317</td>
- <td>Meta_Album_PLK_Extended</td>
- <td>473273.0</td>
- <td>3.0</td>
- <td>102.0</td>
- </tr>
- <tr>
- <th>44318</th>
- <td>44318</td>
- <td>Meta_Album_FLW_Extended</td>
- <td>8189.0</td>
- <td>3.0</td>
- <td>102.0</td>
- </tr>
- <tr>
- <th>44319</th>
- <td>44319</td>
- <td>Meta_Album_SPT_Extended</td>
- <td>10416.0</td>
- <td>3.0</td>
- <td>73.0</td>
- </tr>
- <tr>
- <th>44320</th>
- <td>44320</td>
- <td>Meta_Album_BRD_Extended</td>
- <td>49054.0</td>
- <td>3.0</td>
- <td>315.0</td>
- </tr>
- <tr>
- <th>44322</th>
- <td>44322</td>
- <td>Meta_Album_TEX_Extended</td>
- <td>8675.0</td>
- <td>3.0</td>
- <td>64.0</td>
- </tr>
- <tr>
- <th>44323</th>
- <td>44323</td>
- <td>Meta_Album_CRS_Extended</td>
- <td>16185.0</td>
- <td>3.0</td>
- <td>196.0</td>
- </tr>
- <tr>
- <th>44326</th>
- <td>44326</td>
- <td>Meta_Album_INS_2_Extended</td>
- <td>75222.0</td>
- <td>3.0</td>
- <td>102.0</td>
- </tr>
- <tr>
- <th>44331</th>
- <td>44331</td>
- <td>Meta_Album_DOG_Extended</td>
- <td>20480.0</td>
- <td>3.0</td>
- <td>120.0</td>
- </tr>
- <tr>
- <th>44337</th>
- <td>44337</td>
- <td>Meta_Album_TEX_ALOT_Extended</td>
- <td>25000.0</td>
- <td>3.0</td>
- <td>250.0</td>
- </tr>
- <tr>
- <th>44340</th>
- <td>44340</td>
- <td>Meta_Album_INS_Extended</td>
- <td>170506.0</td>
- <td>3.0</td>
- <td>117.0</td>
- </tr>
- <tr>
- <th>44533</th>
- <td>44533</td>
- <td>dionis_seed_0_nrows_2000_nclasses_10_ncols_100...</td>
- <td>2000.0</td>
- <td>61.0</td>
- <td>355.0</td>
- </tr>
- <tr>
- <th>44534</th>
- <td>44534</td>
- <td>dionis_seed_1_nrows_2000_nclasses_10_ncols_100...</td>
- <td>2000.0</td>
- <td>61.0</td>
- <td>355.0</td>
- </tr>
- <tr>
- <th>44535</th>
- <td>44535</td>
- <td>dionis_seed_2_nrows_2000_nclasses_10_ncols_100...</td>
- <td>2000.0</td>
- <td>61.0</td>
- <td>355.0</td>
- </tr>
- <tr>
- <th>44536</th>
- <td>44536</td>
- <td>dionis_seed_3_nrows_2000_nclasses_10_ncols_100...</td>
- <td>2000.0</td>
- <td>61.0</td>
- <td>355.0</td>
- </tr>
- <tr>
- <th>44537</th>
- <td>44537</td>
- <td>dionis_seed_4_nrows_2000_nclasses_10_ncols_100...</td>
- <td>2000.0</td>
- <td>61.0</td>
- <td>355.0</td>
- </tr>
- <tr>
- <th>44728</th>
- <td>44728</td>
- <td>helena_seed_0_nrows_2000_nclasses_10_ncols_100...</td>
- <td>2000.0</td>
- <td>28.0</td>
- <td>100.0</td>
- </tr>
- <tr>
- <th>44729</th>
- <td>44729</td>
- <td>helena_seed_1_nrows_2000_nclasses_10_ncols_100...</td>
- <td>2000.0</td>
- <td>28.0</td>
- <td>100.0</td>
- </tr>
- <tr>
- <th>44730</th>
- <td>44730</td>
- <td>helena_seed_2_nrows_2000_nclasses_10_ncols_100...</td>
- <td>2000.0</td>
- <td>28.0</td>
- <td>100.0</td>
- </tr>
- <tr>
- <th>44731</th>
- <td>44731</td>
- <td>helena_seed_3_nrows_2000_nclasses_10_ncols_100...</td>
- <td>2000.0</td>
- <td>28.0</td>
- <td>100.0</td>
- </tr>
- <tr>
- <th>44732</th>
- <td>44732</td>
- <td>helena_seed_4_nrows_2000_nclasses_10_ncols_100...</td>
- <td>2000.0</td>
- <td>28.0</td>
- <td>100.0</td>
- </tr>
- <tr>
- <th>45049</th>
- <td>45049</td>
- <td>MD_MIX_Mini_Copy</td>
- <td>28240.0</td>
- <td>69.0</td>
- <td>706.0</td>
- </tr>
- <tr>
- <th>45102</th>
- <td>45102</td>
- <td>dailybike</td>
- <td>731.0</td>
- <td>13.0</td>
- <td>606.0</td>
- </tr>
- <tr>
- <th>45103</th>
- <td>45103</td>
- <td>dailybike</td>
- <td>731.0</td>
- <td>13.0</td>
- <td>606.0</td>
- </tr>
- <tr>
- <th>45104</th>
- <td>45104</td>
- <td>PLK_Mini_Copy</td>
- <td>3440.0</td>
- <td>3.0</td>
- <td>86.0</td>
- </tr>
- <tr>
- <th>45274</th>
- <td>45274</td>
- <td>PASS</td>
- <td>1439588.0</td>
- <td>7.0</td>
- <td>94137.0</td>
- </tr>
- <tr>
- <th>45569</th>
- <td>45569</td>
- <td>DBLP-QuAD</td>
- <td>10000.0</td>
- <td>10.0</td>
- <td>9999.0</td>
- </tr>
- </tbody>
-</table>
-</div>
-</div>
-<br />
-<br />
-
-### Download datasets
-
-``` default
-# This is done based on the dataset ID.
-dataset = openml.datasets.get_dataset(1471)
-
-# Print a summary
-print(
- f"This is dataset '{dataset.name}', the target feature is "
- f"'{dataset.default_target_attribute}'"
-)
-print(f"URL: {dataset.url}")
-print(dataset.description[:500])
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-This is dataset 'eeg-eye-state', the target feature is 'Class'
-URL: https://api.openml.org/data/v1/download/1587924/eeg-eye-state.arff
-**Author**: Oliver Roesler
-**Source**: [UCI](https://archive.ics.uci.edu/ml/datasets/EEG+Eye+State), Baden-Wuerttemberg, Cooperative State University (DHBW), Stuttgart, Germany
-**Please cite**: [UCI](https://archive.ics.uci.edu/ml/citation_policy.html)
-
-All data is from one continuous EEG measurement with the Emotiv EEG Neuroheadset. The duration of the measurement was 117 seconds. The eye state was detected via a camera during the EEG measurement and added later manually to the file after
-```
-:::
-
-Get the actual data.
-
-openml-python returns data as pandas dataframes (stored in the
-[eeg]{.title-ref} variable below), and also some additional metadata
-that we don\'t care about right now.
-
-``` default
-eeg, *_ = dataset.get_data()
-```
-
-You can optionally choose to have openml separate out a column from the
-dataset. In particular, many datasets for supervised problems have a set
-[default_target_attribute]{.title-ref} which may help identify the
-target variable.
-
-``` default
-X, y, categorical_indicator, attribute_names = dataset.get_data(
- target=dataset.default_target_attribute
-)
-print(X.head())
-print(X.info())
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-V1 V2 V3 V4 ... V11 V12 V13 V14
-0 4329.23 4009.23 4289.23 4148.21 ... 4211.28 4280.51 4635.90 4393.85
-1 4324.62 4004.62 4293.85 4148.72 ... 4207.69 4279.49 4632.82 4384.10
-2 4327.69 4006.67 4295.38 4156.41 ... 4206.67 4282.05 4628.72 4389.23
-3 4328.72 4011.79 4296.41 4155.90 ... 4210.77 4287.69 4632.31 4396.41
-4 4326.15 4011.79 4292.31 4151.28 ... 4212.82 4288.21 4632.82 4398.46
-
-[5 rows x 14 columns]
-<class 'pandas.core.frame.DataFrame'>
-RangeIndex: 14980 entries, 0 to 14979
-Data columns (total 14 columns):
-# Column Non-Null Count Dtype
---- ------ -------------- -----
-0 V1 14980 non-null float64
-1 V2 14980 non-null float64
-2 V3 14980 non-null float64
-3 V4 14980 non-null float64
-4 V5 14980 non-null float64
-5 V6 14980 non-null float64
-6 V7 14980 non-null float64
-7 V8 14980 non-null float64
-8 V9 14980 non-null float64
-9 V10 14980 non-null float64
-10 V11 14980 non-null float64
-11 V12 14980 non-null float64
-12 V13 14980 non-null float64
-13 V14 14980 non-null float64
-dtypes: float64(14)
-memory usage: 1.6 MB
-None
-```
-:::
-
-Sometimes you only need access to a dataset\'s metadata. In those cases,
-you can download the dataset without downloading the data file. The
-dataset object can be used as normal. Whenever you use any functionality
-that requires the data, such as [get_data]{.title-ref}, the data will be
-downloaded. Starting from 0.15, not downloading data will be the default
-behavior instead. The data will be downloading automatically when you
-try to access it through openml objects, e.g., using
-[dataset.features]{.title-ref}.
-
-``` default
-dataset = openml.datasets.get_dataset(1471, download_data=False)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-```
-:::
-
-## Exercise 2
-
-- Explore the data visually.
-
-``` default
-eegs = eeg.sample(n=1000)
-_ = pd.plotting.scatter_matrix(
- X.iloc[:100, :4],
- c=y[:100],
- figsize=(10, 10),
- marker="o",
- hist_kwds={"bins": 20},
- alpha=0.8,
- cmap="plasma",
-)
-```
-
-::: {.image-sg .sphx-glr-single-img alt="datasets tutorial" srcset="/examples/30_extended/images/sphx_glr_datasets_tutorial_001.png"}
-/examples/30_extended/images/sphx_glr_datasets_tutorial_001.png
-:::
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/openml/venv/lib/python3.10/site-packages/pandas/plotting/_matplotlib/misc.py:97: UserWarning: No data for colormapping provided via 'c'. Parameters 'cmap' will be ignored
- ax.scatter(
-```
-:::
-
-### Edit a created dataset
-
-This example uses the test server, to avoid editing a dataset on the
-main server.
-
-:::: warning
-::: title
-Warning
-:::
-
-.. include:: ../../test_server_usage_warning.txt
-::::
-
-``` default
-openml.config.start_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!
- warnings.warn(
-```
-:::
-
-Edit non-critical fields, allowed for all authorized users: description,
-creator, contributor, collection_date, language, citation,
-original_data_url, paper_url
-
-``` default
-desc = (
- "This data sets consists of 3 different types of irises' "
- "(Setosa, Versicolour, and Virginica) petal and sepal length,"
- " stored in a 150x4 numpy.ndarray"
-)
-did = 128
-data_id = edit_dataset(
- did,
- description=desc,
- creator="R.A.Fisher",
- collection_date="1937",
- citation="The use of multiple measurements in taxonomic problems",
- language="English",
-)
-edited_dataset = get_dataset(data_id)
-print(f"Edited dataset ID: {data_id}")
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-Edited dataset ID: 128
-```
-:::
-
-Editing critical fields (default_target_attribute, row_id_attribute,
-ignore_attribute) is allowed only for the dataset owner. Further,
-critical fields cannot be edited if the dataset has any tasks associated
-with it. To edit critical fields of a dataset (without tasks) owned by
-you, configure the API key: openml.config.apikey =
-\'FILL_IN_OPENML_API_KEY\' This example here only shows a failure when
-trying to work on a dataset not owned by you:
-
-``` default
-try:
- data_id = edit_dataset(1, default_target_attribute="shape")
-except openml.exceptions.OpenMLServerException as e:
- print(e)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-https://test.openml.org/api/v1/xml/data/edit returned code 1065: Critical features default_target_attribute, row_id_attribute and ignore_attribute can be edited only by the owner. Fork the dataset if changes are required. - None
-```
-:::
-
-### Fork dataset
-
-Used to create a copy of the dataset with you as the owner. Use this API
-only if you are unable to edit the critical fields
-(default_target_attribute, ignore_attribute, row_id_attribute) of a
-dataset through the edit_dataset API. After the dataset is forked, you
-can edit the new version of the dataset using edit_dataset.
-
-``` default
-data_id = fork_dataset(1)
-print(data_id)
-data_id = edit_dataset(data_id, default_target_attribute="shape")
-print(f"Forked dataset ID: {data_id}")
-
-openml.config.stop_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-1306
-Forked dataset ID: 1306
-```
-:::
-
-::: rst-class
-sphx-glr-timing
-
-**Total running time of the script:** ( 0 minutes 8.138 seconds)
-:::
-
-::::::: {#sphx_glr_download_examples_30_extended_datasets_tutorial.py}
-:::::: only
-html
-
-::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
-::: {.container .sphx-glr-download .sphx-glr-download-python}
-`Download Python source code: datasets_tutorial.py <datasets_tutorial.py>`{.interpreted-text
-role="download"}
-:::
-
-::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
-`Download Jupyter notebook: datasets_tutorial.ipynb <datasets_tutorial.ipynb>`{.interpreted-text
-role="download"}
-:::
-:::::
-::::::
-:::::::
-
-:::: only
-html
-
-::: rst-class
-sphx-glr-signature
-
-[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
-:::
-::::
diff --git a/docs/examples/30_extended/fetch_evaluations_tutorial.rst.md b/docs/examples/30_extended/fetch_evaluations_tutorial.rst.md
deleted file mode 100644
index 953363f9a..000000000
--- a/docs/examples/30_extended/fetch_evaluations_tutorial.rst.md
+++ /dev/null
@@ -1,360 +0,0 @@
-::::: only
-html
-
-:::: {.note .sphx-glr-download-link-note}
-::: title
-Note
-:::
-
-`Go to the end <sphx_glr_download_examples_30_extended_fetch_evaluations_tutorial.py>`{.interpreted-text
-role="ref"} to download the full example code
-::::
-:::::
-
-::: rst-class
-sphx-glr-example-title
-:::
-
-# Fetching Evaluations {#sphx_glr_examples_30_extended_fetch_evaluations_tutorial.py}
-
-Evaluations contain a concise summary of the results of all runs made.
-Each evaluation provides information on the dataset used, the flow
-applied, the setup used, the metric evaluated, and the result obtained
-on the metric, for each such run made. These collection of results can
-be used for efficient benchmarking of an algorithm and also allow
-transparent reuse of results from previous experiments on similar
-parameters.
-
-In this example, we shall do the following:
-
-- Retrieve evaluations based on different metrics
-- Fetch evaluations pertaining to a specific task
-- Sort the obtained results in descending order of the metric
-- Plot a cumulative distribution function for the evaluations
-- Compare the top 10 performing flows based on the evaluation
- performance
-- Retrieve evaluations with hyperparameter settings
-
-``` default
-# License: BSD 3-Clause
-
-import openml
-```
-
-## Listing evaluations
-
-Evaluations can be retrieved from the database in the chosen output
-format. Required filters can be applied to retrieve results from runs as
-required.
-
-``` default
-# We shall retrieve a small set (only 10 entries) to test the listing function for evaluations
-openml.evaluations.list_evaluations(
- function="predictive_accuracy", size=10, output_format="dataframe"
-)
-
-# Using other evaluation metrics, 'precision' in this case
-evals = openml.evaluations.list_evaluations(
- function="precision", size=10, output_format="dataframe"
-)
-
-# Querying the returned results for precision above 0.98
-print(evals[evals.value > 0.98])
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-run_id task_id ... values array_data
-0 62 1 ... None [0.714286,0.98,0.992658,0,0.985294,0.904762]
-1 237 1 ... None [1,0.942857,0.991215,0,1,0.95]
-3 413 1 ... None [1,0.980198,0.994152,0,1,0.948718]
-4 500 1 ... None [1,0.99,0.997059,0,0.985294,0.863636]
-
-[4 rows x 14 columns]
-```
-:::
-
-### Viewing a sample task
-
-Over here we shall briefly take a look at the details of the task.
-
-``` default
-# We will start by displaying a simple *supervised classification* task:
-task_id = 167140 # https://www.openml.org/t/167140
-task = openml.tasks.get_task(task_id)
-print(task)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-OpenML Classification Task
-==========================
-Task Type Description: https://www.openml.org/tt/TaskType.SUPERVISED_CLASSIFICATION
-Task ID..............: 167140
-Task URL.............: https://www.openml.org/t/167140
-Estimation Procedure.: crossvalidation
-Target Feature.......: class
-# of Classes.........: 3
-Cost Matrix..........: Available
-```
-:::
-
-### Obtaining all the evaluations for the task
-
-We\'ll now obtain all the evaluations that were uploaded for the task we
-displayed previously. Note that we now filter the evaluations based on
-another parameter \'task\'.
-
-``` default
-metric = "predictive_accuracy"
-evals = openml.evaluations.list_evaluations(
- function=metric, tasks=[task_id], output_format="dataframe"
-)
-# Displaying the first 10 rows
-print(evals.head(n=10))
-# Sorting the evaluations in decreasing order of the metric chosen
-evals = evals.sort_values(by="value", ascending=False)
-print("\nDisplaying head of sorted dataframe: ")
-print(evals.head())
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-run_id task_id setup_id ... value values array_data
-0 9199772 167140 7130140 ... 0.938481 None None
-1 9199845 167140 7130139 ... 0.623352 None None
-2 9202086 167140 7131884 ... 0.918393 None None
-3 9202092 167140 7131890 ... 0.962335 None None
-4 9202096 167140 7131894 ... 0.961707 None None
-5 9202099 167140 7131897 ... 0.519146 None None
-6 9202125 167140 7131923 ... 0.957313 None None
-7 9202139 167140 7131937 ... 0.519146 None None
-8 9202159 167140 7131957 ... 0.628060 None None
-9 9202254 167140 7132052 ... 0.610483 None None
-
-[10 rows x 14 columns]
-
-Displaying head of sorted dataframe:
- run_id task_id setup_id ... value values array_data
-2788 10279797 167140 8166877 ... 0.968613 None None
-2789 10279798 167140 8166877 ... 0.968613 None None
-2246 10232158 167140 8157180 ... 0.967671 None None
-2263 10232987 167140 8157961 ... 0.967671 None None
-2314 10271558 167140 8154506 ... 0.967043 None None
-
-[5 rows x 14 columns]
-```
-:::
-
-## Obtaining CDF of metric for chosen task
-
-We shall now analyse how the performance of various flows have been on
-this task, by seeing the likelihood of the accuracy obtained across all
-runs. We shall now plot a cumulative distributive function (CDF) for the
-accuracies obtained.
-
-``` default
-from matplotlib import pyplot as plt
-
-
-def plot_cdf(values, metric="predictive_accuracy"):
- max_val = max(values)
- n, bins, patches = plt.hist(values, density=True, histtype="step", cumulative=True, linewidth=3)
- patches[0].set_xy(patches[0].get_xy()[:-1])
- plt.xlim(max(0, min(values) - 0.1), 1)
- plt.title("CDF")
- plt.xlabel(metric)
- plt.ylabel("Likelihood")
- plt.grid(visible=True, which="major", linestyle="-")
- plt.minorticks_on()
- plt.grid(visible=True, which="minor", linestyle="--")
- plt.axvline(max_val, linestyle="--", color="gray")
- plt.text(max_val, 0, "%.3f" % max_val, fontsize=9)
- plt.show()
-
-
-plot_cdf(evals.value, metric)
-# This CDF plot shows that for the given task, based on the results of the
-# runs uploaded, it is almost certain to achieve an accuracy above 52%, i.e.,
-# with non-zero probability. While the maximum accuracy seen till now is 96.5%.
-```
-
-::: {.image-sg .sphx-glr-single-img alt="CDF" srcset="/examples/30_extended/images/sphx_glr_fetch_evaluations_tutorial_001.png"}
-/examples/30_extended/images/sphx_glr_fetch_evaluations_tutorial_001.png
-:::
-
-## Comparing top 10 performing flows
-
-Let us now try to see which flows generally performed the best for this
-task. For this, we shall compare the top performing flows.
-
-``` default
-import numpy as np
-import pandas as pd
-
-
-def plot_flow_compare(evaluations, top_n=10, metric="predictive_accuracy"):
- # Collecting the top 10 performing unique flow_id
- flow_ids = evaluations.flow_id.unique()[:top_n]
-
- df = pd.DataFrame()
- # Creating a data frame containing only the metric values of the selected flows
- # assuming evaluations is sorted in decreasing order of metric
- for i in range(len(flow_ids)):
- flow_values = evaluations[evaluations.flow_id == flow_ids[i]].value
- df = pd.concat([df, flow_values], ignore_index=True, axis=1)
- fig, axs = plt.subplots()
- df.boxplot()
- axs.set_title("Boxplot comparing " + metric + " for different flows")
- axs.set_ylabel(metric)
- axs.set_xlabel("Flow ID")
- axs.set_xticklabels(flow_ids)
- axs.grid(which="major", linestyle="-", linewidth="0.5", color="gray", axis="y")
- axs.minorticks_on()
- axs.grid(which="minor", linestyle="--", linewidth="0.5", color="gray", axis="y")
- # Counting the number of entries for each flow in the data frame
- # which gives the number of runs for each flow
- flow_freq = list(df.count(axis=0, numeric_only=True))
- for i in range(len(flow_ids)):
- axs.text(i + 1.05, np.nanmin(df.values), str(flow_freq[i]) + "\nrun(s)", fontsize=7)
- plt.show()
-
-
-plot_flow_compare(evals, metric=metric, top_n=10)
-# The boxplots below show how the flows perform across multiple runs on the chosen
-# task. The green horizontal lines represent the median accuracy of all the runs for
-# that flow (number of runs denoted at the bottom of the boxplots). The higher the
-# green line, the better the flow is for the task at hand. The ordering of the flows
-# are in the descending order of the higest accuracy value seen under that flow.
-
-# Printing the corresponding flow names for the top 10 performing flow IDs
-top_n = 10
-flow_ids = evals.flow_id.unique()[:top_n]
-flow_names = evals.flow_name.unique()[:top_n]
-for i in range(top_n):
- print((flow_ids[i], flow_names[i]))
-```
-
-::: {.image-sg .sphx-glr-single-img alt="Boxplot comparing predictive_accuracy for different flows" srcset="/examples/30_extended/images/sphx_glr_fetch_evaluations_tutorial_002.png"}
-/examples/30_extended/images/sphx_glr_fetch_evaluations_tutorial_002.png
-:::
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-(12736, 'sklearn.pipeline.Pipeline(simpleimputer=sklearn.impute._base.SimpleImputer,histgradientboostingclassifier=sklearn.ensemble._hist_gradient_boosting.gradient_boosting.HistGradientBoostingClassifier)(1)')
-(8399, 'sklearn.model_selection._search.RandomizedSearchCV(estimator=sklearn.pipeline.Pipeline(imputation=hyperimp.utils.preprocessing.ConditionalImputer,hotencoding=sklearn.preprocessing.data.OneHotEncoder,scaling=sklearn.preprocessing.data.StandardScaler,variencethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,clf=sklearn.svm.classes.SVC))(1)')
-(8353, 'sklearn.pipeline.Pipeline(imputation=hyperimp.utils.preprocessing.ConditionalImputer2,hotencoding=sklearn.preprocessing.data.OneHotEncoder,scaling=sklearn.preprocessing.data.StandardScaler,variencethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,clf=sklearn.svm.classes.SVC)(1)')
-(12127, 'sklearn.ensemble._hist_gradient_boosting.gradient_boosting.HistGradientBoostingClassifier(1)')
-(16347, 'sklearn.pipeline.Pipeline(simpleimputer=sklearn.impute._base.SimpleImputer,onehotencoder=sklearn.preprocessing._encoders.OneHotEncoder,svc=sklearn.svm.classes.SVC)(1)')
-(17373, 'sklearn.model_selection._search_successive_halving.HalvingRandomSearchCV(estimator=sklearn.ensemble._hist_gradient_boosting.gradient_boosting.HistGradientBoostingClassifier)(4)')
-(8317, 'sklearn.pipeline.Pipeline(imputation=hyperimp.utils.preprocessing.ConditionalImputer,hotencoding=sklearn.preprocessing.data.OneHotEncoder,scaling=sklearn.preprocessing.data.StandardScaler,variencethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,clf=sklearn.svm.classes.SVC)(1)')
-(17369, 'sklearn.model_selection._search_successive_halving.HalvingRandomSearchCV(estimator=sklearn.ensemble._hist_gradient_boosting.gradient_boosting.HistGradientBoostingClassifier)(3)')
-(17374, 'sklearn.model_selection._search.RandomizedSearchCV(estimator=sklearn.ensemble._hist_gradient_boosting.gradient_boosting.HistGradientBoostingClassifier)(4)')
-(17371, 'sklearn.model_selection._search.RandomizedSearchCV(estimator=sklearn.ensemble._hist_gradient_boosting.gradient_boosting.HistGradientBoostingClassifier)(3)')
-```
-:::
-
-### Obtaining evaluations with hyperparameter settings
-
-We\'ll now obtain the evaluations of a task and a flow with the
-hyperparameters
-
-``` default
-# List evaluations in descending order based on predictive_accuracy with
-# hyperparameters
-evals_setups = openml.evaluations.list_evaluations_setups(
- function="predictive_accuracy", tasks=[31], size=100, sort_order="desc"
-)
-
-""
-print(evals_setups.head())
-
-""
-# Return evaluations for flow_id in descending order based on predictive_accuracy
-# with hyperparameters. parameters_in_separate_columns returns parameters in
-# separate columns
-evals_setups = openml.evaluations.list_evaluations_setups(
- function="predictive_accuracy", flows=[6767], size=100, parameters_in_separate_columns=True
-)
-
-""
-print(evals_setups.head(10))
-
-""
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-run_id ... parameters
-0 6725054 ... {'mlr.classif.ranger(13)_num.trees': '48', 'ml...
-1 2083190 ... {'sklearn.ensemble.forest.RandomForestClassifi...
-2 3962979 ... {'mlr.classif.ranger(13)_num.trees': '42', 'ml...
-3 2083596 ... {'sklearn.ensemble.forest.RandomForestClassifi...
-4 6162649 ... {'mlr.classif.ranger(13)_num.trees': '715', 'm...
-
-[5 rows x 15 columns]
-run_id ... mlr.classif.xgboost(9)_nthread
-0 2420531 ... NaN
-1 2420532 ... NaN
-2 2420533 ... NaN
-3 2420534 ... NaN
-4 2420535 ... NaN
-5 2420536 ... NaN
-6 2420537 ... NaN
-7 2431903 ... NaN
-8 2432142 ... NaN
-9 2432377 ... NaN
-
-[10 rows x 29 columns]
-
-''
-```
-:::
-
-::: rst-class
-sphx-glr-timing
-
-**Total running time of the script:** ( 0 minutes 13.992 seconds)
-:::
-
-::::::: {#sphx_glr_download_examples_30_extended_fetch_evaluations_tutorial.py}
-:::::: only
-html
-
-::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
-::: {.container .sphx-glr-download .sphx-glr-download-python}
-`Download Python source code: fetch_evaluations_tutorial.py <fetch_evaluations_tutorial.py>`{.interpreted-text
-role="download"}
-:::
-
-::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
-`Download Jupyter notebook: fetch_evaluations_tutorial.ipynb <fetch_evaluations_tutorial.ipynb>`{.interpreted-text
-role="download"}
-:::
-:::::
-::::::
-:::::::
-
-:::: only
-html
-
-::: rst-class
-sphx-glr-signature
-
-[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
-:::
-::::
diff --git a/docs/examples/30_extended/fetch_runtimes_tutorial.rst.md b/docs/examples/30_extended/fetch_runtimes_tutorial.rst.md
deleted file mode 100644
index a5235fa32..000000000
--- a/docs/examples/30_extended/fetch_runtimes_tutorial.rst.md
+++ /dev/null
@@ -1,1135 +0,0 @@
-::::: only
-html
-
-:::: {.note .sphx-glr-download-link-note}
-::: title
-Note
-:::
-
-`Go to the end <sphx_glr_download_examples_30_extended_fetch_runtimes_tutorial.py>`{.interpreted-text
-role="ref"} to download the full example code
-::::
-:::::
-
-::: rst-class
-sphx-glr-example-title
-:::
-
-# Measuring runtimes for Scikit-learn models {#sphx_glr_examples_30_extended_fetch_runtimes_tutorial.py}
-
-The runtime of machine learning models on specific datasets can be a
-deciding factor on the choice of algorithms, especially for benchmarking
-and comparison purposes. OpenML\'s scikit-learn extension provides
-runtime data from runs of model fit and prediction on tasks or datasets,
-for both the CPU-clock as well as the actual wallclock-time incurred.
-The objective of this example is to illustrate how to retrieve such
-timing measures, and also offer some potential means of usage and
-interpretation of the same.
-
-It should be noted that there are multiple levels at which parallelism
-can occur.
-
-- At the outermost level, OpenML tasks contain fixed data splits, on
- which the defined model/flow is executed. Thus, a model can be fit
- on each OpenML dataset fold in parallel using the
- [n_jobs]{.title-ref} parameter to [run_model_on_task]{.title-ref} or
- [run_flow_on_task]{.title-ref} (illustrated under Case 2 & 3 below).
-- The model/flow specified can also include scikit-learn models that
- perform their own parallelization. For instance, by specifying
- [n_jobs]{.title-ref} in a Random Forest model definition (covered
- under Case 2 below).
-- The sklearn model can further be an HPO estimator and contain it\'s
- own parallelization. If the base estimator used also supports
- [parallelization]{.title-ref}, then there\'s at least a 2-level
- nested definition for parallelization possible (covered under Case 3
- below).
-
-We shall cover these 5 representative scenarios for:
-
-- (Case 1) Retrieving runtimes for Random Forest training and
- prediction on each of the cross-validation folds
-- (Case 2) Testing the above setting in a parallel setup and monitor
- the difference using runtimes retrieved
-- (Case 3) Comparing RandomSearchCV and GridSearchCV on the above task
- based on runtimes
-- (Case 4) Running models that don\'t run in parallel or models which
- scikit-learn doesn\'t parallelize
-- (Case 5) Running models that do not release the Python Global
- Interpreter Lock (GIL)
-
-``` default
-# License: BSD 3-Clause
-
-import openml
-import numpy as np
-from matplotlib import pyplot as plt
-from joblib.parallel import parallel_backend
-
-from sklearn.naive_bayes import GaussianNB
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.neural_network import MLPClassifier
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
-```
-
-## Preparing tasks and scikit-learn models
-
-``` default
-task_id = 167119
-
-task = openml.tasks.get_task(task_id)
-print(task)
-
-# Viewing associated data
-n_repeats, n_folds, n_samples = task.get_split_dimensions()
-print(
- "Task {}: number of repeats: {}, number of folds: {}, number of samples {}.".format(
- task_id,
- n_repeats,
- n_folds,
- n_samples,
- )
-)
-
-
-# Creating utility function
-def print_compare_runtimes(measures):
- for repeat, val1 in measures["usercpu_time_millis_training"].items():
- for fold, val2 in val1.items():
- print(
- "Repeat #{}-Fold #{}: CPU-{:.3f} vs Wall-{:.3f}".format(
- repeat, fold, val2, measures["wall_clock_time_millis_training"][repeat][fold]
- )
- )
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-OpenML Classification Task
-==========================
-Task Type Description: https://www.openml.org/tt/TaskType.SUPERVISED_CLASSIFICATION
-Task ID..............: 167119
-Task URL.............: https://www.openml.org/t/167119
-Estimation Procedure.: crossvalidation
-Target Feature.......: class
-# of Classes.........: 3
-Cost Matrix..........: Available
-Task 167119: number of repeats: 1, number of folds: 10, number of samples 1.
-```
-:::
-
-## Case 1: Running a Random Forest model on an OpenML task
-
-We\'ll run a Random Forest model and obtain an OpenML run object. We can
-see the evaluations recorded per fold for the dataset and the
-information available for this run.
-
-``` default
-clf = RandomForestClassifier(n_estimators=10)
-
-run1 = openml.runs.run_model_on_task(
- model=clf,
- task=task,
- upload_flow=False,
- avoid_duplicate_runs=False,
-)
-measures = run1.fold_evaluations
-
-print("The timing and performance metrics available: ")
-for key in measures.keys():
- print(key)
-print()
-
-print(
- "The performance metric is recorded under `predictive_accuracy` per "
- "fold and can be retrieved as: "
-)
-for repeat, val1 in measures["predictive_accuracy"].items():
- for fold, val2 in val1.items():
- print("Repeat #{}-Fold #{}: {:.4f}".format(repeat, fold, val2))
- print()
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-The timing and performance metrics available:
-usercpu_time_millis_training
-wall_clock_time_millis_training
-usercpu_time_millis_testing
-usercpu_time_millis
-wall_clock_time_millis_testing
-wall_clock_time_millis
-predictive_accuracy
-
-The performance metric is recorded under `predictive_accuracy` per fold and can be retrieved as:
-Repeat #0-Fold #0: 0.7834
-Repeat #0-Fold #1: 0.7773
-Repeat #0-Fold #2: 0.7813
-Repeat #0-Fold #3: 0.7811
-Repeat #0-Fold #4: 0.7863
-Repeat #0-Fold #5: 0.7829
-Repeat #0-Fold #6: 0.7718
-Repeat #0-Fold #7: 0.7798
-Repeat #0-Fold #8: 0.7825
-Repeat #0-Fold #9: 0.7735
-```
-:::
-
-The remaining entries recorded in [measures]{.title-ref} are the runtime
-records related as:
-
-usercpu_time_millis = usercpu_time_millis_training +
-usercpu_time_millis_testing
-
-wall_clock_time_millis = wall_clock_time_millis_training +
-wall_clock_time_millis_testing
-
-The timing measures recorded as [\*\_millis_training]{.title-ref}
-contain the per repeat-per fold timing incurred for the execution of the
-[.fit()]{.title-ref} procedure of the model. For
-[usercpu_time\_\*]{.title-ref} the time recorded using
-[time.process_time()]{.title-ref} is converted to
-[milliseconds]{.title-ref} and stored. Similarly,
-[time.time()]{.title-ref} is used to record the time entry for
-[wall_clock_time\_\*]{.title-ref}. The [\*\_millis_testing]{.title-ref}
-entry follows the same procedure but for time taken for the
-[.predict()]{.title-ref} procedure.
-
-``` default
-# Comparing the CPU and wall-clock training times of the Random Forest model
-print_compare_runtimes(measures)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-Repeat #0-Fold #0: CPU-235.799 vs Wall-235.806
-Repeat #0-Fold #1: CPU-234.915 vs Wall-234.942
-Repeat #0-Fold #2: CPU-215.000 vs Wall-215.677
-Repeat #0-Fold #3: CPU-218.780 vs Wall-218.210
-Repeat #0-Fold #4: CPU-210.419 vs Wall-210.423
-Repeat #0-Fold #5: CPU-210.813 vs Wall-211.084
-Repeat #0-Fold #6: CPU-210.696 vs Wall-218.624
-Repeat #0-Fold #7: CPU-206.779 vs Wall-206.782
-Repeat #0-Fold #8: CPU-219.276 vs Wall-218.185
-Repeat #0-Fold #9: CPU-211.948 vs Wall-211.660
-```
-:::
-
-## Case 2: Running Scikit-learn model on an OpenML task in parallel
-
-Redefining the model to allow parallelism with [n_jobs=2]{.title-ref} (2
-cores)
-
-``` default
-clf = RandomForestClassifier(n_estimators=10, n_jobs=2)
-
-run2 = openml.runs.run_model_on_task(
- model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False
-)
-measures = run2.fold_evaluations
-# The wall-clock time recorded per fold should be lesser than Case 1 above
-print_compare_runtimes(measures)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-Repeat #0-Fold #0: CPU-291.231 vs Wall-177.857
-Repeat #0-Fold #1: CPU-247.121 vs Wall-160.623
-Repeat #0-Fold #2: CPU-299.050 vs Wall-171.314
-Repeat #0-Fold #3: CPU-219.565 vs Wall-130.540
-Repeat #0-Fold #4: CPU-214.352 vs Wall-128.408
-Repeat #0-Fold #5: CPU-214.530 vs Wall-129.069
-Repeat #0-Fold #6: CPU-216.249 vs Wall-131.271
-Repeat #0-Fold #7: CPU-214.707 vs Wall-129.280
-Repeat #0-Fold #8: CPU-215.784 vs Wall-130.975
-Repeat #0-Fold #9: CPU-224.041 vs Wall-127.277
-```
-:::
-
-Running a Random Forest model on an OpenML task in parallel (all cores
-available):
-
-``` default
-# Redefining the model to use all available cores with `n_jobs=-1`
-clf = RandomForestClassifier(n_estimators=10, n_jobs=-1)
-
-run3 = openml.runs.run_model_on_task(
- model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False
-)
-measures = run3.fold_evaluations
-# The wall-clock time recorded per fold should be lesser than the case above,
-# if more than 2 CPU cores are available. The speed-up is more pronounced for
-# larger datasets.
-print_compare_runtimes(measures)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-Repeat #0-Fold #0: CPU-223.135 vs Wall-141.702
-Repeat #0-Fold #1: CPU-252.852 vs Wall-157.798
-Repeat #0-Fold #2: CPU-225.624 vs Wall-144.946
-Repeat #0-Fold #3: CPU-215.932 vs Wall-137.153
-Repeat #0-Fold #4: CPU-218.327 vs Wall-132.138
-Repeat #0-Fold #5: CPU-216.836 vs Wall-128.753
-Repeat #0-Fold #6: CPU-217.526 vs Wall-141.789
-Repeat #0-Fold #7: CPU-218.256 vs Wall-135.525
-Repeat #0-Fold #8: CPU-218.904 vs Wall-134.427
-Repeat #0-Fold #9: CPU-221.687 vs Wall-130.505
-```
-:::
-
-We can now observe that the ratio of CPU time to wallclock time is lower
-than in case 1. This happens because joblib by default spawns
-subprocesses for the workloads for which CPU time cannot be tracked.
-Therefore, interpreting the reported CPU and wallclock time requires
-knowledge of the parallelization applied at runtime.
-
-Running the same task with a different parallel backend. Joblib provides
-multiple backends: {[loky]{.title-ref} (default),
-[multiprocessing]{.title-ref}, [dask]{.title-ref},
-[threading]{.title-ref}, [sequential]{.title-ref}}. The backend can be
-explicitly set using a joblib context manager. The behaviour of the job
-distribution can change and therefore the scale of runtimes recorded
-too.
-
-``` default
-with parallel_backend(backend="multiprocessing", n_jobs=-1):
- run3_ = openml.runs.run_model_on_task(
- model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False
- )
-measures = run3_.fold_evaluations
-print_compare_runtimes(measures)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-Repeat #0-Fold #0: CPU-324.893 vs Wall-545.866
-Repeat #0-Fold #1: CPU-248.720 vs Wall-412.322
-Repeat #0-Fold #2: CPU-269.108 vs Wall-424.729
-Repeat #0-Fold #3: CPU-264.310 vs Wall-401.721
-Repeat #0-Fold #4: CPU-262.941 vs Wall-379.026
-Repeat #0-Fold #5: CPU-242.700 vs Wall-376.538
-Repeat #0-Fold #6: CPU-257.596 vs Wall-418.498
-Repeat #0-Fold #7: CPU-233.435 vs Wall-437.289
-Repeat #0-Fold #8: CPU-257.681 vs Wall-409.761
-Repeat #0-Fold #9: CPU-282.320 vs Wall-428.106
-```
-:::
-
-The CPU time interpretation becomes ambiguous when jobs are distributed
-over an unknown number of cores or when subprocesses are spawned for
-which the CPU time cannot be tracked, as in the examples above. It is
-impossible for OpenML-Python to capture the availability of the number
-of cores/threads, their eventual utilisation and whether workloads are
-executed in subprocesses, for various cases that can arise as
-demonstrated in the rest of the example. Therefore, the final
-interpretation of the runtimes is left to the [user]{.title-ref}.
-
-## Case 3: Running and benchmarking HPO algorithms with their runtimes
-
-We shall now optimize a similar RandomForest model for the same task
-using scikit-learn\'s HPO support by using GridSearchCV to optimize our
-earlier RandomForest model\'s hyperparameter [n_estimators]{.title-ref}.
-Scikit-learn also provides a [refit_time\_]{.title-ref} for such HPO
-models, i.e., the time incurred by training and evaluating the model on
-the best found parameter setting. This is included in the
-[wall_clock_time_millis_training]{.title-ref} measure recorded.
-
-``` default
-from sklearn.model_selection import GridSearchCV
-
-
-clf = RandomForestClassifier(n_estimators=10, n_jobs=2)
-
-# GridSearchCV model
-n_iter = 5
-grid_pipe = GridSearchCV(
- estimator=clf,
- param_grid={"n_estimators": np.linspace(start=1, stop=50, num=n_iter).astype(int).tolist()},
- cv=2,
- n_jobs=2,
-)
-
-run4 = openml.runs.run_model_on_task(
- model=grid_pipe, task=task, upload_flow=False, avoid_duplicate_runs=False, n_jobs=2
-)
-measures = run4.fold_evaluations
-print_compare_runtimes(measures)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-Repeat #0-Fold #0: CPU-4216.657 vs Wall-3927.152
-Repeat #0-Fold #1: CPU-4267.256 vs Wall-3532.315
-Repeat #0-Fold #2: CPU-4153.419 vs Wall-3756.332
-Repeat #0-Fold #3: CPU-4165.637 vs Wall-3484.633
-Repeat #0-Fold #4: CPU-4148.764 vs Wall-3386.673
-Repeat #0-Fold #5: CPU-4409.518 vs Wall-3800.539
-Repeat #0-Fold #6: CPU-4381.566 vs Wall-3520.530
-Repeat #0-Fold #7: CPU-4377.360 vs Wall-3966.456
-Repeat #0-Fold #8: CPU-4427.580 vs Wall-3939.647
-Repeat #0-Fold #9: CPU-4459.380 vs Wall-3545.670
-```
-:::
-
-Like any optimisation problem, scikit-learn\'s HPO estimators also
-generate a sequence of configurations which are evaluated, using which
-the best found configuration is tracked throughout the trace. The OpenML
-run object stores these traces as OpenMLRunTrace objects accessible
-using keys of the pattern (repeat, fold, iterations). Here
-[fold]{.title-ref} implies the outer-cross validation fold as obtained
-from the task data splits in OpenML. GridSearchCV here performs grid
-search over the inner-cross validation folds as parameterized by the
-[cv]{.title-ref} parameter. Since [GridSearchCV]{.title-ref} in this
-example performs a [2-fold]{.title-ref} cross validation, the runtime
-recorded per repeat-per fold in the run object is for the entire
-[fit()]{.title-ref} procedure of GridSearchCV thus subsuming the
-runtimes of the 2-fold (inner) CV search performed.
-
-``` default
-# We earlier extracted the number of repeats and folds for this task:
-print("# repeats: {}\n# folds: {}".format(n_repeats, n_folds))
-
-# To extract the training runtime of the first repeat, first fold:
-print(run4.fold_evaluations["wall_clock_time_millis_training"][0][0])
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-# repeats: 1
-# folds: 10
-3927.151918411255
-```
-:::
-
-To extract the training runtime of the 1-st repeat, 4-th (outer) fold
-and also to fetch the parameters and performance of the evaluations made
-during the 1-st repeat, 4-th fold evaluation by the Grid Search model.
-
-``` default
-_repeat = 0
-_fold = 3
-print(
- "Total runtime for repeat {}'s fold {}: {:4f} ms".format(
- _repeat, _fold, run4.fold_evaluations["wall_clock_time_millis_training"][_repeat][_fold]
- )
-)
-for i in range(n_iter):
- key = (_repeat, _fold, i)
- r = run4.trace.trace_iterations[key]
- print(
- "n_estimators: {:>2} - score: {:.3f}".format(
- r.parameters["parameter_n_estimators"], r.evaluation
- )
- )
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-Total runtime for repeat 0's fold 3: 3484.633446 ms
-n_estimators: 1 - score: 0.759
-n_estimators: 13 - score: 0.799
-n_estimators: 25 - score: 0.803
-n_estimators: 37 - score: 0.804
-n_estimators: 50 - score: 0.803
-```
-:::
-
-Scikit-learn\'s HPO estimators also come with an argument
-[refit=True]{.title-ref} as a default. In our previous model definition
-it was set to True by default, which meant that the best found
-hyperparameter configuration was used to refit or retrain the model
-without any inner cross validation. This extra refit time measure is
-provided by the scikit-learn model as the attribute
-[refit_time\_]{.title-ref}. This time is included in the
-[wall_clock_time_millis_training]{.title-ref} measure.
-
-For non-HPO estimators, [wall_clock_time_millis =
-wall_clock_time_millis_training +
-wall_clock_time_millis_testing]{.title-ref}.
-
-For HPO estimators, [wall_clock_time_millis =
-wall_clock_time_millis_training + wall_clock_time_millis_testing +
-refit_time]{.title-ref}.
-
-This refit time can therefore be explicitly extracted in this manner:
-
-``` default
-def extract_refit_time(run, repeat, fold):
- refit_time = (
- run.fold_evaluations["wall_clock_time_millis"][repeat][fold]
- - run.fold_evaluations["wall_clock_time_millis_training"][repeat][fold]
- - run.fold_evaluations["wall_clock_time_millis_testing"][repeat][fold]
- )
- return refit_time
-
-
-for repeat in range(n_repeats):
- for fold in range(n_folds):
- print(
- "Repeat #{}-Fold #{}: {:.4f}".format(
- repeat, fold, extract_refit_time(run4, repeat, fold)
- )
- )
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-Repeat #0-Fold #0: 665.5693
-Repeat #0-Fold #1: 539.9771
-Repeat #0-Fold #2: 706.4621
-Repeat #0-Fold #3: 634.3455
-Repeat #0-Fold #4: 598.8989
-Repeat #0-Fold #5: 755.9273
-Repeat #0-Fold #6: 806.9417
-Repeat #0-Fold #7: 764.6010
-Repeat #0-Fold #8: 825.0468
-Repeat #0-Fold #9: 550.1802
-```
-:::
-
-Along with the GridSearchCV already used above, we demonstrate how such
-optimisation traces can be retrieved by showing an application of these
-traces - comparing the speed of finding the best configuration using
-RandomizedSearchCV and GridSearchCV available with scikit-learn.
-
-``` default
-# RandomizedSearchCV model
-rs_pipe = RandomizedSearchCV(
- estimator=clf,
- param_distributions={
- "n_estimators": np.linspace(start=1, stop=50, num=15).astype(int).tolist()
- },
- cv=2,
- n_iter=n_iter,
- n_jobs=2,
-)
-run5 = openml.runs.run_model_on_task(
- model=rs_pipe, task=task, upload_flow=False, avoid_duplicate_runs=False, n_jobs=2
-)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-```
-:::
-
-Since for the call to `openml.runs.run_model_on_task` the parameter
-`n_jobs` is set to its default `None`, the evaluations across the OpenML
-folds are not parallelized. Hence, the time recorded is agnostic to the
-`n_jobs` being set at both the HPO estimator `GridSearchCV` as well as
-the base estimator `RandomForestClassifier` in this case. The OpenML
-extension only records the time taken for the completion of the complete
-`fit()` call, per-repeat per-fold.
-
-This notion can be used to extract and plot the best found performance
-per fold by the HPO model and the corresponding time taken for search
-across that fold. Moreover, since `n_jobs=None` for
-`openml.runs.run_model_on_task` the runtimes per fold can be
-cumulatively added to plot the trace against time.
-
-``` default
-def extract_trace_data(run, n_repeats, n_folds, n_iter, key=None):
- key = "wall_clock_time_millis_training" if key is None else key
- data = {"score": [], "runtime": []}
- for i_r in range(n_repeats):
- for i_f in range(n_folds):
- data["runtime"].append(run.fold_evaluations[key][i_r][i_f])
- for i_i in range(n_iter):
- r = run.trace.trace_iterations[(i_r, i_f, i_i)]
- if r.selected:
- data["score"].append(r.evaluation)
- break
- return data
-
-
-def get_incumbent_trace(trace):
- best_score = 1
- inc_trace = []
- for i, r in enumerate(trace):
- if i == 0 or (1 - r) < best_score:
- best_score = 1 - r
- inc_trace.append(best_score)
- return inc_trace
-
-
-grid_data = extract_trace_data(run4, n_repeats, n_folds, n_iter)
-rs_data = extract_trace_data(run5, n_repeats, n_folds, n_iter)
-
-plt.clf()
-plt.plot(
- np.cumsum(grid_data["runtime"]), get_incumbent_trace(grid_data["score"]), label="Grid Search"
-)
-plt.plot(
- np.cumsum(rs_data["runtime"]), get_incumbent_trace(rs_data["score"]), label="Random Search"
-)
-plt.xscale("log")
-plt.yscale("log")
-plt.xlabel("Wallclock time (in milliseconds)")
-plt.ylabel("1 - Accuracy")
-plt.title("Optimisation Trace Comparison")
-plt.legend()
-plt.show()
-```
-
-::: {.image-sg .sphx-glr-single-img alt="Optimisation Trace Comparison" srcset="/examples/30_extended/images/sphx_glr_fetch_runtimes_tutorial_001.png"}
-/examples/30_extended/images/sphx_glr_fetch_runtimes_tutorial_001.png
-:::
-
-## Case 4: Running models that scikit-learn doesn\'t parallelize
-
-Both scikit-learn and OpenML depend on parallelism implemented through
-[joblib]{.title-ref}. However, there can be cases where either models
-cannot be parallelized or don\'t depend on joblib for its parallelism. 2
-such cases are illustrated below.
-
-Running a Decision Tree model that doesn\'t support parallelism
-implicitly, but using OpenML to parallelize evaluations for the
-outer-cross validation folds.
-
-``` default
-dt = DecisionTreeClassifier()
-
-run6 = openml.runs.run_model_on_task(
- model=dt, task=task, upload_flow=False, avoid_duplicate_runs=False, n_jobs=2
-)
-measures = run6.fold_evaluations
-print_compare_runtimes(measures)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-Repeat #0-Fold #0: CPU-73.760 vs Wall-73.762
-Repeat #0-Fold #1: CPU-72.873 vs Wall-72.874
-Repeat #0-Fold #2: CPU-72.735 vs Wall-72.745
-Repeat #0-Fold #3: CPU-74.092 vs Wall-74.092
-Repeat #0-Fold #4: CPU-73.657 vs Wall-73.658
-Repeat #0-Fold #5: CPU-72.254 vs Wall-72.257
-Repeat #0-Fold #6: CPU-73.470 vs Wall-73.470
-Repeat #0-Fold #7: CPU-72.530 vs Wall-72.534
-Repeat #0-Fold #8: CPU-74.001 vs Wall-74.018
-Repeat #0-Fold #9: CPU-73.600 vs Wall-73.602
-```
-:::
-
-Although the decision tree does not run in parallel, it can release the
-[Python
-GIL](https://docs.python.org/dev/glossary.html#term-global-interpreter-lock).
-This can result in surprising runtime measures as demonstrated below:
-
-``` default
-with parallel_backend("threading", n_jobs=-1):
- run7 = openml.runs.run_model_on_task(
- model=dt, task=task, upload_flow=False, avoid_duplicate_runs=False
- )
-measures = run7.fold_evaluations
-print_compare_runtimes(measures)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-Repeat #0-Fold #0: CPU-80.470 vs Wall-80.463
-Repeat #0-Fold #1: CPU-74.982 vs Wall-74.985
-Repeat #0-Fold #2: CPU-83.721 vs Wall-83.721
-Repeat #0-Fold #3: CPU-88.097 vs Wall-88.118
-Repeat #0-Fold #4: CPU-77.615 vs Wall-77.624
-Repeat #0-Fold #5: CPU-95.236 vs Wall-95.238
-Repeat #0-Fold #6: CPU-98.978 vs Wall-100.188
-Repeat #0-Fold #7: CPU-83.243 vs Wall-83.185
-Repeat #0-Fold #8: CPU-89.183 vs Wall-89.200
-Repeat #0-Fold #9: CPU-97.797 vs Wall-95.633
-```
-:::
-
-Running a Neural Network from scikit-learn that uses scikit-learn
-independent parallelism using libraries such as [MKL, OpenBLAS or
-BLIS](https://scikit-learn.org/stable/computing/parallelism.html#parallel-numpy-and-scipy-routines-from-numerical-libraries).
-
-``` default
-mlp = MLPClassifier(max_iter=10)
-
-run8 = openml.runs.run_model_on_task(
- model=mlp, task=task, upload_flow=False, avoid_duplicate_runs=False
-)
-measures = run8.fold_evaluations
-print_compare_runtimes(measures)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/openml/venv/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/openml/venv/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/openml/venv/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/openml/venv/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/openml/venv/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/openml/venv/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/openml/venv/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/openml/venv/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/openml/venv/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/openml/venv/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.
- warnings.warn(
-Repeat #0-Fold #0: CPU-866.453 vs Wall-866.533
-Repeat #0-Fold #1: CPU-918.903 vs Wall-756.131
-Repeat #0-Fold #2: CPU-997.260 vs Wall-727.315
-Repeat #0-Fold #3: CPU-859.797 vs Wall-657.584
-Repeat #0-Fold #4: CPU-922.494 vs Wall-701.211
-Repeat #0-Fold #5: CPU-1112.121 vs Wall-817.650
-Repeat #0-Fold #6: CPU-908.489 vs Wall-682.244
-Repeat #0-Fold #7: CPU-889.331 vs Wall-624.544
-Repeat #0-Fold #8: CPU-908.252 vs Wall-630.604
-Repeat #0-Fold #9: CPU-888.762 vs Wall-654.652
-```
-:::
-
-## Case 5: Running Scikit-learn models that don\'t release GIL
-
-Certain Scikit-learn models do not release the [Python
-GIL](https://docs.python.org/dev/glossary.html#term-global-interpreter-lock)
-and are also not executed in parallel via a BLAS library. In such cases,
-the CPU times and wallclock times are most likely trustworthy. Note
-however that only very few models such as naive Bayes models are of this
-kind.
-
-``` default
-clf = GaussianNB()
-
-with parallel_backend("multiprocessing", n_jobs=-1):
- run9 = openml.runs.run_model_on_task(
- model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False
- )
-measures = run9.fold_evaluations
-print_compare_runtimes(measures)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-Repeat #0-Fold #0: CPU-26.803 vs Wall-26.800
-Repeat #0-Fold #1: CPU-26.351 vs Wall-26.349
-Repeat #0-Fold #2: CPU-26.371 vs Wall-26.370
-Repeat #0-Fold #3: CPU-26.402 vs Wall-26.400
-Repeat #0-Fold #4: CPU-26.782 vs Wall-26.781
-Repeat #0-Fold #5: CPU-26.479 vs Wall-26.479
-Repeat #0-Fold #6: CPU-26.393 vs Wall-26.390
-Repeat #0-Fold #7: CPU-26.430 vs Wall-26.427
-Repeat #0-Fold #8: CPU-26.654 vs Wall-26.657
-Repeat #0-Fold #9: CPU-26.409 vs Wall-26.406
-```
-:::
-
-## Summmary
-
-The scikit-learn extension for OpenML-Python records model runtimes for
-the CPU-clock and the wall-clock times. The above examples illustrated
-how these recorded runtimes can be extracted when using a scikit-learn
-model and under parallel setups too. To summarize, the scikit-learn
-extension measures the:
-
-- [CPU-time]{.title-ref} & [wallclock-time]{.title-ref} for the whole
- run
- - A run here corresponds to a call to
- [run_model_on_task]{.title-ref} or
- [run_flow_on_task]{.title-ref}
- - The recorded time is for the model fit for each of the
- outer-cross validations folds, i.e., the OpenML data splits
-- Python\'s [time]{.title-ref} module is used to compute the runtimes
- - [CPU-time]{.title-ref} is recorded using the responses of
- [time.process_time()]{.title-ref}
- - [wallclock-time]{.title-ref} is recorded using the responses of
- [time.time()]{.title-ref}
-- The timings recorded by OpenML per outer-cross validation fold is
- agnostic to model parallelisation
- - The wallclock times reported in Case 2 above highlights the
- speed-up on using [n_jobs=-1]{.title-ref} in comparison to
- [n_jobs=2]{.title-ref}, since the timing recorded by OpenML is
- for the entire [fit()]{.title-ref} procedure, whereas the
- parallelisation is performed inside [fit()]{.title-ref} by
- scikit-learn
- - The CPU-time for models that are run in parallel can be
- difficult to interpret
-- [CPU-time]{.title-ref} & [wallclock-time]{.title-ref} for each
- search per outer fold in an HPO run
- - Reports the total time for performing search on each of the
- OpenML data split, subsuming any sort of parallelism that
- happened as part of the HPO estimator or the underlying base
- estimator
- - Also allows extraction of the [refit_time]{.title-ref} that
- scikit-learn measures using [time.time()]{.title-ref} for
- retraining the model per outer fold, for the best found
- configuration
-- [CPU-time]{.title-ref} & [wallclock-time]{.title-ref} for models
- that scikit-learn doesn\'t parallelize
- - Models like Decision Trees or naive Bayes don\'t parallelize and
- thus both the wallclock and CPU times are similar in runtime for
- the OpenML call
- - However, models implemented in Cython, such as the Decision
- Trees can release the GIL and still run in parallel if a
- [threading]{.title-ref} backend is used by joblib.
- - Scikit-learn Neural Networks can undergo parallelization
- implicitly owing to thread-level parallelism involved in the
- linear algebraic operations and thus the wallclock-time and
- CPU-time can differ.
-
-Because of all the cases mentioned above it is crucial to understand
-which case is triggered when reporting runtimes for scikit-learn models
-measured with OpenML-Python!
-
-::: rst-class
-sphx-glr-timing
-
-**Total running time of the script:** ( 1 minutes 13.261 seconds)
-:::
-
-::::::: {#sphx_glr_download_examples_30_extended_fetch_runtimes_tutorial.py}
-:::::: only
-html
-
-::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
-::: {.container .sphx-glr-download .sphx-glr-download-python}
-`Download Python source code: fetch_runtimes_tutorial.py <fetch_runtimes_tutorial.py>`{.interpreted-text
-role="download"}
-:::
-
-::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
-`Download Jupyter notebook: fetch_runtimes_tutorial.ipynb <fetch_runtimes_tutorial.ipynb>`{.interpreted-text
-role="download"}
-:::
-:::::
-::::::
-:::::::
-
-:::: only
-html
-
-::: rst-class
-sphx-glr-signature
-
-[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
-:::
-::::
diff --git a/docs/examples/30_extended/flow_id_tutorial.rst.md b/docs/examples/30_extended/flow_id_tutorial.rst.md
deleted file mode 100644
index 62e3049e8..000000000
--- a/docs/examples/30_extended/flow_id_tutorial.rst.md
+++ /dev/null
@@ -1,208 +0,0 @@
-::::: only
-html
-
-:::: {.note .sphx-glr-download-link-note}
-::: title
-Note
-:::
-
-`Go to the end <sphx_glr_download_examples_30_extended_flow_id_tutorial.py>`{.interpreted-text
-role="ref"} to download the full example code
-::::
-:::::
-
-::: rst-class
-sphx-glr-example-title
-:::
-
-# Obtaining Flow IDs {#sphx_glr_examples_30_extended_flow_id_tutorial.py}
-
-This tutorial discusses different ways to obtain the ID of a flow in
-order to perform further analysis.
-
-``` default
-# License: BSD 3-Clause
-
-import sklearn.tree
-
-import openml
-```
-
-:::: warning
-::: title
-Warning
-:::
-
-.. include:: ../../test_server_usage_warning.txt
-::::
-
-``` default
-openml.config.start_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!
- warnings.warn(
-```
-:::
-
-Defining a classifier
-
-``` default
-clf = sklearn.tree.DecisionTreeClassifier()
-```
-
-## 1. Obtaining a flow given a classifier
-
-``` default
-flow = openml.extensions.get_extension_by_model(clf).model_to_flow(clf).publish()
-flow_id = flow.flow_id
-print(flow_id)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-4595
-```
-:::
-
-This piece of code is rather involved. First, it retrieves a
-`~openml.extensions.Extension`{.interpreted-text role="class"} which is
-registered and can handle the given model, in our case it is
-`openml.extensions.sklearn.SklearnExtension`{.interpreted-text
-role="class"}. Second, the extension converts the classifier into an
-instance of `openml.OpenMLFlow`{.interpreted-text role="class"}. Third
-and finally, the publish method checks whether the current flow is
-already present on OpenML. If not, it uploads the flow, otherwise, it
-updates the current instance with all information computed by the server
-(which is obviously also done when uploading/publishing a flow).
-
-To simplify the usage we have created a helper function which automates
-all these steps:
-
-``` default
-flow_id = openml.flows.get_flow_id(model=clf)
-print(flow_id)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-4595
-```
-:::
-
-## 2. Obtaining a flow given its name
-
-The schema of a flow is given in XSD
-([here](https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/openml.implementation.upload.xsd)).
-\# noqa E501 Only two fields are required, a unique name, and an
-external version. While it should be pretty obvious why we need a name,
-the need for the additional external version information might not be
-immediately clear. However, this information is very important as it
-allows to have multiple flows with the same name for different versions
-of a software. This might be necessary if an algorithm or implementation
-introduces, renames or drop hyperparameters over time.
-
-``` default
-print(flow.name, flow.external_version)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-sklearn.tree._classes.DecisionTreeClassifier openml==0.14.1,sklearn==1.3.0
-```
-:::
-
-The name and external version are automatically added to a flow when
-constructing it from a model. We can then use them to retrieve the flow
-id as follows:
-
-``` default
-flow_id = openml.flows.flow_exists(name=flow.name, external_version=flow.external_version)
-print(flow_id)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-4595
-```
-:::
-
-We can also retrieve all flows for a given name:
-
-``` default
-flow_ids = openml.flows.get_flow_id(name=flow.name)
-print(flow_ids)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-[22, 26, 30, 267, 1178, 4102, 4112, 4127, 4595, 6211]
-```
-:::
-
-This also works with the actual model (generalizing the first part of
-this example):
-
-``` default
-flow_ids = openml.flows.get_flow_id(model=clf, exact_version=False)
-print(flow_ids)
-
-# Deactivating test server
-openml.config.stop_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-[22, 26, 30, 267, 1178, 4102, 4112, 4127, 4595, 6211]
-```
-:::
-
-::: rst-class
-sphx-glr-timing
-
-**Total running time of the script:** ( 0 minutes 4.274 seconds)
-:::
-
-::::::: {#sphx_glr_download_examples_30_extended_flow_id_tutorial.py}
-:::::: only
-html
-
-::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
-::: {.container .sphx-glr-download .sphx-glr-download-python}
-`Download Python source code: flow_id_tutorial.py <flow_id_tutorial.py>`{.interpreted-text
-role="download"}
-:::
-
-::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
-`Download Jupyter notebook: flow_id_tutorial.ipynb <flow_id_tutorial.ipynb>`{.interpreted-text
-role="download"}
-:::
-:::::
-::::::
-:::::::
-
-:::: only
-html
-
-::: rst-class
-sphx-glr-signature
-
-[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
-:::
-::::
diff --git a/docs/examples/30_extended/flows_and_runs_tutorial.rst.md b/docs/examples/30_extended/flows_and_runs_tutorial.rst.md
deleted file mode 100644
index 7b5066735..000000000
--- a/docs/examples/30_extended/flows_and_runs_tutorial.rst.md
+++ /dev/null
@@ -1,604 +0,0 @@
-::::: only
-html
-
-:::: {.note .sphx-glr-download-link-note}
-::: title
-Note
-:::
-
-`Go to the end <sphx_glr_download_examples_30_extended_flows_and_runs_tutorial.py>`{.interpreted-text
-role="ref"} to download the full example code
-::::
-:::::
-
-::: rst-class
-sphx-glr-example-title
-:::
-
-# Flows and Runs {#sphx_glr_examples_30_extended_flows_and_runs_tutorial.py}
-
-How to train/run a model and how to upload the results.
-
-``` default
-# License: BSD 3-Clause
-
-import openml
-from sklearn import compose, ensemble, impute, neighbors, preprocessing, pipeline, tree
-```
-
-We\'ll use the test server for the rest of this tutorial.
-
-:::: warning
-::: title
-Warning
-:::
-
-.. include:: ../../test_server_usage_warning.txt
-::::
-
-``` default
-openml.config.start_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!
- warnings.warn(
-```
-:::
-
-## Train machine learning models
-
-Train a scikit-learn model on the data manually.
-
-``` default
-# NOTE: We are using dataset 68 from the test server: https://test.openml.org/d/68
-dataset = openml.datasets.get_dataset(68)
-X, y, categorical_indicator, attribute_names = dataset.get_data(
- target=dataset.default_target_attribute
-)
-clf = neighbors.KNeighborsClassifier(n_neighbors=1)
-clf.fit(X, y)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-```
-:::
-
-<div class="output_subarea output_html rendered_html output_result">
-<style>#sk-container-id-2 {color: black;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: "▸";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: "▾";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: "";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: "";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: "";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id="sk-container-id-2" class="sk-top-container"><div class="sk-text-repr-fallback"><pre>KNeighborsClassifier(n_neighbors=1)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class="sk-container" hidden><div class="sk-item"><div class="sk-estimator sk-toggleable"><input class="sk-toggleable__control sk-hidden--visually" id="sk-estimator-id-2" type="checkbox" checked><label for="sk-estimator-id-2" class="sk-toggleable__label sk-toggleable__label-arrow">KNeighborsClassifier</label><div class="sk-toggleable__content"><pre>KNeighborsClassifier(n_neighbors=1)</pre></div></div></div></div></div>
-</div>
-<br />
-<br />
-
-You can also ask for meta-data to automatically preprocess the data.
-
-- e.g. categorical features -\> do feature encoding
-
-``` default
-dataset = openml.datasets.get_dataset(17)
-X, y, categorical_indicator, attribute_names = dataset.get_data(
- target=dataset.default_target_attribute
-)
-print(f"Categorical features: {categorical_indicator}")
-transformer = compose.ColumnTransformer(
- [("one_hot_encoder", preprocessing.OneHotEncoder(categories="auto"), categorical_indicator)]
-)
-X = transformer.fit_transform(X)
-clf.fit(X, y)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-Categorical features: [True, False, True, True, False, True, True, False, True, True, False, True, False, True, True, False, True, False, True, True]
-```
-:::
-
-<div class="output_subarea output_html rendered_html output_result">
-<style>#sk-container-id-3 {color: black;}#sk-container-id-3 pre{padding: 0;}#sk-container-id-3 div.sk-toggleable {background-color: white;}#sk-container-id-3 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-3 label.sk-toggleable__label-arrow:before {content: "▸";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-3 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-3 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-3 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: "▾";}#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-3 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-3 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-3 div.sk-parallel-item::after {content: "";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-serial::before {content: "";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-3 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-3 div.sk-item {position: relative;z-index: 1;}#sk-container-id-3 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-3 div.sk-item::before, #sk-container-id-3 div.sk-parallel-item::before {content: "";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-3 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-3 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-3 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-3 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-3 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-3 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-3 div.sk-label-container {text-align: center;}#sk-container-id-3 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-3 div.sk-text-repr-fallback {display: none;}</style><div id="sk-container-id-3" class="sk-top-container"><div class="sk-text-repr-fallback"><pre>KNeighborsClassifier(n_neighbors=1)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class="sk-container" hidden><div class="sk-item"><div class="sk-estimator sk-toggleable"><input class="sk-toggleable__control sk-hidden--visually" id="sk-estimator-id-3" type="checkbox" checked><label for="sk-estimator-id-3" class="sk-toggleable__label sk-toggleable__label-arrow">KNeighborsClassifier</label><div class="sk-toggleable__content"><pre>KNeighborsClassifier(n_neighbors=1)</pre></div></div></div></div></div>
-</div>
-<br />
-<br />
-
-## Runs: Easily explore models
-
-We can run (many) scikit-learn algorithms on (many) OpenML tasks.
-
-``` default
-# Get a task
-task = openml.tasks.get_task(403)
-
-# Build any classifier or pipeline
-clf = tree.DecisionTreeClassifier()
-
-# Run the flow
-run = openml.runs.run_model_on_task(clf, task)
-
-print(run)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-OpenML Run
-==========
-Uploader Name...................: None
-Metric..........................: None
-Local Result - Accuracy (+- STD): 0.8469 +- 0.0120
-Local Runtime - ms (+- STD).....: 143.4586 +- 6.3831
-Run ID..........................: None
-Task ID.........................: 403
-Task Type.......................: None
-Task URL........................: https://test.openml.org/t/403
-Flow ID.........................: 4595
-Flow Name.......................: sklearn.tree._classes.DecisionTreeClassifier
-Flow URL........................: https://test.openml.org/f/4595
-Setup ID........................: None
-Setup String....................: Python_3.10.12. Sklearn_1.3.0. NumPy_1.25.1. SciPy_1.11.1.
-Dataset ID......................: 68
-Dataset URL.....................: https://test.openml.org/d/68
-```
-:::
-
-Share the run on the OpenML server
-
-So far the run is only available locally. By calling the publish
-function, the run is sent to the OpenML server:
-
-``` default
-myrun = run.publish()
-# For this tutorial, our configuration publishes to the test server
-# as to not pollute the main server.
-print(f"Uploaded to {myrun.openml_url}")
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-Uploaded to https://test.openml.org/r/7358
-```
-:::
-
-We can now also inspect the flow object which was automatically created:
-
-``` default
-flow = openml.flows.get_flow(run.flow_id)
-print(flow)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-OpenML Flow
-===========
-Flow ID.........: 4595 (version 9)
-Flow URL........: https://test.openml.org/f/4595
-Flow Name.......: sklearn.tree._classes.DecisionTreeClassifier
-Flow Description: A decision tree classifier.
-Upload Date.....: 2023-07-20 19:37:57
-Dependencies....: sklearn==1.3.0
-numpy>=1.17.3
-scipy>=1.5.0
-joblib>=1.1.1
-threadpoolctl>=2.0.0
-```
-:::
-
-### It also works with pipelines
-
-When you need to handle \'dirty\' data, build pipelines to model then
-automatically. To demonstrate this using the dataset
-[credit-a](https://test.openml.org/d/16) via
-[task](https://test.openml.org/t/96) as it contains both numerical and
-categorical variables and missing values in both.
-
-``` default
-task = openml.tasks.get_task(96)
-
-# OpenML helper functions for sklearn can be plugged in directly for complicated pipelines
-from openml.extensions.sklearn import cat, cont
-
-pipe = pipeline.Pipeline(
- steps=[
- (
- "Preprocessing",
- compose.ColumnTransformer(
- [
- (
- "categorical",
- preprocessing.OneHotEncoder(sparse=False, handle_unknown="ignore"),
- cat, # returns the categorical feature indices
- ),
- (
- "continuous",
- impute.SimpleImputer(strategy="median"),
- cont,
- ), # returns the numeric feature indices
- ]
- ),
- ),
- ("Classifier", ensemble.RandomForestClassifier(n_estimators=10)),
- ]
-)
-
-run = openml.runs.run_model_on_task(pipe, task, avoid_duplicate_runs=False)
-myrun = run.publish()
-print(f"Uploaded to {myrun.openml_url}")
-
-
-# The above pipeline works with the helper functions that internally deal with pandas DataFrame.
-# In the case, pandas is not available, or a NumPy based data processing is the requirement, the
-# above pipeline is presented below to work with NumPy.
-
-# Extracting the indices of the categorical columns
-features = task.get_dataset().features
-categorical_feature_indices = []
-numeric_feature_indices = []
-for i in range(len(features)):
- if features[i].name == task.target_name:
- continue
- if features[i].data_type == "nominal":
- categorical_feature_indices.append(i)
- else:
- numeric_feature_indices.append(i)
-
-pipe = pipeline.Pipeline(
- steps=[
- (
- "Preprocessing",
- compose.ColumnTransformer(
- [
- (
- "categorical",
- preprocessing.OneHotEncoder(sparse=False, handle_unknown="ignore"),
- categorical_feature_indices,
- ),
- (
- "continuous",
- impute.SimpleImputer(strategy="median"),
- numeric_feature_indices,
- ),
- ]
- ),
- ),
- ("Classifier", ensemble.RandomForestClassifier(n_estimators=10)),
- ]
-)
-
-run = openml.runs.run_model_on_task(pipe, task, avoid_duplicate_runs=False)
-myrun = run.publish()
-print(f"Uploaded to {myrun.openml_url}")
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/openml/venv/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:972: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.
- warnings.warn(
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-Uploaded to https://test.openml.org/r/7359
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/openml/venv/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:972: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.
- warnings.warn(
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-Uploaded to https://test.openml.org/r/7360
-```
-:::
-
-## Running flows on tasks offline for later upload
-
-For those scenarios where there is no access to internet, it is possible
-to run a model on a task without uploading results or flows to the
-server immediately.
-
-``` default
-# To perform the following line offline, it is required to have been called before
-# such that the task is cached on the local openml cache directory:
-task = openml.tasks.get_task(96)
-
-# The following lines can then be executed offline:
-run = openml.runs.run_model_on_task(
- pipe,
- task,
- avoid_duplicate_runs=False,
- upload_flow=False,
-)
-
-# The run may be stored offline, and the flow will be stored along with it:
-run.to_filesystem(directory="myrun")
-
-# They may be loaded and uploaded at a later time
-run = openml.runs.OpenMLRun.from_filesystem(directory="myrun")
-run.publish()
-
-# Publishing the run will automatically upload the related flow if
-# it does not yet exist on the server.
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/openml/venv/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:972: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.
- warnings.warn(
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-
-OpenML Run
-==========
-Uploader Name...................: None
-Metric..........................: None
-Local Result - Accuracy (+- STD): 0.8678 +- 0.0000
-Local Runtime - ms (+- STD).....: 27.5688 +- 0.0000
-Run ID..........................: 7361
-Run URL.........................: https://test.openml.org/r/7361
-Task ID.........................: 96
-Task Type.......................: None
-Task URL........................: https://test.openml.org/t/96
-Flow ID.........................: 5042
-Flow Name.......................: sklearn.pipeline.Pipeline(Preprocessing=sklearn.compose._column_transformer.ColumnTransformer(categorical=sklearn.preprocessing._encoders.OneHotEncoder,continuous=sklearn.impute._base.SimpleImputer),Classifier=sklearn.ensemble._forest.RandomForestClassifier)
-Flow URL........................: https://test.openml.org/f/5042
-Setup ID........................: None
-Setup String....................: Python_3.10.12. Sklearn_1.3.0. NumPy_1.25.1. SciPy_1.11.1.
-Dataset ID......................: None
-Dataset URL.....................: https://test.openml.org/d/None
-```
-:::
-
-Alternatively, one can also directly run flows.
-
-``` default
-# Get a task
-task = openml.tasks.get_task(403)
-
-# Build any classifier or pipeline
-clf = tree.ExtraTreeClassifier()
-
-# Obtain the scikit-learn extension interface to convert the classifier
-# into a flow object.
-extension = openml.extensions.get_extension_by_model(clf)
-flow = extension.model_to_flow(clf)
-
-run = openml.runs.run_flow_on_task(flow, task)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-```
-:::
-
-## Challenge
-
-Try to build the best possible models on several OpenML tasks, compare
-your results with the rest of the class and learn from them. Some tasks
-you could try (or browse openml.org):
-
-- EEG eye state: data_id:[1471](https://www.openml.org/d/1471),
- task_id:[14951](https://www.openml.org/t/14951)
-- Volcanoes on Venus: data_id:[1527](https://www.openml.org/d/1527),
- task_id:[10103](https://www.openml.org/t/10103)
-- Walking activity: data_id:[1509](https://www.openml.org/d/1509),
- task_id:[9945](https://www.openml.org/t/9945), 150k instances.
-- Covertype (Satellite): data_id:[150](https://www.openml.org/d/150),
- task_id:[218](https://www.openml.org/t/218), 500k instances.
-- Higgs (Physics): data_id:[23512](https://www.openml.org/d/23512),
- task_id:[52950](https://www.openml.org/t/52950), 100k instances,
- missing values.
-
-``` default
-# Easy benchmarking:
-for task_id in [115]: # Add further tasks. Disclaimer: they might take some time
- task = openml.tasks.get_task(task_id)
- data = openml.datasets.get_dataset(task.dataset_id)
- clf = neighbors.KNeighborsClassifier(n_neighbors=5)
-
- run = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False)
- myrun = run.publish()
- print(f"kNN on {data.name}: {myrun.openml_url}")
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-kNN on diabetes: https://test.openml.org/r/7362
-```
-:::
-
-``` default
-openml.config.stop_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-timing
-
-**Total running time of the script:** ( 0 minutes 12.993 seconds)
-:::
-
-::::::: {#sphx_glr_download_examples_30_extended_flows_and_runs_tutorial.py}
-:::::: only
-html
-
-::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
-::: {.container .sphx-glr-download .sphx-glr-download-python}
-`Download Python source code: flows_and_runs_tutorial.py <flows_and_runs_tutorial.py>`{.interpreted-text
-role="download"}
-:::
-
-::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
-`Download Jupyter notebook: flows_and_runs_tutorial.ipynb <flows_and_runs_tutorial.ipynb>`{.interpreted-text
-role="download"}
-:::
-:::::
-::::::
-:::::::
-
-:::: only
-html
-
-::: rst-class
-sphx-glr-signature
-
-[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
-:::
-::::
diff --git a/docs/examples/30_extended/index.rst.md b/docs/examples/30_extended/index.rst.md
deleted file mode 100644
index 7311ca69e..000000000
--- a/docs/examples/30_extended/index.rst.md
+++ /dev/null
@@ -1,205 +0,0 @@
-# In-Depth Examples {#sphx_glr_examples_30_extended}
-
-Extended examples for the usage of the OpenML python connector.
-
-<div class="sphx-glr-thumbnails">
-<div class="sphx-glr-thumbcontainer" tooltip="This tutorial discusses different ways to obtain the ID of a flow in order to perform further a...">
-
-::: only
-html
-
-
-
-`sphx_glr_examples_30_extended_flow_id_tutorial.py`{.interpreted-text
-role="ref"}
-:::
-
-<div class="sphx-glr-thumbnail-title">Obtaining Flow IDs</div>
-</div>
-<div class="sphx-glr-thumbcontainer" tooltip="How to list, download and upload benchmark suites.">
-
-::: only
-html
-
-
-
-`sphx_glr_examples_30_extended_suites_tutorial.py`{.interpreted-text
-role="ref"}
-:::
-
-<div class="sphx-glr-thumbnail-title">Benchmark suites</div>
-</div>
-<div class="sphx-glr-thumbcontainer" tooltip="Explains openml-python logging, and shows how to configure it.">
-
-::: only
-html
-
-
-
-`sphx_glr_examples_30_extended_configure_logging.py`{.interpreted-text
-role="ref"}
-:::
-
-<div class="sphx-glr-thumbnail-title">Logging</div>
-</div>
-<div class="sphx-glr-thumbcontainer" tooltip="Benchmark studies">
-
-::: only
-html
-
-
-
-`sphx_glr_examples_30_extended_study_tutorial.py`{.interpreted-text
-role="ref"}
-:::
-
-<div class="sphx-glr-thumbnail-title">Benchmark studies</div>
-</div>
-<div class="sphx-glr-thumbcontainer" tooltip="Plotting hyperparameter surfaces">
-
-::: only
-html
-
-
-
-`sphx_glr_examples_30_extended_plot_svm_hyperparameters_tutorial.py`{.interpreted-text
-role="ref"}
-:::
-
-<div class="sphx-glr-thumbnail-title">Plotting hyperparameter surfaces</div>
-</div>
-<div class="sphx-glr-thumbcontainer" tooltip="A tutorial on how to list and download tasks.">
-
-::: only
-html
-
-
-
-`sphx_glr_examples_30_extended_tasks_tutorial.py`{.interpreted-text
-role="ref"}
-:::
-
-<div class="sphx-glr-thumbnail-title">Tasks</div>
-</div>
-<div class="sphx-glr-thumbcontainer" tooltip="How to list and download datasets.">
-
-::: only
-html
-
-
-
-`sphx_glr_examples_30_extended_datasets_tutorial.py`{.interpreted-text
-role="ref"}
-:::
-
-<div class="sphx-glr-thumbnail-title">Datasets</div>
-</div>
-<div class="sphx-glr-thumbcontainer" tooltip="By: Jan N. van Rijn">
-
-::: only
-html
-
-
-
-`sphx_glr_examples_30_extended_run_setup_tutorial.py`{.interpreted-text
-role="ref"}
-:::
-
-<div class="sphx-glr-thumbnail-title">Run Setup</div>
-</div>
-<div class="sphx-glr-thumbcontainer" tooltip="The most convenient way to create a flow for your machine learning workflow is to generate it a...">
-
-::: only
-html
-
-
-
-`sphx_glr_examples_30_extended_custom_flow_.py`{.interpreted-text
-role="ref"}
-:::
-
-<div class="sphx-glr-thumbnail-title">Creating and Using a Custom Flow</div>
-</div>
-<div class="sphx-glr-thumbcontainer" tooltip="Evaluations contain a concise summary of the results of all runs made. Each evaluation provides...">
-
-::: only
-html
-
-
-
-`sphx_glr_examples_30_extended_fetch_evaluations_tutorial.py`{.interpreted-text
-role="ref"}
-:::
-
-<div class="sphx-glr-thumbnail-title">Fetching Evaluations</div>
-</div>
-<div class="sphx-glr-thumbcontainer" tooltip="Tasks define a target and a train/test split. Normally, they are the input to the function open...">
-
-::: only
-html
-
-
-
-`sphx_glr_examples_30_extended_task_manual_iteration_tutorial.py`{.interpreted-text
-role="ref"}
-:::
-
-<div class="sphx-glr-thumbnail-title">Tasks: retrieving splits</div>
-</div>
-<div class="sphx-glr-thumbcontainer" tooltip="How to train/run a model and how to upload the results.">
-
-::: only
-html
-
-
-
-`sphx_glr_examples_30_extended_flows_and_runs_tutorial.py`{.interpreted-text
-role="ref"}
-:::
-
-<div class="sphx-glr-thumbnail-title">Flows and Runs</div>
-</div>
-<div class="sphx-glr-thumbcontainer" tooltip="A tutorial on how to create and upload a dataset to OpenML.">
-
-::: only
-html
-
-
-
-`sphx_glr_examples_30_extended_create_upload_tutorial.py`{.interpreted-text
-role="ref"}
-:::
-
-<div class="sphx-glr-thumbnail-title">Dataset upload tutorial</div>
-</div>
-<div class="sphx-glr-thumbcontainer" tooltip="The runtime of machine learning models on specific datasets can be a deciding factor on the cho...">
-
-::: only
-html
-
-
-
-`sphx_glr_examples_30_extended_fetch_runtimes_tutorial.py`{.interpreted-text
-role="ref"}
-:::
-
-<div class="sphx-glr-thumbnail-title">Measuring runtimes for Scikit-learn models</div>
-</div>
-</div>
-
-::: {.toctree hidden=""}
-/examples/30_extended/flow_id_tutorial
-/examples/30_extended/suites_tutorial
-/examples/30_extended/configure_logging
-/examples/30_extended/study_tutorial
-/examples/30_extended/plot_svm_hyperparameters_tutorial
-/examples/30_extended/tasks_tutorial
-/examples/30_extended/datasets_tutorial
-/examples/30_extended/run_setup_tutorial
-/examples/30_extended/[custom_flow]()
-/examples/30_extended/fetch_evaluations_tutorial
-/examples/30_extended/task_manual_iteration_tutorial
-/examples/30_extended/flows_and_runs_tutorial
-/examples/30_extended/create_upload_tutorial
-/examples/30_extended/fetch_runtimes_tutorial
-:::
diff --git a/docs/examples/30_extended/plot_svm_hyperparameters_tutorial.rst.md b/docs/examples/30_extended/plot_svm_hyperparameters_tutorial.rst.md
deleted file mode 100644
index f272bbe33..000000000
--- a/docs/examples/30_extended/plot_svm_hyperparameters_tutorial.rst.md
+++ /dev/null
@@ -1,235 +0,0 @@
-::::: only
-html
-
-:::: {.note .sphx-glr-download-link-note}
-::: title
-Note
-:::
-
-`Go to the end <sphx_glr_download_examples_30_extended_plot_svm_hyperparameters_tutorial.py>`{.interpreted-text
-role="ref"} to download the full example code
-::::
-:::::
-
-::: rst-class
-sphx-glr-example-title
-:::
-
-# Plotting hyperparameter surfaces {#sphx_glr_examples_30_extended_plot_svm_hyperparameters_tutorial.py}
-
-``` default
-# License: BSD 3-Clause
-
-import openml
-import numpy as np
-```
-
-## First step - obtaining the data
-
-First, we need to choose an SVM flow, for example 8353, and a task.
-Finding the IDs of them are not part of this tutorial, this could for
-example be done via the website.
-
-For this we use the function `list_evaluations_setup` which can
-automatically join evaluations conducted by the server with the
-hyperparameter settings extracted from the uploaded runs (called
-*setup*).
-
-``` default
-df = openml.evaluations.list_evaluations_setups(
- function="predictive_accuracy",
- flows=[8353],
- tasks=[6],
- output_format="dataframe",
- # Using this flag incorporates the hyperparameters into the returned dataframe. Otherwise,
- # the dataframe would contain a field ``paramaters`` containing an unparsed dictionary.
- parameters_in_separate_columns=True,
-)
-print(df.head(n=10))
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-run_id ... sklearn.pipeline.Pipeline(imputation=hyperimp.utils.preprocessing.ConditionalImputer2,hotencoding=sklearn.preprocessing.data.OneHotEncoder,scaling=sklearn.preprocessing.data.StandardScaler,variencethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,clf=sklearn.svm.classes.SVC)(1)_memory
-0 9154533 ... null
-1 9154536 ... null
-2 9154539 ... null
-3 9154544 ... null
-4 9154546 ... null
-5 9154558 ... null
-6 9154563 ... null
-7 9154569 ... null
-8 9154572 ... null
-9 9154576 ... null
-
-[10 rows x 46 columns]
-```
-:::
-
-We can see all the hyperparameter names in the columns of the dataframe:
-
-``` default
-for name in df.columns:
- print(name)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-run_id
-task_id
-setup_id
-flow_id
-flow_name
-data_id
-data_name
-function
-upload_time
-uploader
-uploader_name
-value
-values
-array_data
-sklearn.preprocessing.data.OneHotEncoder(17)_categorical_features
-sklearn.preprocessing.data.OneHotEncoder(17)_dtype
-sklearn.preprocessing.data.OneHotEncoder(17)_handle_unknown
-sklearn.preprocessing.data.OneHotEncoder(17)_n_values
-sklearn.preprocessing.data.OneHotEncoder(17)_sparse
-sklearn.feature_selection.variance_threshold.VarianceThreshold(11)_threshold
-sklearn.preprocessing.data.StandardScaler(5)_copy
-sklearn.preprocessing.data.StandardScaler(5)_with_mean
-sklearn.preprocessing.data.StandardScaler(5)_with_std
-sklearn.svm.classes.SVC(16)_C
-sklearn.svm.classes.SVC(16)_cache_size
-sklearn.svm.classes.SVC(16)_class_weight
-sklearn.svm.classes.SVC(16)_coef0
-sklearn.svm.classes.SVC(16)_decision_function_shape
-sklearn.svm.classes.SVC(16)_degree
-sklearn.svm.classes.SVC(16)_gamma
-sklearn.svm.classes.SVC(16)_kernel
-sklearn.svm.classes.SVC(16)_max_iter
-sklearn.svm.classes.SVC(16)_probability
-sklearn.svm.classes.SVC(16)_random_state
-sklearn.svm.classes.SVC(16)_shrinking
-sklearn.svm.classes.SVC(16)_tol
-sklearn.svm.classes.SVC(16)_verbose
-hyperimp.utils.preprocessing.ConditionalImputer2(1)_axis
-hyperimp.utils.preprocessing.ConditionalImputer2(1)_categorical_features
-hyperimp.utils.preprocessing.ConditionalImputer2(1)_copy
-hyperimp.utils.preprocessing.ConditionalImputer2(1)_fill_empty
-hyperimp.utils.preprocessing.ConditionalImputer2(1)_missing_values
-hyperimp.utils.preprocessing.ConditionalImputer2(1)_strategy
-hyperimp.utils.preprocessing.ConditionalImputer2(1)_strategy_nominal
-hyperimp.utils.preprocessing.ConditionalImputer2(1)_verbose
-sklearn.pipeline.Pipeline(imputation=hyperimp.utils.preprocessing.ConditionalImputer2,hotencoding=sklearn.preprocessing.data.OneHotEncoder,scaling=sklearn.preprocessing.data.StandardScaler,variencethreshold=sklearn.feature_selection.variance_threshold.VarianceThreshold,clf=sklearn.svm.classes.SVC)(1)_memory
-```
-:::
-
-Next, we cast and transform the hyperparameters of interest (`C` and
-`gamma`) so that we can nicely plot them.
-
-``` default
-hyperparameters = ["sklearn.svm.classes.SVC(16)_C", "sklearn.svm.classes.SVC(16)_gamma"]
-df[hyperparameters] = df[hyperparameters].astype(float).apply(np.log10)
-```
-
-## Option 1 - plotting via the pandas helper functions
-
-``` default
-df.plot.hexbin(
- x="sklearn.svm.classes.SVC(16)_C",
- y="sklearn.svm.classes.SVC(16)_gamma",
- C="value",
- reduce_C_function=np.mean,
- gridsize=25,
- title="SVM performance landscape",
-)
-```
-
-::: {.image-sg .sphx-glr-single-img alt="SVM performance landscape" srcset="/examples/30_extended/images/sphx_glr_plot_svm_hyperparameters_tutorial_001.png"}
-/examples/30_extended/images/sphx_glr_plot_svm_hyperparameters_tutorial_001.png
-:::
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-<Axes: title={'center': 'SVM performance landscape'}, xlabel='sklearn.svm.classes.SVC(16)_C', ylabel='sklearn.svm.classes.SVC(16)_gamma'>
-```
-:::
-
-## Option 2 - plotting via matplotlib
-
-``` default
-import matplotlib.pyplot as plt
-
-fig, ax = plt.subplots()
-
-C = df["sklearn.svm.classes.SVC(16)_C"]
-gamma = df["sklearn.svm.classes.SVC(16)_gamma"]
-score = df["value"]
-
-# Plotting all evaluations:
-ax.plot(C, gamma, "ko", ms=1)
-# Create a contour plot
-cntr = ax.tricontourf(C, gamma, score, levels=12, cmap="RdBu_r")
-# Adjusting the colorbar
-fig.colorbar(cntr, ax=ax, label="accuracy")
-# Adjusting the axis limits
-ax.set(
- xlim=(min(C), max(C)),
- ylim=(min(gamma), max(gamma)),
- xlabel="C (log10)",
- ylabel="gamma (log10)",
-)
-ax.set_title("SVM performance landscape")
-```
-
-::: {.image-sg .sphx-glr-single-img alt="SVM performance landscape" srcset="/examples/30_extended/images/sphx_glr_plot_svm_hyperparameters_tutorial_002.png"}
-/examples/30_extended/images/sphx_glr_plot_svm_hyperparameters_tutorial_002.png
-:::
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-Text(0.5, 1.0, 'SVM performance landscape')
-```
-:::
-
-::: rst-class
-sphx-glr-timing
-
-**Total running time of the script:** ( 0 minutes 6.466 seconds)
-:::
-
-::::::: {#sphx_glr_download_examples_30_extended_plot_svm_hyperparameters_tutorial.py}
-:::::: only
-html
-
-::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
-::: {.container .sphx-glr-download .sphx-glr-download-python}
-`Download Python source code: plot_svm_hyperparameters_tutorial.py <plot_svm_hyperparameters_tutorial.py>`{.interpreted-text
-role="download"}
-:::
-
-::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
-`Download Jupyter notebook: plot_svm_hyperparameters_tutorial.ipynb <plot_svm_hyperparameters_tutorial.ipynb>`{.interpreted-text
-role="download"}
-:::
-:::::
-::::::
-:::::::
-
-:::: only
-html
-
-::: rst-class
-sphx-glr-signature
-
-[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
-:::
-::::
diff --git a/docs/examples/30_extended/run_setup_tutorial.rst.md b/docs/examples/30_extended/run_setup_tutorial.rst.md
deleted file mode 100644
index 656ce560f..000000000
--- a/docs/examples/30_extended/run_setup_tutorial.rst.md
+++ /dev/null
@@ -1,215 +0,0 @@
-::::: only
-html
-
-:::: {.note .sphx-glr-download-link-note}
-::: title
-Note
-:::
-
-`Go to the end <sphx_glr_download_examples_30_extended_run_setup_tutorial.py>`{.interpreted-text
-role="ref"} to download the full example code
-::::
-:::::
-
-::: rst-class
-sphx-glr-example-title
-:::
-
-# Run Setup {#sphx_glr_examples_30_extended_run_setup_tutorial.py}
-
-By: Jan N. van Rijn
-
-One of the key features of the openml-python library is that is allows
-to reinstantiate flows with hyperparameter settings that were uploaded
-before. This tutorial uses the concept of setups. Although setups are
-not extensively described in the OpenML documentation (because most
-users will not directly use them), they form a important concept within
-OpenML distinguishing between hyperparameter configurations. A setup is
-the combination of a flow with all its hyperparameters set.
-
-A key requirement for reinstantiating a flow is to have the same
-scikit-learn version as the flow that was uploaded. However, this
-tutorial will upload the flow (that will later be reinstantiated)
-itself, so it can be ran with any scikit-learn version that is supported
-by this library. In this case, the requirement of the corresponding
-scikit-learn versions is automatically met.
-
-In this tutorial we will
-
-: 1) Create a flow and use it to solve a task;
- 2) Download the flow, reinstantiate the model with same
- hyperparameters, and solve the same task again;
- 3) We will verify that the obtained results are exactly the same.
-
-``` default
-# License: BSD 3-Clause
-
-import numpy as np
-import openml
-from openml.extensions.sklearn import cat, cont
-
-from sklearn.pipeline import make_pipeline, Pipeline
-from sklearn.compose import ColumnTransformer
-from sklearn.impute import SimpleImputer
-from sklearn.preprocessing import OneHotEncoder, FunctionTransformer
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.decomposition import TruncatedSVD
-```
-
-:::: warning
-::: title
-Warning
-:::
-
-.. include:: ../../test_server_usage_warning.txt
-::::
-
-``` default
-openml.config.start_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!
- warnings.warn(
-```
-:::
-
-## 1) Create a flow and use it to solve a task
-
-``` default
-# first, let's download the task that we are interested in
-task = openml.tasks.get_task(6)
-
-
-# we will create a fairly complex model, with many preprocessing components and
-# many potential hyperparameters. Of course, the model can be as complex and as
-# easy as you want it to be
-
-
-cat_imp = make_pipeline(
- OneHotEncoder(handle_unknown="ignore", sparse=False),
- TruncatedSVD(),
-)
-cont_imp = SimpleImputer(strategy="median")
-ct = ColumnTransformer([("cat", cat_imp, cat), ("cont", cont_imp, cont)])
-model_original = Pipeline(
- steps=[
- ("transform", ct),
- ("estimator", RandomForestClassifier()),
- ]
-)
-
-# Let's change some hyperparameters. Of course, in any good application we
-# would tune them using, e.g., Random Search or Bayesian Optimization, but for
-# the purpose of this tutorial we set them to some specific values that might
-# or might not be optimal
-hyperparameters_original = {
- "estimator__criterion": "gini",
- "estimator__n_estimators": 50,
- "estimator__max_depth": 10,
- "estimator__min_samples_leaf": 1,
-}
-model_original.set_params(**hyperparameters_original)
-
-# solve the task and upload the result (this implicitly creates the flow)
-run = openml.runs.run_model_on_task(model_original, task, avoid_duplicate_runs=False)
-run_original = run.publish() # this implicitly uploads the flow
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/openml/venv/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:972: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.
- warnings.warn(
-/code/openml/extensions/sklearn/extension.py:1785: UserWarning: Estimator only predicted for 5/6 classes!
- warnings.warn(message)
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-```
-:::
-
-## 2) Download the flow and solve the same task again.
-
-``` default
-# obtain setup id (note that the setup id is assigned by the OpenML server -
-# therefore it was not yet available in our local copy of the run)
-run_downloaded = openml.runs.get_run(run_original.run_id)
-setup_id = run_downloaded.setup_id
-
-# after this, we can easily reinstantiate the model
-model_duplicate = openml.setups.initialize_model(setup_id)
-# it will automatically have all the hyperparameters set
-
-# and run the task again
-run_duplicate = openml.runs.run_model_on_task(model_duplicate, task, avoid_duplicate_runs=False)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/openml/venv/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:972: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.
- warnings.warn(
-/code/openml/extensions/sklearn/extension.py:1785: UserWarning: Estimator only predicted for 5/6 classes!
- warnings.warn(message)
-```
-:::
-
-## 3) We will verify that the obtained results are exactly the same.
-
-``` default
-# the run has stored all predictions in the field data content
-np.testing.assert_array_equal(run_original.data_content, run_duplicate.data_content)
-```
-
-``` default
-openml.config.stop_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-timing
-
-**Total running time of the script:** ( 0 minutes 6.595 seconds)
-:::
-
-::::::: {#sphx_glr_download_examples_30_extended_run_setup_tutorial.py}
-:::::: only
-html
-
-::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
-::: {.container .sphx-glr-download .sphx-glr-download-python}
-`Download Python source code: run_setup_tutorial.py <run_setup_tutorial.py>`{.interpreted-text
-role="download"}
-:::
-
-::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
-`Download Jupyter notebook: run_setup_tutorial.ipynb <run_setup_tutorial.ipynb>`{.interpreted-text
-role="download"}
-:::
-:::::
-::::::
-:::::::
-
-:::: only
-html
-
-::: rst-class
-sphx-glr-signature
-
-[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
-:::
-::::
diff --git a/docs/examples/30_extended/sg_execution_times.rst.md b/docs/examples/30_extended/sg_execution_times.rst.md
deleted file mode 100644
index f9800d22b..000000000
--- a/docs/examples/30_extended/sg_execution_times.rst.md
+++ /dev/null
@@ -1,51 +0,0 @@
-orphan
-
-:
-
-# Computation times {#sphx_glr_examples_30_extended_sg_execution_times}
-
-**03:46.366** total execution time for **examples_30_extended** files:
-
- ---------------------------------------------------------------------------------------- ----------- -----
- `sphx_glr_examples_30_extended_fetch_runtimes_tutorial.py`{.interpreted-text role="ref"} 01:13.261 0.0
- (`fetch_runtimes_tutorial.py`) MB
-
- `sphx_glr_examples_30_extended_tasks_tutorial.py`{.interpreted-text role="ref"} 00:47.382 0.0
- (`tasks_tutorial.py`) MB
-
- `sphx_glr_examples_30_extended_suites_tutorial.py`{.interpreted-text role="ref"} 00:30.398 0.0
- (`suites_tutorial.py`) MB
-
- `sphx_glr_examples_30_extended_study_tutorial.py`{.interpreted-text role="ref"} 00:16.432 0.0
- (`study_tutorial.py`) MB
-
- `sphx_glr_examples_30_extended_fetch_evaluations_tutorial.py`{.interpreted-text 00:13.992 0.0
- role="ref"} (`fetch_evaluations_tutorial.py`) MB
-
- `sphx_glr_examples_30_extended_flows_and_runs_tutorial.py`{.interpreted-text role="ref"} 00:12.993 0.0
- (`flows_and_runs_tutorial.py`) MB
-
- `sphx_glr_examples_30_extended_datasets_tutorial.py`{.interpreted-text role="ref"} 00:08.138 0.0
- (`datasets_tutorial.py`) MB
-
- `sphx_glr_examples_30_extended_run_setup_tutorial.py`{.interpreted-text role="ref"} 00:06.595 0.0
- (`run_setup_tutorial.py`) MB
-
- `sphx_glr_examples_30_extended_plot_svm_hyperparameters_tutorial.py`{.interpreted-text 00:06.466 0.0
- role="ref"} (`plot_svm_hyperparameters_tutorial.py`) MB
-
- `sphx_glr_examples_30_extended_flow_id_tutorial.py`{.interpreted-text role="ref"} 00:04.274 0.0
- (`flow_id_tutorial.py`) MB
-
- `sphx_glr_examples_30_extended_task_manual_iteration_tutorial.py`{.interpreted-text 00:03.602 0.0
- role="ref"} (`task_manual_iteration_tutorial.py`) MB
-
- `sphx_glr_examples_30_extended_create_upload_tutorial.py`{.interpreted-text role="ref"} 00:02.833 0.0
- (`create_upload_tutorial.py`) MB
-
- `sphx_glr_examples_30_extended_configure_logging.py`{.interpreted-text role="ref"} 00:00.000 0.0
- (`configure_logging.py`) MB
-
- `sphx_glr_examples_30_extended_custom_flow_.py`{.interpreted-text role="ref"} 00:00.000 0.0
- (`custom_flow_.py`) MB
- ---------------------------------------------------------------------------------------- ----------- -----
diff --git a/docs/examples/30_extended/study_tutorial.rst.md b/docs/examples/30_extended/study_tutorial.rst.md
deleted file mode 100644
index fcf4f1569..000000000
--- a/docs/examples/30_extended/study_tutorial.rst.md
+++ /dev/null
@@ -1,402 +0,0 @@
-::::: only
-html
-
-:::: {.note .sphx-glr-download-link-note}
-::: title
-Note
-:::
-
-`Go to the end <sphx_glr_download_examples_30_extended_study_tutorial.py>`{.interpreted-text
-role="ref"} to download the full example code
-::::
-:::::
-
-::: rst-class
-sphx-glr-example-title
-:::
-
-# Benchmark studies {#sphx_glr_examples_30_extended_study_tutorial.py}
-
-How to list, download and upload benchmark studies. In contrast to
-[benchmark
-suites](https://docs.openml.org/benchmark/#benchmarking-suites) which
-hold a list of tasks, studies hold a list of runs. As runs contain all
-information on flows and tasks, all required information about a study
-can be retrieved.
-
-``` default
-# License: BSD 3-Clause
-
-import uuid
-
-from sklearn.ensemble import RandomForestClassifier
-
-import openml
-```
-
-## Listing studies
-
-- Use the output_format parameter to select output type
-- Default gives `dict`, but we\'ll use `dataframe` to obtain an
- easier-to-work-with data structure
-
-``` default
-studies = openml.study.list_studies(output_format="dataframe", status="all")
-print(studies.head(n=10))
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-id alias main_entity_type ... status creation_date creator
-1 1 Study_1 run ... in_preparation 2015-10-20 15:27:26 2
-2 2 Study_2 run ... in_preparation 2015-10-20 15:28:44 2
-3 3 Study_3 run ... in_preparation 2015-10-20 15:34:39 2
-5 5 Study_5 run ... in_preparation 2015-11-19 11:20:33 749
-7 7 Study_7 run ... in_preparation 2016-01-06 17:49:36 64
-8 8 Study_8 run ... in_preparation 2016-03-13 13:38:31 1135
-10 10 Study_10 run ... in_preparation 2016-03-16 12:16:08 507
-11 11 Study_11 run ... in_preparation 2016-03-22 16:48:06 1
-12 12 Study_12 run ... in_preparation 2016-03-31 15:05:45 1195
-13 13 Study_13 run ... in_preparation 2016-04-05 13:57:41 62
-
-[10 rows x 7 columns]
-```
-:::
-
-### Downloading studies
-
-This is done based on the study ID.
-
-``` default
-study = openml.study.get_study(123)
-print(study)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-OpenML Study
-============
-ID..............: 123
-Name............: Linear vs. Non Linear
-Status..........: active
-Main Entity Type: run
-Study URL.......: https://www.openml.org/s/123
-# of Data.......: 299
-# of Tasks......: 299
-# of Flows......: 5
-# of Runs.......: 1693
-Creator.........: https://www.openml.org/u/1
-Upload Time.....: 2019-02-21 19:55:30
-```
-:::
-
-Studies also features a description:
-
-``` default
-print(study.description)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-Comparison of linear and non-linear models.
-
-[Jupyter Notebook](https://github.com/janvanrijn/linear-vs-non-linear/blob/master/notebook/Linear-vs-Non-Linear.ipynb)
-```
-:::
-
-Studies are a container for runs:
-
-``` default
-print(study.runs)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-[9199877, 9199878, 9199879, 9199880, 9199881, 9199882, 9199883, 9199884, 9199885, 9199886, 9199887, 9199888, 9199889, 9199890, 9199891, 9199892, 9199893, 9199894, 9199895, 9199897, 9199898, 9199899, 9199900, 9199901, 9199902, 9199903, 9199904, 9199905, 9199906, 9199907, 9199908, 9199909, 9199910, 9199911, 9199912, 9199913, 9199914, 9199915, 9199916, 9199917, 9199918, 9199919, 9199920, 9199921, 9199922, 9199923, 9199924, 9199925, 9199926, 9199927, 9199928, 9199929, 9199930, 9199931, 9199932, 9199933, 9199934, 9199935, 9199936, 9199937, 9199938, 9199939, 9199940, 9199941, 9199942, 9199943, 9199944, 9199945, 9199946, 9199947, 9199948, 9199950, 9199951, 9199952, 9199953, 9199954, 9199955, 9199956, 9199957, 9199958, 9199959, 9199960, 9199961, 9199963, 9199964, 9199965, 9199966, 9199967, 9199968, 9199969, 9199970, 9199971, 9199972, 9199973, 9199974, 9199975, 9199976, 9199977, 9199978, 9199979, 9199981, 9199982, 9199983, 9199984, 9199985, 9199986, 9199987, 9199988, 9199989, 9199990, 9199991, 9199992, 9199993, 9199994, 9199995, 9199996, 9199997, 9199998, 9199999, 9200000, 9200001, 9200002, 9200003, 9200004, 9200006, 9200007, 9200008, 9200009, 9200010, 9200011, 9200012, 9200013, 9200014, 9200015, 9200016, 9200017, 9200018, 9200019, 9200020, 9200021, 9200022, 9200023, 9200024, 9200025, 9200026, 9200027, 9200028, 9200029, 9200030, 9200031, 9200032, 9200033, 9200034, 9200035, 9200036, 9200037, 9200038, 9200039, 9200040, 9200041, 9200042, 9200043, 9200044, 9200045, 9200046, 9200047, 9200048, 9200049, 9200050, 9200051, 9200052, 9200053, 9200054, 9200055, 9200056, 9200057, 9200058, 9200059, 9200060, 9200061, 9200062, 9200063, 9200064, 9200065, 9200066, 9200067, 9200068, 9200069, 9200070, 9200071, 9200072, 9200073, 9200074, 9200075, 9200076, 9200077, 9200078, 9200079, 9200080, 9200081, 9200082, 9200083, 9200084, 9200085, 9200086, 9200087, 9200088, 9200089, 9200090, 9200091, 9200092, 9200093, 9200094, 9200095, 9200096, 9200097, 9200098, 9200099, 9200100, 9200101, 9200102, 9200103, 9200104, 9200105, 9200106, 9200107, 9200108, 9200109, 9200110, 9200111, 9200112, 9200113, 9200114, 9200115, 9200116, 9200117, 9200118, 9200119, 9200120, 9200121, 9200122, 9200123, 9200124, 9200125, 9200126, 9200127, 9200128, 9200129, 9200130, 9200131, 9200132, 9200133, 9200134, 9200135, 9200136, 9200137, 9200138, 9200139, 9200140, 9200141, 9200142, 9200143, 9200144, 9200145, 9200146, 9200147, 9200148, 9200149, 9200150, 9200151, 9200152, 9200153, 9200154, 9200155, 9200156, 9200157, 9200158, 9200159, 9200160, 9200161, 9200162, 9200163, 9200164, 9200165, 9200166, 9200167, 9200168, 9200169, 9200171, 9200173, 9200174, 9200175, 9200176, 9200177, 9200178, 9200180, 9200181, 9200182, 9200183, 9200184, 9200185, 9200186, 9200187, 9200188, 9200189, 9200190, 9200191, 9200192, 9200193, 9200194, 9200195, 9200196, 9200197, 9200198, 9200199, 9200200, 9200201, 9200202, 9200203, 9200204, 9200205, 9200206, 9200207, 9200208, 9200209, 9200210, 9200211, 9200212, 9200213, 9200214, 9200215, 9200216, 9200217, 9200218, 9200219, 9200220, 9200221, 9200222, 9200223, 9200224, 9200225, 9200226, 9200227, 9200228, 9200229, 9200230, 9200231, 9200232, 9200233, 9200234, 9200235, 9200236, 9200237, 9200238, 9200239, 9200240, 9200241, 9200242, 9200243, 9200244, 9200245, 9200246, 9200247, 9200248, 9200249, 9200250, 9200251, 9200252, 9200253, 9200254, 9200255, 9200256, 9200257, 9200258, 9200259, 9200260, 9200261, 9200262, 9200263, 9200264, 9200265, 9200266, 9200267, 9200268, 9200269, 9200270, 9200271, 9200272, 9200273, 9200274, 9200275, 9200276, 9200277, 9200278, 9200279, 9200280, 9200281, 9200282, 9200283, 9200284, 9200285, 9200286, 9200287, 9200288, 9200289, 9200290, 9200291, 9200292, 9200293, 9200294, 9200295, 9200296, 9200297, 9200298, 9200299, 9200300, 9200301, 9200302, 9200303, 9200304, 9200305, 9200306, 9200307, 9200308, 9200310, 9200311, 9200312, 9200313, 9200314, 9200315, 9200316, 9200317, 9200318, 9200319, 9200320, 9200321, 9200322, 9200324, 9200325, 9200326, 9200327, 9200328, 9200329, 9200330, 9200331, 9200332, 9200333, 9200334, 9200335, 9200336, 9200337, 9200338, 9200339, 9200340, 9200341, 9200342, 9200343, 9200344, 9200345, 9200346, 9200347, 9200348, 9200349, 9200350, 9200351, 9200352, 9200353, 9200354, 9200355, 9200356, 9200357, 9200358, 9200359, 9200361, 9200362, 9200364, 9200365, 9200366, 9200367, 9200368, 9200369, 9200370, 9200371, 9200372, 9200373, 9200374, 9200375, 9200376, 9200377, 9200378, 9200379, 9200380, 9200382, 9200383, 9200384, 9200385, 9200386, 9200387, 9200388, 9200389, 9200390, 9200391, 9200392, 9200393, 9200394, 9200395, 9200396, 9200397, 9200398, 9200399, 9200400, 9200401, 9200402, 9200403, 9200404, 9200405, 9200406, 9200407, 9200408, 9200409, 9200410, 9200411, 9200412, 9200413, 9200414, 9200415, 9200416, 9200417, 9200418, 9200419, 9200420, 9200421, 9200422, 9200424, 9200425, 9200426, 9200427, 9200428, 9200429, 9200430, 9200431, 9200432, 9200433, 9200434, 9200435, 9200436, 9200437, 9200438, 9200439, 9200440, 9200441, 9200442, 9200443, 9200444, 9200445, 9200446, 9200447, 9200448, 9200449, 9200450, 9200451, 9200452, 9200453, 9200454, 9200455, 9200456, 9200457, 9200458, 9200459, 9200460, 9200461, 9200462, 9200463, 9200464, 9200465, 9200466, 9200467, 9200468, 9200469, 9200470, 9200471, 9200472, 9200473, 9200474, 9200475, 9200476, 9200477, 9200478, 9200479, 9200480, 9200481, 9200482, 9200483, 9200484, 9200485, 9200486, 9200487, 9200488, 9200489, 9200490, 9200491, 9200492, 9200493, 9200494, 9200495, 9200496, 9200497, 9200498, 9200499, 9200500, 9200501, 9200502, 9200503, 9200504, 9200505, 9200506, 9200507, 9200508, 9200509, 9200510, 9200511, 9200512, 9200513, 9200514, 9200515, 9200516, 9200517, 9200518, 9200519, 9200520, 9200521, 9200522, 9200523, 9200524, 9200525, 9200526, 9200527, 9200528, 9200529, 9200530, 9200531, 9200532, 9200533, 9200534, 9200535, 9200536, 9200537, 9200538, 9200539, 9200540, 9200541, 9200542, 9200543, 9200544, 9200545, 9200546, 9200547, 9200548, 9200549, 9200550, 9200551, 9200552, 9200553, 9200554, 9200555, 9200556, 9200557, 9200558, 9200559, 9200560, 9200561, 9200562, 9200563, 9200564, 9200565, 9200566, 9200567, 9200568, 9200569, 9200570, 9200571, 9200572, 9200573, 9200574, 9200575, 9200576, 9200577, 9200578, 9200579, 9200580, 9200581, 9200582, 9200583, 9200584, 9200585, 9200586, 9200587, 9200588, 9200589, 9200590, 9200591, 9200592, 9200593, 9200594, 9200595, 9200596, 9200597, 9200598, 9200599, 9200600, 9200601, 9200602, 9200603, 9200604, 9200605, 9200606, 9200607, 9200608, 9200609, 9200610, 9200611, 9200612, 9200613, 9200614, 9200615, 9200616, 9200617, 9200618, 9200619, 9200620, 9200621, 9200622, 9200623, 9200624, 9200625, 9200626, 9200627, 9200628, 9200629, 9200630, 9200631, 9200632, 9200633, 9200634, 9200635, 9200636, 9200637, 9200638, 9200639, 9200640, 9200641, 9200642, 9200643, 9200644, 9200645, 9200646, 9200647, 9200648, 9200649, 9200650, 9200651, 9200652, 9200653, 9200654, 9200655, 9200656, 9200657, 9200658, 9200659, 9200660, 9200661, 9200662, 9200663, 9200664, 9200665, 9200666, 9200667, 9200668, 9200669, 9200670, 9200671, 9200672, 9200673, 9200674, 9200675, 9200676, 9200677, 9200678, 9200679, 9200680, 9200681, 9200682, 9200683, 9200684, 9200685, 9200686, 9200687, 9200688, 9200689, 9200690, 9200691, 9200692, 9200693, 9200694, 9200695, 9200696, 9200697, 9200698, 9200699, 9200700, 9200701, 9200702, 9200703, 9200704, 9200705, 9200706, 9200707, 9200708, 9200709, 9200710, 9200711, 9200712, 9200713, 9200714, 9200715, 9200716, 9200717, 9200719, 9200720, 9200721, 9200722, 9200723, 9200724, 9200725, 9200726, 9200727, 9200728, 9200729, 9200730, 9200731, 9200732, 9200733, 9200734, 9200735, 9200736, 9200737, 9200738, 9200739, 9200740, 9200741, 9200742, 9200743, 9200744, 9200745, 9200746, 9200747, 9200748, 9200749, 9200750, 9200751, 9200752, 9200753, 9200754, 9200755, 9200756, 9200757, 9200758, 9200759, 9200760, 9200761, 9200762, 9200763, 9200764, 9200765, 9200766, 9200767, 9200768, 9200769, 9200770, 9200771, 9200772, 9200773, 9200774, 9200775, 9200776, 9200778, 9200779, 9200780, 9200781, 9200782, 9200783, 9200784, 9200785, 9200786, 9200787, 9200788, 9200789, 9200790, 9200791, 9200792, 9200793, 9200794, 9200795, 9200796, 9200797, 9200798, 9200799, 9200800, 9200801, 9200802, 9200803, 9200804, 9200806, 9200807, 9200808, 9200809, 9200810, 9200811, 9200812, 9200813, 9200814, 9200815, 9200816, 9200817, 9200818, 9200819, 9200820, 9200821, 9200822, 9200823, 9200824, 9200825, 9200826, 9200827, 9200828, 9200829, 9200830, 9200831, 9200832, 9200833, 9200834, 9200835, 9200836, 9200837, 9200838, 9200839, 9200840, 9200841, 9200842, 9200843, 9200844, 9200845, 9200846, 9200847, 9200848, 9200849, 9200850, 9200851, 9200852, 9200853, 9200854, 9200855, 9200856, 9200857, 9200858, 9200859, 9200860, 9200861, 9200862, 9200863, 9200864, 9200865, 9200866, 9200867, 9200868, 9200869, 9200870, 9200871, 9200872, 9200873, 9200874, 9200875, 9200876, 9200877, 9200878, 9200879, 9200880, 9200881, 9200882, 9200883, 9200884, 9200885, 9200886, 9200887, 9200888, 9200889, 9200890, 9200891, 9200892, 9200893, 9200894, 9200895, 9200896, 9200897, 9200898, 9200899, 9200900, 9200901, 9200902, 9200903, 9200904, 9200905, 9200906, 9200907, 9200908, 9200909, 9200910, 9200911, 9200912, 9200913, 9200914, 9200915, 9200916, 9200917, 9200918, 9200919, 9200920, 9200921, 9200922, 9200923, 9200924, 9200925, 9200926, 9200927, 9200928, 9200929, 9200930, 9200931, 9200932, 9200933, 9200934, 9200935, 9200936, 9200937, 9200938, 9200939, 9200940, 9200941, 9200942, 9200943, 9200944, 9200945, 9200946, 9200947, 9200948, 9200949, 9200950, 9200951, 9200952, 9200953, 9200954, 9200955, 9200956, 9200957, 9200958, 9200959, 9200960, 9200961, 9200962, 9200963, 9200964, 9200965, 9200966, 9200967, 9200968, 9200969, 9200970, 9200971, 9200972, 9200973, 9200974, 9200975, 9200976, 9200977, 9200978, 9200979, 9200980, 9200981, 9200982, 9200983, 9200984, 9200985, 9200986, 9200987, 9200988, 9200989, 9200990, 9200991, 9200993, 9200994, 9200995, 9200996, 9200997, 9200998, 9200999, 9201000, 9201001, 9201002, 9201003, 9201004, 9201005, 9201006, 9201007, 9201008, 9201009, 9201010, 9201011, 9201012, 9201013, 9201014, 9201015, 9201016, 9201017, 9201018, 9201019, 9201020, 9201021, 9201022, 9201023, 9201024, 9201025, 9201026, 9201027, 9201028, 9201029, 9201030, 9201031, 9201032, 9201033, 9201034, 9201035, 9201036, 9201037, 9201038, 9201039, 9201040, 9201041, 9201042, 9201043, 9201044, 9201045, 9201046, 9201047, 9201048, 9201049, 9201050, 9201051, 9201052, 9201053, 9201054, 9201055, 9201056, 9201057, 9201058, 9201059, 9201060, 9201061, 9201062, 9201063, 9201064, 9201065, 9201066, 9201067, 9201068, 9201069, 9201070, 9201071, 9201072, 9201073, 9201074, 9201075, 9201076, 9201077, 9201078, 9201079, 9201080, 9201081, 9201082, 9201083, 9201084, 9201085, 9201086, 9201087, 9201088, 9201089, 9201090, 9201091, 9201092, 9201093, 9201094, 9201095, 9201096, 9201097, 9201098, 9201099, 9201100, 9201101, 9201102, 9201103, 9201104, 9201105, 9201106, 9201107, 9201108, 9201109, 9201110, 9201111, 9201112, 9201113, 9201114, 9201115, 9201116, 9201117, 9201118, 9201119, 9201120, 9201121, 9201122, 9201124, 9201125, 9201126, 9201127, 9201128, 9201129, 9201130, 9201131, 9201132, 9201133, 9201134, 9201135, 9201136, 9201137, 9201138, 9201139, 9201140, 9201141, 9201142, 9201143, 9201144, 9201145, 9201146, 9201147, 9201148, 9201149, 9201150, 9201151, 9201152, 9201153, 9201154, 9201155, 9201156, 9201157, 9201158, 9201159, 9201160, 9201161, 9201162, 9201163, 9201164, 9201165, 9201166, 9201167, 9201168, 9201169, 9201170, 9201171, 9201172, 9201173, 9201174, 9201175, 9201176, 9201177, 9201178, 9201179, 9201180, 9201181, 9201182, 9201183, 9201184, 9201185, 9201186, 9201187, 9201188, 9201189, 9201190, 9201191, 9201192, 9201193, 9201194, 9201195, 9201196, 9201197, 9201198, 9201199, 9201200, 9201201, 9201202, 9201203, 9201204, 9201205, 9201206, 9201207, 9201208, 9201209, 9201210, 9201211, 9201212, 9201213, 9201214, 9201215, 9201216, 9201217, 9201218, 9201219, 9201220, 9201221, 9201222, 9201223, 9201224, 9201225, 9201226, 9201227, 9201228, 9201229, 9201230, 9201231, 9201232, 9201233, 9201234, 9201235, 9201236, 9201237, 9201238, 9201239, 9201240, 9201241, 9201242, 9201243, 9201244, 9201245, 9201246, 9201247, 9201248, 9201249, 9201250, 9201251, 9201252, 9201253, 9201254, 9201255, 9201256, 9201257, 9201258, 9201259, 9201260, 9201261, 9201262, 9201263, 9201264, 9201265, 9201266, 9201267, 9201268, 9201269, 9201270, 9201271, 9201272, 9201273, 9201274, 9201275, 9201276, 9201277, 9201278, 9201279, 9201280, 9201281, 9201282, 9201283, 9201284, 9201285, 9201286, 9201287, 9201288, 9201289, 9201290, 9201291, 9201292, 9201293, 9201294, 9201295, 9201296, 9201297, 9201298, 9201299, 9201300, 9201301, 9201302, 9201303, 9201304, 9201305, 9201306, 9201307, 9201308, 9201309, 9201310, 9201311, 9201312, 9201313, 9201314, 9201315, 9201316, 9201317, 9201318, 9201319, 9201320, 9201321, 9201322, 9201323, 9201324, 9201325, 9201326, 9201327, 9201328, 9201329, 9201330, 9201331, 9201332, 9201333, 9201334, 9201335, 9201336, 9201337, 9201338, 9201339, 9201340, 9201341, 9201342, 9201343, 9201344, 9201345, 9201346, 9201347, 9201348, 9201349, 9201350, 9201351, 9201352, 9201353, 9201354, 9201355, 9201356, 9201357, 9201358, 9201359, 9201360, 9201361, 9201362, 9201363, 9201364, 9201365, 9201366, 9201367, 9201368, 9201369, 9201370, 9201371, 9201372, 9201373, 9201374, 9201375, 9201376, 9201377, 9201378, 9201379, 9201380, 9201381, 9201382, 9201384, 9201385, 9201386, 9201387, 9201388, 9201389, 9201390, 9201391, 9201392, 9201393, 9201394, 9201395, 9201396, 9201397, 9201398, 9201399, 9201400, 9201401, 9201402, 9201403, 9201404, 9201405, 9201406, 9201407, 9201408, 9201409, 9201410, 9201411, 9201412, 9201413, 9201414, 9201415, 9201416, 9201417, 9201418, 9201419, 9201420, 9201422, 9201423, 9201424, 9201425, 9201426, 9201427, 9201428, 9201429, 9201430, 9201431, 9201432, 9201433, 9201434, 9201435, 9201436, 9201437, 9201438, 9201439, 9201440, 9201441, 9201442, 9201443, 9201444, 9201445, 9201446, 9201447, 9201448, 9201449, 9201450, 9201451, 9201452, 9201453, 9201454, 9201455, 9201456, 9201457, 9201458, 9201459, 9201460, 9201461, 9201462, 9201463, 9201464, 9201465, 9201466, 9201467, 9201468, 9201469, 9201470, 9201471, 9201472, 9201473, 9201474, 9201475, 9201476, 9201477, 9201478, 9201479, 9201480, 9201481, 9201482, 9201483, 9201484, 9201485, 9201486, 9201487, 9201488, 9201489, 9201490, 9201491, 9201492, 9201493, 9201494, 9201495, 9201496, 9201497, 9201498, 9201499, 9201500, 9201501, 9201502, 9201503, 9201504, 9201505, 9201506, 9201507, 9201508, 9201509, 9201510, 9201511, 9201512, 9201514, 9201515, 9201516, 9201517, 9201518, 9201519, 9201520, 9201521, 9201522, 9201523, 9201524, 9201525, 9201526, 9201527, 9201528, 9201529, 9201530, 9201531, 9201532, 9201533, 9201534, 9201535, 9201536, 9201537, 9201538, 9201539, 9201540, 9201541, 9201542, 9201543, 9201544, 9201545, 9201546, 9201547, 9201548, 9201549, 9201550, 9201551, 9201552, 9201553, 9201554, 9201555, 9201556, 9201557, 9201558, 9201559, 9201560, 9201561, 9201562, 9201563, 9201564, 9201565, 9201566, 9201567, 9201568, 9201569, 9201570, 9201571, 9201572, 9201573, 9201574, 9201575, 9201576, 9201577, 9201578, 9201579, 9201580, 9201581, 9201582, 9201583, 9201584, 9201585, 9201586, 9201587, 9201588, 9201589, 9201590, 9201591]
-```
-:::
-
-And we can use the evaluation listing functionality to learn more about
-the evaluations available for the conducted runs:
-
-``` default
-evaluations = openml.evaluations.list_evaluations(
- function="predictive_accuracy",
- output_format="dataframe",
- study=study.study_id,
-)
-print(evaluations.head())
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-run_id task_id setup_id ... value values array_data
-0 9199877 3 7130157 ... 0.974969 None None
-1 9199878 6 7130158 ... 0.716500 None None
-2 9199879 6 7130159 ... 0.967200 None None
-3 9199880 11 7130158 ... 0.886400 None None
-4 9199881 11 7130159 ... 0.976000 None None
-
-[5 rows x 14 columns]
-```
-:::
-
-We\'ll use the test server for the rest of this tutorial.
-
-:::: warning
-::: title
-Warning
-:::
-
-.. include:: ../../test_server_usage_warning.txt
-::::
-
-``` default
-openml.config.start_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!
- warnings.warn(
-```
-:::
-
-### Uploading studies
-
-Creating a study is as simple as creating any kind of other OpenML
-entity. In this examples we\'ll create a few runs for the OpenML-100
-benchmark suite which is available on the OpenML test server.
-
-``` default
-# Model to be used
-clf = RandomForestClassifier()
-
-# We'll create a study with one run on 3 datasets present in the suite
-tasks = [115, 259, 307]
-
-# To verify
-suite = openml.study.get_suite(1)
-print(all([t_id in suite.tasks for t_id in tasks]))
-
-run_ids = []
-for task_id in tasks:
- task = openml.tasks.get_task(task_id)
- run = openml.runs.run_model_on_task(clf, task)
- run.publish()
- run_ids.append(run.run_id)
-
-# The study needs a machine-readable and unique alias. To obtain this,
-# we simply generate a random uuid.
-alias = uuid.uuid4().hex
-
-new_study = openml.study.create_study(
- name="Test-Study",
- description="Test study for the Python tutorial on studies",
- run_ids=run_ids,
- alias=alias,
- benchmark_suite=suite.study_id,
-)
-new_study.publish()
-print(new_study)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-True
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-OpenML Study
-============
-ID..............: 333
-Name............: Test-Study
-Status..........: None
-Main Entity Type: run
-Study URL.......: https://test.openml.org/s/333
-# of Runs.......: 3
-```
-:::
-
-``` default
-openml.config.stop_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-timing
-
-**Total running time of the script:** ( 0 minutes 16.432 seconds)
-:::
-
-::::::: {#sphx_glr_download_examples_30_extended_study_tutorial.py}
-:::::: only
-html
-
-::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
-::: {.container .sphx-glr-download .sphx-glr-download-python}
-`Download Python source code: study_tutorial.py <study_tutorial.py>`{.interpreted-text
-role="download"}
-:::
-
-::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
-`Download Jupyter notebook: study_tutorial.ipynb <study_tutorial.ipynb>`{.interpreted-text
-role="download"}
-:::
-:::::
-::::::
-:::::::
-
-:::: only
-html
-
-::: rst-class
-sphx-glr-signature
-
-[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
-:::
-::::
diff --git a/docs/examples/30_extended/suites_tutorial.rst.md b/docs/examples/30_extended/suites_tutorial.rst.md
deleted file mode 100644
index c1d74f8e4..000000000
--- a/docs/examples/30_extended/suites_tutorial.rst.md
+++ /dev/null
@@ -1,291 +0,0 @@
-::::: only
-html
-
-:::: {.note .sphx-glr-download-link-note}
-::: title
-Note
-:::
-
-`Go to the end <sphx_glr_download_examples_30_extended_suites_tutorial.py>`{.interpreted-text
-role="ref"} to download the full example code
-::::
-:::::
-
-::: rst-class
-sphx-glr-example-title
-:::
-
-# Benchmark suites {#sphx_glr_examples_30_extended_suites_tutorial.py}
-
-How to list, download and upload benchmark suites.
-
-If you want to learn more about benchmark suites, check out our brief
-introductory tutorial
-`sphx_glr_examples_20_basic_simple_suites_tutorial.py`{.interpreted-text
-role="ref"} or the [OpenML benchmark
-docs](https://docs.openml.org/benchmark/#benchmarking-suites).
-
-``` default
-# License: BSD 3-Clause
-
-import uuid
-
-import numpy as np
-
-import openml
-```
-
-## Listing suites
-
-- Use the output_format parameter to select output type
-- Default gives `dict`, but we\'ll use `dataframe` to obtain an
- easier-to-work-with data structure
-
-``` default
-suites = openml.study.list_suites(output_format="dataframe", status="all")
-print(suites.head(n=10))
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-id alias ... creation_date creator
-14 14 OpenML100 ... 2019-02-21 18:40:13 1
-99 99 OpenML-CC18 ... 2019-02-21 18:47:13 1
-218 218 AutoML-Benchmark ... 2019-05-02 13:35:08 869
-219 219 FOREX ... 2019-06-04 00:45:17 1
-225 225 OpenML-friendly ... 2019-09-16 19:41:46 1
-236 236 a9ee1f0b2a4b48b6b6da1653fe92890e ... 2020-04-06 21:38:55 64
-239 239 c638a5d3d31241179f9b4853951fdb79 ... 2020-04-19 22:15:30 64
-240 240 e5e7f56c8655433eb2418c240ec8b8c0 ... 2020-04-28 02:30:38 2902
-253 253 testecc18 ... 2020-09-01 00:57:54 8598
-258 258 NaN ... 2020-09-30 08:30:00 8598
-
-[10 rows x 7 columns]
-```
-:::
-
-### Downloading suites
-
-This is done based on the dataset ID.
-
-``` default
-suite = openml.study.get_suite(99)
-print(suite)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-OpenML Benchmark Suite
-======================
-ID..............: 99
-Name............: OpenML-CC18 Curated Classification benchmark
-Status..........: active
-Main Entity Type: task
-Study URL.......: https://www.openml.org/s/99
-# of Data.......: 72
-# of Tasks......: 72
-Creator.........: https://www.openml.org/u/1
-Upload Time.....: 2019-02-21 18:47:13
-```
-:::
-
-Suites also feature a description:
-
-``` default
-print(suite.description)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-```` none
-We advocate the use of curated, comprehensive benchmark suites of machine learning datasets, backed by standardized OpenML-based interfaces and complementary software toolkits written in Python, Java and R. We demonstrate how to easily execute comprehensive benchmarking studies using standardized OpenML-based benchmarking suites and complementary software toolkits written in Python, Java and R. Major distinguishing features of OpenML benchmark suites are (i) ease of use through standardized data formats, APIs, and existing client libraries; (ii) machine-readable meta-information regarding the contents of the suite; and (iii) online sharing of results, enabling large scale comparisons. As a first such suite, we propose the OpenML-CC18, a machine learning benchmark suite of 72 classification datasets carefully curated from the thousands of datasets on OpenML.
-
-The inclusion criteria are:
-* classification tasks on dense data set
-independent observations
-* number of classes larger or equal to 2, each class with at least 20 observations and ratio of minority to majority class must exceed 5%
-* number of observations between 500 and 100000
-* number of features after one-hot-encoding less than 5000
-* no artificial data sets
-* no subsets of larger data sets nor binarizations of other data sets
-* no data sets which are perfectly predictable by using a single feature or by using a simple decision tree
-* source or reference available
-
-
-If you use this benchmarking suite, please cite:
-
-Bernd Bischl, Giuseppe Casalicchio, Matthias Feurer, Frank Hutter, Michel Lang, Rafael G. Mantovani, Jan N. van Rijn and Joaquin Vanschoren. “OpenML Benchmarking Suites” arXiv:1708.03731v2 [stats.ML] (2019).
-
-```
-@article{oml-benchmarking-suites,
- title={OpenML Benchmarking Suites},
- author={Bernd Bischl and Giuseppe Casalicchio and Matthias Feurer and Frank Hutter and Michel Lang and Rafael G. Mantovani and Jan N. van Rijn and Joaquin Vanschoren},
- year={2019},
- journal={arXiv:1708.03731v2 [stat.ML]}
-}
-```
-````
-:::
-
-Suites are a container for tasks:
-
-``` default
-print(suite.tasks)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-[3, 6, 11, 12, 14, 15, 16, 18, 22, 23, 28, 29, 31, 32, 37, 43, 45, 49, 53, 219, 2074, 2079, 3021, 3022, 3481, 3549, 3560, 3573, 3902, 3903, 3904, 3913, 3917, 3918, 7592, 9910, 9946, 9952, 9957, 9960, 9964, 9971, 9976, 9977, 9978, 9981, 9985, 10093, 10101, 14952, 14954, 14965, 14969, 14970, 125920, 125922, 146195, 146800, 146817, 146819, 146820, 146821, 146822, 146824, 146825, 167119, 167120, 167121, 167124, 167125, 167140, 167141]
-```
-:::
-
-And we can use the task listing functionality to learn more about them:
-
-``` default
-tasks = openml.tasks.list_tasks(output_format="dataframe")
-
-# Using ``@`` in `pd.DataFrame.query <
-# https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.query.html>`_
-# accesses variables outside of the current dataframe.
-tasks = tasks.query("tid in @suite.tasks")
-print(tasks.describe().transpose())
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-count mean ... 75% max
-tid 72.0 41849.305556 ... 42707.50 167141.0
-did 72.0 10472.263889 ... 10594.25 41027.0
-MajorityClassSize 72.0 4820.166667 ... 2796.00 48658.0
-MaxNominalAttDistinctValues 71.0 8.718310 ... 10.00 71.0
-MinorityClassSize 72.0 2199.777778 ... 1326.50 47662.0
-NumberOfClasses 72.0 5.680556 ... 8.25 46.0
-NumberOfFeatures 72.0 196.597222 ... 71.50 3073.0
-NumberOfInstances 72.0 12148.972222 ... 6771.75 96320.0
-NumberOfInstancesWithMissingValues 72.0 121.347222 ... 0.00 3772.0
-NumberOfMissingValues 72.0 226.597222 ... 0.00 6465.0
-NumberOfNumericFeatures 72.0 166.875000 ... 64.00 3072.0
-NumberOfSymbolicFeatures 72.0 29.722222 ... 5.25 1556.0
-target_feature_left 0.0 NaN ... NaN NaN
-
-[13 rows x 8 columns]
-```
-:::
-
-We\'ll use the test server for the rest of this tutorial.
-
-:::: warning
-::: title
-Warning
-:::
-
-.. include:: ../../test_server_usage_warning.txt
-::::
-
-``` default
-openml.config.start_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!
- warnings.warn(
-```
-:::
-
-### Uploading suites
-
-Uploading suites is as simple as uploading any kind of other OpenML
-entity - the only reason why we need so much code in this example is
-because we upload some random data.
-
-``` default
-# We'll take a random subset of at least ten tasks of all available tasks on
-# the test server:
-all_tasks = list(openml.tasks.list_tasks(output_format="dataframe")["tid"])
-task_ids_for_suite = sorted(np.random.choice(all_tasks, replace=False, size=20))
-
-# The study needs a machine-readable and unique alias. To obtain this,
-# we simply generate a random uuid.
-
-alias = uuid.uuid4().hex
-
-new_suite = openml.study.create_benchmark_suite(
- name="Test-Suite",
- description="Test suite for the Python tutorial on benchmark suites",
- task_ids=task_ids_for_suite,
- alias=alias,
-)
-new_suite.publish()
-print(new_suite)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/tasks/functions.py:112: RuntimeWarning: Could not create task type id for 10 due to error 10 is not a valid TaskType
- warnings.warn(
-/code/openml/tasks/functions.py:112: RuntimeWarning: Could not create task type id for 11 due to error 11 is not a valid TaskType
- warnings.warn(
-/code/openml/tasks/functions.py:263: RuntimeWarning: Could not create task type id for 10 due to error 10 is not a valid TaskType
- warnings.warn(
-OpenML Benchmark Suite
-======================
-ID..............: 332
-Name............: Test-Suite
-Status..........: None
-Main Entity Type: task
-Study URL.......: https://test.openml.org/s/332
-# of Tasks......: 20
-```
-:::
-
-``` default
-openml.config.stop_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-timing
-
-**Total running time of the script:** ( 0 minutes 30.398 seconds)
-:::
-
-::::::: {#sphx_glr_download_examples_30_extended_suites_tutorial.py}
-:::::: only
-html
-
-::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
-::: {.container .sphx-glr-download .sphx-glr-download-python}
-`Download Python source code: suites_tutorial.py <suites_tutorial.py>`{.interpreted-text
-role="download"}
-:::
-
-::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
-`Download Jupyter notebook: suites_tutorial.ipynb <suites_tutorial.ipynb>`{.interpreted-text
-role="download"}
-:::
-:::::
-::::::
-:::::::
-
-:::: only
-html
-
-::: rst-class
-sphx-glr-signature
-
-[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
-:::
-::::
diff --git a/docs/examples/30_extended/task_manual_iteration_tutorial.rst.md b/docs/examples/30_extended/task_manual_iteration_tutorial.rst.md
deleted file mode 100644
index 059b5cc6f..000000000
--- a/docs/examples/30_extended/task_manual_iteration_tutorial.rst.md
+++ /dev/null
@@ -1,522 +0,0 @@
-::::: only
-html
-
-:::: {.note .sphx-glr-download-link-note}
-::: title
-Note
-:::
-
-`Go to the end <sphx_glr_download_examples_30_extended_task_manual_iteration_tutorial.py>`{.interpreted-text
-role="ref"} to download the full example code
-::::
-:::::
-
-::: rst-class
-sphx-glr-example-title
-:::
-
-# Tasks: retrieving splits {#sphx_glr_examples_30_extended_task_manual_iteration_tutorial.py}
-
-Tasks define a target and a train/test split. Normally, they are the
-input to the function `openml.runs.run_model_on_task` which
-automatically runs the model on all splits of the task. However,
-sometimes it is necessary to manually split a dataset to perform
-experiments outside of the functions provided by OpenML. One such
-example is in the benchmark library
-[HPOBench](https://github.com/automl/HPOBench) which extensively uses
-data from OpenML, but not OpenML\'s functionality to conduct runs.
-
-``` default
-# License: BSD 3-Clause
-
-import openml
-```
-
-For this tutorial we will use the famous King+Rook versus King+Pawn on
-A7 dataset, which has the dataset ID 3 ([dataset on
-OpenML](https://www.openml.org/d/3)), and for which there exist tasks
-with all important estimation procedures. It is small enough (less than
-5000 samples) to efficiently use it in an example.
-
-We will first start with ([task 233](https://www.openml.org/t/233)),
-which is a task with a holdout estimation procedure.
-
-``` default
-task_id = 233
-task = openml.tasks.get_task(task_id)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-```
-:::
-
-Now that we have a task object we can obtain the number of repetitions,
-folds and samples as defined by the task:
-
-``` default
-n_repeats, n_folds, n_samples = task.get_split_dimensions()
-```
-
-- `n_repeats`: Number of times the model quality estimation is
- performed
-- `n_folds`: Number of folds per repeat
-- `n_samples`: How many data points to use. This is only relevant for
- learning curve tasks
-
-A list of all available estimation procedures is available
-[here](https://www.openml.org/search?q=%2520measure_type%3Aestimation_procedure&type=measure).
-
-Task `233` is a simple task using the holdout estimation procedure and
-therefore has only a single repeat, a single fold and a single sample
-size:
-
-``` default
-print(
- "Task {}: number of repeats: {}, number of folds: {}, number of samples {}.".format(
- task_id,
- n_repeats,
- n_folds,
- n_samples,
- )
-)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-Task 233: number of repeats: 1, number of folds: 1, number of samples 1.
-```
-:::
-
-We can now retrieve the train/test split for this combination of
-repeats, folds and number of samples (indexing is zero-based). Usually,
-one would loop over all repeats, folds and sample sizes, but we can
-neglect this here as there is only a single repetition.
-
-``` default
-train_indices, test_indices = task.get_train_test_split_indices(
- repeat=0,
- fold=0,
- sample=0,
-)
-
-print(train_indices.shape, train_indices.dtype)
-print(test_indices.shape, test_indices.dtype)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-(2142,) int32
-(1054,) int32
-```
-:::
-
-And then split the data based on this:
-
-``` default
-X, y = task.get_X_and_y(dataset_format="dataframe")
-X_train = X.iloc[train_indices]
-y_train = y.iloc[train_indices]
-X_test = X.iloc[test_indices]
-y_test = y.iloc[test_indices]
-
-print(
- "X_train.shape: {}, y_train.shape: {}, X_test.shape: {}, y_test.shape: {}".format(
- X_train.shape,
- y_train.shape,
- X_test.shape,
- y_test.shape,
- )
-)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-X_train.shape: (2142, 36), y_train.shape: (2142,), X_test.shape: (1054, 36), y_test.shape: (1054,)
-```
-:::
-
-Obviously, we can also retrieve cross-validation versions of the dataset
-used in task `233`:
-
-``` default
-task_id = 3
-task = openml.tasks.get_task(task_id)
-X, y = task.get_X_and_y(dataset_format="dataframe")
-n_repeats, n_folds, n_samples = task.get_split_dimensions()
-print(
- "Task {}: number of repeats: {}, number of folds: {}, number of samples {}.".format(
- task_id,
- n_repeats,
- n_folds,
- n_samples,
- )
-)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-Task 3: number of repeats: 1, number of folds: 10, number of samples 1.
-```
-:::
-
-And then perform the aforementioned iteration over all splits:
-
-``` default
-for repeat_idx in range(n_repeats):
- for fold_idx in range(n_folds):
- for sample_idx in range(n_samples):
- train_indices, test_indices = task.get_train_test_split_indices(
- repeat=repeat_idx,
- fold=fold_idx,
- sample=sample_idx,
- )
- X_train = X.iloc[train_indices]
- y_train = y.iloc[train_indices]
- X_test = X.iloc[test_indices]
- y_test = y.iloc[test_indices]
-
- print(
- "Repeat #{}, fold #{}, samples {}: X_train.shape: {}, "
- "y_train.shape {}, X_test.shape {}, y_test.shape {}".format(
- repeat_idx,
- fold_idx,
- sample_idx,
- X_train.shape,
- y_train.shape,
- X_test.shape,
- y_test.shape,
- )
- )
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-Repeat #0, fold #0, samples 0: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #1, samples 0: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #2, samples 0: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #3, samples 0: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #4, samples 0: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #5, samples 0: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #6, samples 0: X_train.shape: (2877, 36), y_train.shape (2877,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #7, samples 0: X_train.shape: (2877, 36), y_train.shape (2877,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #8, samples 0: X_train.shape: (2877, 36), y_train.shape (2877,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #9, samples 0: X_train.shape: (2877, 36), y_train.shape (2877,), X_test.shape (319, 36), y_test.shape (319,)
-```
-:::
-
-And also versions with multiple repeats:
-
-``` default
-task_id = 1767
-task = openml.tasks.get_task(task_id)
-X, y = task.get_X_and_y(dataset_format="dataframe")
-n_repeats, n_folds, n_samples = task.get_split_dimensions()
-print(
- "Task {}: number of repeats: {}, number of folds: {}, number of samples {}.".format(
- task_id,
- n_repeats,
- n_folds,
- n_samples,
- )
-)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-Task 1767: number of repeats: 5, number of folds: 2, number of samples 1.
-```
-:::
-
-And then again perform the aforementioned iteration over all splits:
-
-``` default
-for repeat_idx in range(n_repeats):
- for fold_idx in range(n_folds):
- for sample_idx in range(n_samples):
- train_indices, test_indices = task.get_train_test_split_indices(
- repeat=repeat_idx,
- fold=fold_idx,
- sample=sample_idx,
- )
- X_train = X.iloc[train_indices]
- y_train = y.iloc[train_indices]
- X_test = X.iloc[test_indices]
- y_test = y.iloc[test_indices]
-
- print(
- "Repeat #{}, fold #{}, samples {}: X_train.shape: {}, "
- "y_train.shape {}, X_test.shape {}, y_test.shape {}".format(
- repeat_idx,
- fold_idx,
- sample_idx,
- X_train.shape,
- y_train.shape,
- X_test.shape,
- y_test.shape,
- )
- )
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-Repeat #0, fold #0, samples 0: X_train.shape: (1598, 36), y_train.shape (1598,), X_test.shape (1598, 36), y_test.shape (1598,)
-Repeat #0, fold #1, samples 0: X_train.shape: (1598, 36), y_train.shape (1598,), X_test.shape (1598, 36), y_test.shape (1598,)
-Repeat #1, fold #0, samples 0: X_train.shape: (1598, 36), y_train.shape (1598,), X_test.shape (1598, 36), y_test.shape (1598,)
-Repeat #1, fold #1, samples 0: X_train.shape: (1598, 36), y_train.shape (1598,), X_test.shape (1598, 36), y_test.shape (1598,)
-Repeat #2, fold #0, samples 0: X_train.shape: (1598, 36), y_train.shape (1598,), X_test.shape (1598, 36), y_test.shape (1598,)
-Repeat #2, fold #1, samples 0: X_train.shape: (1598, 36), y_train.shape (1598,), X_test.shape (1598, 36), y_test.shape (1598,)
-Repeat #3, fold #0, samples 0: X_train.shape: (1598, 36), y_train.shape (1598,), X_test.shape (1598, 36), y_test.shape (1598,)
-Repeat #3, fold #1, samples 0: X_train.shape: (1598, 36), y_train.shape (1598,), X_test.shape (1598, 36), y_test.shape (1598,)
-Repeat #4, fold #0, samples 0: X_train.shape: (1598, 36), y_train.shape (1598,), X_test.shape (1598, 36), y_test.shape (1598,)
-Repeat #4, fold #1, samples 0: X_train.shape: (1598, 36), y_train.shape (1598,), X_test.shape (1598, 36), y_test.shape (1598,)
-```
-:::
-
-And finally a task based on learning curves:
-
-``` default
-task_id = 1702
-task = openml.tasks.get_task(task_id)
-X, y = task.get_X_and_y(dataset_format="dataframe")
-n_repeats, n_folds, n_samples = task.get_split_dimensions()
-print(
- "Task {}: number of repeats: {}, number of folds: {}, number of samples {}.".format(
- task_id,
- n_repeats,
- n_folds,
- n_samples,
- )
-)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-Task 1702: number of repeats: 1, number of folds: 10, number of samples 12.
-```
-:::
-
-And then again perform the aforementioned iteration over all splits:
-
-``` default
-for repeat_idx in range(n_repeats):
- for fold_idx in range(n_folds):
- for sample_idx in range(n_samples):
- train_indices, test_indices = task.get_train_test_split_indices(
- repeat=repeat_idx,
- fold=fold_idx,
- sample=sample_idx,
- )
- X_train = X.iloc[train_indices]
- y_train = y.iloc[train_indices]
- X_test = X.iloc[test_indices]
- y_test = y.iloc[test_indices]
-
- print(
- "Repeat #{}, fold #{}, samples {}: X_train.shape: {}, "
- "y_train.shape {}, X_test.shape {}, y_test.shape {}".format(
- repeat_idx,
- fold_idx,
- sample_idx,
- X_train.shape,
- y_train.shape,
- X_test.shape,
- y_test.shape,
- )
- )
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-Repeat #0, fold #0, samples 0: X_train.shape: (64, 36), y_train.shape (64,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #0, samples 1: X_train.shape: (91, 36), y_train.shape (91,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #0, samples 2: X_train.shape: (128, 36), y_train.shape (128,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #0, samples 3: X_train.shape: (181, 36), y_train.shape (181,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #0, samples 4: X_train.shape: (256, 36), y_train.shape (256,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #0, samples 5: X_train.shape: (362, 36), y_train.shape (362,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #0, samples 6: X_train.shape: (512, 36), y_train.shape (512,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #0, samples 7: X_train.shape: (724, 36), y_train.shape (724,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #0, samples 8: X_train.shape: (1024, 36), y_train.shape (1024,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #0, samples 9: X_train.shape: (1448, 36), y_train.shape (1448,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #0, samples 10: X_train.shape: (2048, 36), y_train.shape (2048,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #0, samples 11: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #1, samples 0: X_train.shape: (64, 36), y_train.shape (64,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #1, samples 1: X_train.shape: (91, 36), y_train.shape (91,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #1, samples 2: X_train.shape: (128, 36), y_train.shape (128,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #1, samples 3: X_train.shape: (181, 36), y_train.shape (181,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #1, samples 4: X_train.shape: (256, 36), y_train.shape (256,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #1, samples 5: X_train.shape: (362, 36), y_train.shape (362,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #1, samples 6: X_train.shape: (512, 36), y_train.shape (512,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #1, samples 7: X_train.shape: (724, 36), y_train.shape (724,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #1, samples 8: X_train.shape: (1024, 36), y_train.shape (1024,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #1, samples 9: X_train.shape: (1448, 36), y_train.shape (1448,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #1, samples 10: X_train.shape: (2048, 36), y_train.shape (2048,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #1, samples 11: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #2, samples 0: X_train.shape: (64, 36), y_train.shape (64,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #2, samples 1: X_train.shape: (91, 36), y_train.shape (91,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #2, samples 2: X_train.shape: (128, 36), y_train.shape (128,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #2, samples 3: X_train.shape: (181, 36), y_train.shape (181,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #2, samples 4: X_train.shape: (256, 36), y_train.shape (256,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #2, samples 5: X_train.shape: (362, 36), y_train.shape (362,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #2, samples 6: X_train.shape: (512, 36), y_train.shape (512,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #2, samples 7: X_train.shape: (724, 36), y_train.shape (724,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #2, samples 8: X_train.shape: (1024, 36), y_train.shape (1024,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #2, samples 9: X_train.shape: (1448, 36), y_train.shape (1448,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #2, samples 10: X_train.shape: (2048, 36), y_train.shape (2048,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #2, samples 11: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #3, samples 0: X_train.shape: (64, 36), y_train.shape (64,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #3, samples 1: X_train.shape: (91, 36), y_train.shape (91,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #3, samples 2: X_train.shape: (128, 36), y_train.shape (128,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #3, samples 3: X_train.shape: (181, 36), y_train.shape (181,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #3, samples 4: X_train.shape: (256, 36), y_train.shape (256,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #3, samples 5: X_train.shape: (362, 36), y_train.shape (362,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #3, samples 6: X_train.shape: (512, 36), y_train.shape (512,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #3, samples 7: X_train.shape: (724, 36), y_train.shape (724,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #3, samples 8: X_train.shape: (1024, 36), y_train.shape (1024,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #3, samples 9: X_train.shape: (1448, 36), y_train.shape (1448,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #3, samples 10: X_train.shape: (2048, 36), y_train.shape (2048,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #3, samples 11: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #4, samples 0: X_train.shape: (64, 36), y_train.shape (64,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #4, samples 1: X_train.shape: (91, 36), y_train.shape (91,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #4, samples 2: X_train.shape: (128, 36), y_train.shape (128,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #4, samples 3: X_train.shape: (181, 36), y_train.shape (181,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #4, samples 4: X_train.shape: (256, 36), y_train.shape (256,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #4, samples 5: X_train.shape: (362, 36), y_train.shape (362,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #4, samples 6: X_train.shape: (512, 36), y_train.shape (512,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #4, samples 7: X_train.shape: (724, 36), y_train.shape (724,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #4, samples 8: X_train.shape: (1024, 36), y_train.shape (1024,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #4, samples 9: X_train.shape: (1448, 36), y_train.shape (1448,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #4, samples 10: X_train.shape: (2048, 36), y_train.shape (2048,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #4, samples 11: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #5, samples 0: X_train.shape: (64, 36), y_train.shape (64,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #5, samples 1: X_train.shape: (91, 36), y_train.shape (91,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #5, samples 2: X_train.shape: (128, 36), y_train.shape (128,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #5, samples 3: X_train.shape: (181, 36), y_train.shape (181,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #5, samples 4: X_train.shape: (256, 36), y_train.shape (256,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #5, samples 5: X_train.shape: (362, 36), y_train.shape (362,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #5, samples 6: X_train.shape: (512, 36), y_train.shape (512,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #5, samples 7: X_train.shape: (724, 36), y_train.shape (724,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #5, samples 8: X_train.shape: (1024, 36), y_train.shape (1024,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #5, samples 9: X_train.shape: (1448, 36), y_train.shape (1448,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #5, samples 10: X_train.shape: (2048, 36), y_train.shape (2048,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #5, samples 11: X_train.shape: (2876, 36), y_train.shape (2876,), X_test.shape (320, 36), y_test.shape (320,)
-Repeat #0, fold #6, samples 0: X_train.shape: (64, 36), y_train.shape (64,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #6, samples 1: X_train.shape: (91, 36), y_train.shape (91,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #6, samples 2: X_train.shape: (128, 36), y_train.shape (128,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #6, samples 3: X_train.shape: (181, 36), y_train.shape (181,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #6, samples 4: X_train.shape: (256, 36), y_train.shape (256,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #6, samples 5: X_train.shape: (362, 36), y_train.shape (362,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #6, samples 6: X_train.shape: (512, 36), y_train.shape (512,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #6, samples 7: X_train.shape: (724, 36), y_train.shape (724,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #6, samples 8: X_train.shape: (1024, 36), y_train.shape (1024,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #6, samples 9: X_train.shape: (1448, 36), y_train.shape (1448,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #6, samples 10: X_train.shape: (2048, 36), y_train.shape (2048,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #6, samples 11: X_train.shape: (2877, 36), y_train.shape (2877,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #7, samples 0: X_train.shape: (64, 36), y_train.shape (64,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #7, samples 1: X_train.shape: (91, 36), y_train.shape (91,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #7, samples 2: X_train.shape: (128, 36), y_train.shape (128,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #7, samples 3: X_train.shape: (181, 36), y_train.shape (181,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #7, samples 4: X_train.shape: (256, 36), y_train.shape (256,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #7, samples 5: X_train.shape: (362, 36), y_train.shape (362,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #7, samples 6: X_train.shape: (512, 36), y_train.shape (512,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #7, samples 7: X_train.shape: (724, 36), y_train.shape (724,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #7, samples 8: X_train.shape: (1024, 36), y_train.shape (1024,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #7, samples 9: X_train.shape: (1448, 36), y_train.shape (1448,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #7, samples 10: X_train.shape: (2048, 36), y_train.shape (2048,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #7, samples 11: X_train.shape: (2877, 36), y_train.shape (2877,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #8, samples 0: X_train.shape: (64, 36), y_train.shape (64,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #8, samples 1: X_train.shape: (91, 36), y_train.shape (91,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #8, samples 2: X_train.shape: (128, 36), y_train.shape (128,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #8, samples 3: X_train.shape: (181, 36), y_train.shape (181,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #8, samples 4: X_train.shape: (256, 36), y_train.shape (256,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #8, samples 5: X_train.shape: (362, 36), y_train.shape (362,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #8, samples 6: X_train.shape: (512, 36), y_train.shape (512,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #8, samples 7: X_train.shape: (724, 36), y_train.shape (724,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #8, samples 8: X_train.shape: (1024, 36), y_train.shape (1024,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #8, samples 9: X_train.shape: (1448, 36), y_train.shape (1448,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #8, samples 10: X_train.shape: (2048, 36), y_train.shape (2048,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #8, samples 11: X_train.shape: (2877, 36), y_train.shape (2877,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #9, samples 0: X_train.shape: (64, 36), y_train.shape (64,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #9, samples 1: X_train.shape: (91, 36), y_train.shape (91,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #9, samples 2: X_train.shape: (128, 36), y_train.shape (128,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #9, samples 3: X_train.shape: (181, 36), y_train.shape (181,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #9, samples 4: X_train.shape: (256, 36), y_train.shape (256,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #9, samples 5: X_train.shape: (362, 36), y_train.shape (362,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #9, samples 6: X_train.shape: (512, 36), y_train.shape (512,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #9, samples 7: X_train.shape: (724, 36), y_train.shape (724,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #9, samples 8: X_train.shape: (1024, 36), y_train.shape (1024,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #9, samples 9: X_train.shape: (1448, 36), y_train.shape (1448,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #9, samples 10: X_train.shape: (2048, 36), y_train.shape (2048,), X_test.shape (319, 36), y_test.shape (319,)
-Repeat #0, fold #9, samples 11: X_train.shape: (2877, 36), y_train.shape (2877,), X_test.shape (319, 36), y_test.shape (319,)
-```
-:::
-
-::: rst-class
-sphx-glr-timing
-
-**Total running time of the script:** ( 0 minutes 3.602 seconds)
-:::
-
-::::::: {#sphx_glr_download_examples_30_extended_task_manual_iteration_tutorial.py}
-:::::: only
-html
-
-::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
-::: {.container .sphx-glr-download .sphx-glr-download-python}
-`Download Python source code: task_manual_iteration_tutorial.py <task_manual_iteration_tutorial.py>`{.interpreted-text
-role="download"}
-:::
-
-::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
-`Download Jupyter notebook: task_manual_iteration_tutorial.ipynb <task_manual_iteration_tutorial.ipynb>`{.interpreted-text
-role="download"}
-:::
-:::::
-::::::
-:::::::
-
-:::: only
-html
-
-::: rst-class
-sphx-glr-signature
-
-[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
-:::
-::::
diff --git a/docs/examples/30_extended/tasks_tutorial.rst.md b/docs/examples/30_extended/tasks_tutorial.rst.md
deleted file mode 100644
index c1a07f965..000000000
--- a/docs/examples/30_extended/tasks_tutorial.rst.md
+++ /dev/null
@@ -1,1292 +0,0 @@
-::::: only
-html
-
-:::: {.note .sphx-glr-download-link-note}
-::: title
-Note
-:::
-
-`Go to the end <sphx_glr_download_examples_30_extended_tasks_tutorial.py>`{.interpreted-text
-role="ref"} to download the full example code
-::::
-:::::
-
-::: rst-class
-sphx-glr-example-title
-:::
-
-# Tasks {#sphx_glr_examples_30_extended_tasks_tutorial.py}
-
-A tutorial on how to list and download tasks.
-
-``` default
-# License: BSD 3-Clause
-
-import openml
-from openml.tasks import TaskType
-import pandas as pd
-```
-
-Tasks are identified by IDs and can be accessed in two different ways:
-
-1. In a list providing basic information on all tasks available on
- OpenML. This function will not download the actual tasks, but will
- instead download meta data that can be used to filter the tasks and
- retrieve a set of IDs. We can filter this list, for example, we can
- only list tasks having a special tag or only tasks for a specific
- target such as *supervised classification*.
-2. A single task by its ID. It contains all meta information, the
- target metric, the splits and an iterator which can be used to
- access the splits in a useful manner.
-
-## Listing tasks
-
-We will start by simply listing only *supervised classification* tasks.
-**openml.tasks.list_tasks()** returns a dictionary of dictionaries by
-default, but we request a [pandas
-dataframe](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html)
-instead to have better visualization capabilities and easier access:
-
-``` default
-tasks = openml.tasks.list_tasks(
- task_type=TaskType.SUPERVISED_CLASSIFICATION, output_format="dataframe"
-)
-print(tasks.columns)
-print(f"First 5 of {len(tasks)} tasks:")
-print(tasks.head())
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-Index(['tid', 'ttid', 'did', 'name', 'task_type', 'status',
- 'estimation_procedure', 'evaluation_measures', 'source_data',
- 'target_feature', 'MajorityClassSize', 'MaxNominalAttDistinctValues',
- 'MinorityClassSize', 'NumberOfClasses', 'NumberOfFeatures',
- 'NumberOfInstances', 'NumberOfInstancesWithMissingValues',
- 'NumberOfMissingValues', 'NumberOfNumericFeatures',
- 'NumberOfSymbolicFeatures', 'cost_matrix'],
- dtype='object')
-First 5 of 4239 tasks:
- tid ... cost_matrix
-2 2 ... NaN
-3 3 ... NaN
-4 4 ... NaN
-5 5 ... NaN
-6 6 ... NaN
-
-[5 rows x 21 columns]
-```
-:::
-
-We can filter the list of tasks to only contain datasets with more than
-500 samples, but less than 1000 samples:
-
-``` default
-filtered_tasks = tasks.query("NumberOfInstances > 500 and NumberOfInstances < 1000")
-print(list(filtered_tasks.index))
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-[2, 11, 15, 29, 37, 41, 49, 53, 232, 241, 245, 259, 267, 271, 279, 283, 1766, 1775, 1779, 1793, 1801, 1805, 1813, 1817, 1882, 1891, 1895, 1909, 1917, 1921, 1929, 1933, 1945, 1952, 1956, 1967, 1973, 1977, 1983, 1987, 2079, 2125, 2944, 3022, 3034, 3047, 3049, 3053, 3054, 3055, 3484, 3486, 3492, 3493, 3494, 3512, 3518, 3520, 3521, 3529, 3535, 3549, 3560, 3561, 3583, 3623, 3636, 3640, 3660, 3690, 3691, 3692, 3704, 3706, 3718, 3794, 3803, 3810, 3812, 3813, 3814, 3817, 3833, 3852, 3853, 3857, 3860, 3867, 3877, 3879, 3886, 3913, 3971, 3979, 3992, 3999, 4189, 4191, 4197, 4198, 4199, 4217, 4223, 4225, 4226, 4234, 4240, 4254, 4265, 4266, 4288, 4328, 4341, 4345, 4365, 4395, 4396, 4397, 4409, 4411, 4423, 4499, 4508, 4515, 4517, 4518, 4519, 4522, 4538, 4557, 4558, 4562, 4565, 4572, 4582, 4584, 4591, 4618, 4676, 4684, 4697, 4704, 7286, 7307, 7543, 7548, 7558, 9904, 9905, 9946, 9950, 9971, 9980, 9989, 9990, 10097, 10098, 10101, 12738, 12739, 14954, 14968, 145682, 145800, 145804, 145805, 145825, 145836, 145839, 145848, 145878, 145882, 145914, 145917, 145952, 145959, 145970, 145976, 145978, 146062, 146064, 146065, 146066, 146069, 146092, 146156, 146216, 146219, 146231, 146574, 146576, 146577, 146578, 146583, 146587, 146588, 146593, 146596, 146597, 146600, 146818, 146819, 166859, 166875, 166882, 166884, 166893, 166905, 166906, 166907, 166913, 166915, 166919, 166947, 166953, 166956, 166957, 166958, 166959, 166960, 166967, 166976, 166977, 166978, 166980, 166983, 166988, 166989, 166992, 167016, 167020, 167031, 167037, 167062, 167067, 167068, 167095, 167096, 167100, 167104, 167106, 167151, 167154, 167160, 167163, 167167, 167168, 167171, 167173, 167174, 167175, 167180, 167184, 167187, 167194, 167198, 168300, 168783, 168819, 168820, 168821, 168822, 168823, 168824, 168825, 168907, 189786, 189859, 189899, 189900, 189932, 189937, 189941, 190136, 190138, 190139, 190140, 190143, 190146, 233090, 233094, 233109, 233115, 233171, 233206, 359953, 359954, 359955, 360857, 360865, 360868, 360869, 360951, 360953, 360964, 361107, 361109, 361146, 361147, 361148, 361149, 361150, 361151, 361152, 361153, 361154, 361155, 361156, 361157, 361158, 361159, 361160, 361161, 361163, 361164, 361165, 361166, 361167, 361168, 361169, 361170, 361171, 361172, 361173, 361174, 361175, 361176, 361183, 361185, 361190, 361305, 361338, 361340, 361412, 361415, 361424, 361426, 361432, 361433, 361434, 361436, 361437, 361440, 361442, 361443, 361457, 361463, 361483, 361486, 361492, 361495, 361498, 361499, 361502, 361504, 361506, 361507, 361512, 361517, 361522, 361529, 361531, 361540, 361557, 361572, 361624, 361650, 361660]
-```
-:::
-
-``` default
-# Number of tasks
-print(len(filtered_tasks))
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-368
-```
-:::
-
-Then, we can further restrict the tasks to all have the same resampling
-strategy:
-
-``` default
-filtered_tasks = filtered_tasks.query('estimation_procedure == "10-fold Crossvalidation"')
-print(list(filtered_tasks.index))
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-[2, 11, 15, 29, 37, 41, 49, 53, 2079, 3022, 3484, 3486, 3492, 3493, 3494, 3512, 3518, 3520, 3521, 3529, 3535, 3549, 3560, 3561, 3583, 3623, 3636, 3640, 3660, 3690, 3691, 3692, 3704, 3706, 3718, 3794, 3803, 3810, 3812, 3813, 3814, 3817, 3833, 3852, 3853, 3857, 3860, 3867, 3877, 3879, 3886, 3913, 3971, 3979, 3992, 3999, 7286, 7307, 7548, 7558, 9904, 9905, 9946, 9950, 9971, 9980, 9989, 9990, 10097, 10098, 10101, 14954, 14968, 145682, 145800, 145804, 145805, 145825, 145836, 145839, 145848, 145878, 145882, 145914, 145917, 145952, 145959, 145970, 145976, 145978, 146062, 146064, 146065, 146066, 146069, 146092, 146156, 146216, 146219, 146231, 146818, 146819, 168300, 168907, 189932, 189937, 189941, 190136, 190138, 190139, 190140, 190143, 190146, 233171, 359953, 359954, 359955, 360857, 360865, 360868, 360869, 360951, 360953, 360964, 361107, 361109, 361146, 361147, 361148, 361149, 361150, 361151, 361152, 361153, 361154, 361155, 361156, 361157, 361158, 361159, 361160, 361161, 361163, 361164, 361165, 361166, 361167, 361168, 361169, 361170, 361171, 361172, 361173, 361174, 361175, 361176, 361183, 361185, 361190, 361305, 361338, 361340]
-```
-:::
-
-``` default
-# Number of tasks
-print(len(filtered_tasks))
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-162
-```
-:::
-
-Resampling strategies can be found on the [OpenML
-Website](https://www.openml.org/search?type=measure&q=estimation%20procedure).
-
-Similar to listing tasks by task type, we can list tasks by tags:
-
-``` default
-tasks = openml.tasks.list_tasks(tag="OpenML100", output_format="dataframe")
-print(f"First 5 of {len(tasks)} tasks:")
-print(tasks.head())
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-First 5 of 91 tasks:
- tid ... NumberOfSymbolicFeatures
-3 3 ... 37
-6 6 ... 1
-11 11 ... 1
-12 12 ... 1
-14 14 ... 1
-
-[5 rows x 19 columns]
-```
-:::
-
-Furthermore, we can list tasks based on the dataset id:
-
-``` default
-tasks = openml.tasks.list_tasks(data_id=1471, output_format="dataframe")
-print(f"First 5 of {len(tasks)} tasks:")
-print(tasks.head())
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-First 5 of 25 tasks:
- tid ... number_samples
-9983 9983 ... NaN
-14951 14951 ... NaN
-56483 56483 ... NaN
-56484 56484 ... NaN
-56485 56485 ... NaN
-
-[5 rows x 23 columns]
-```
-:::
-
-In addition, a size limit and an offset can be applied both separately
-and simultaneously:
-
-``` default
-tasks = openml.tasks.list_tasks(size=10, offset=50, output_format="dataframe")
-print(tasks)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-tid ... number_samples
-59 59 ... NaN
-60 60 ... NaN
-62 62 ... 9
-63 63 ... 12
-64 64 ... 1
-65 65 ... 7
-66 66 ... 5
-67 67 ... 6
-68 68 ... 5
-69 69 ... 4
-
-[10 rows x 21 columns]
-```
-:::
-
-**OpenML 100** is a curated list of 100 tasks to start using OpenML.
-They are all supervised classification tasks with more than 500
-instances and less than 50000 instances per task. To make things easier,
-the tasks do not contain highly unbalanced data and sparse data.
-However, the tasks include missing values and categorical features. You
-can find out more about the *OpenML 100* on [the OpenML benchmarking
-page](https://docs.openml.org/benchmark/).
-
-Finally, it is also possible to list all tasks on OpenML with:
-
-``` default
-tasks = openml.tasks.list_tasks(output_format="dataframe")
-print(len(tasks))
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-47087
-```
-:::
-
-### Exercise
-
-Search for the tasks on the \'eeg-eye-state\' dataset.
-
-``` default
-tasks.query('name=="eeg-eye-state"')
-```
-
-<div class="output_subarea output_html rendered_html output_result">
-<div>
-<style scoped>
- .dataframe tbody tr th:only-of-type {
- vertical-align: middle;
- }
-
- .dataframe tbody tr th {
- vertical-align: top;
- }
-
- .dataframe thead th {
- text-align: right;
- }
-</style>
-<table border="1" class="dataframe">
- <thead>
- <tr style="text-align: right;">
- <th></th>
- <th>tid</th>
- <th>ttid</th>
- <th>did</th>
- <th>name</th>
- <th>task_type</th>
- <th>status</th>
- <th>estimation_procedure</th>
- <th>evaluation_measures</th>
- <th>source_data</th>
- <th>target_feature</th>
- <th>MajorityClassSize</th>
- <th>MaxNominalAttDistinctValues</th>
- <th>MinorityClassSize</th>
- <th>NumberOfClasses</th>
- <th>NumberOfFeatures</th>
- <th>NumberOfInstances</th>
- <th>NumberOfInstancesWithMissingValues</th>
- <th>NumberOfMissingValues</th>
- <th>NumberOfNumericFeatures</th>
- <th>NumberOfSymbolicFeatures</th>
- <th>number_samples</th>
- <th>cost_matrix</th>
- <th>source_data_labeled</th>
- <th>target_feature_event</th>
- <th>target_feature_left</th>
- <th>target_feature_right</th>
- <th>quality_measure</th>
- <th>target_value</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <th>3511</th>
- <td>9983</td>
- <td>TaskType.SUPERVISED_CLASSIFICATION</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Supervised Classification</td>
- <td>active</td>
- <td>10-fold Crossvalidation</td>
- <td>NaN</td>
- <td>1471</td>
- <td>Class</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- </tr>
- <tr>
- <th>4692</th>
- <td>14951</td>
- <td>TaskType.SUPERVISED_CLASSIFICATION</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Supervised Classification</td>
- <td>active</td>
- <td>10-fold Crossvalidation</td>
- <td>NaN</td>
- <td>1471</td>
- <td>Class</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- </tr>
- <tr>
- <th>8032</th>
- <td>56483</td>
- <td>TaskType.SUBGROUP_DISCOVERY</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Subgroup Discovery</td>
- <td>active</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>1471</td>
- <td>Class</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>Cortana Quality</td>
- <td>1</td>
- </tr>
- <tr>
- <th>8033</th>
- <td>56484</td>
- <td>TaskType.SUBGROUP_DISCOVERY</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Subgroup Discovery</td>
- <td>active</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>1471</td>
- <td>Class</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>Information gain</td>
- <td>1</td>
- </tr>
- <tr>
- <th>8034</th>
- <td>56485</td>
- <td>TaskType.SUBGROUP_DISCOVERY</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Subgroup Discovery</td>
- <td>active</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>1471</td>
- <td>Class</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>Binomial test</td>
- <td>1</td>
- </tr>
- <tr>
- <th>8035</th>
- <td>56486</td>
- <td>TaskType.SUBGROUP_DISCOVERY</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Subgroup Discovery</td>
- <td>active</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>1471</td>
- <td>Class</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>Jaccard</td>
- <td>1</td>
- </tr>
- <tr>
- <th>8036</th>
- <td>56487</td>
- <td>TaskType.SUBGROUP_DISCOVERY</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Subgroup Discovery</td>
- <td>active</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>1471</td>
- <td>Class</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>Cortana Quality</td>
- <td>2</td>
- </tr>
- <tr>
- <th>8037</th>
- <td>56488</td>
- <td>TaskType.SUBGROUP_DISCOVERY</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Subgroup Discovery</td>
- <td>active</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>1471</td>
- <td>Class</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>Information gain</td>
- <td>2</td>
- </tr>
- <tr>
- <th>8038</th>
- <td>56489</td>
- <td>TaskType.SUBGROUP_DISCOVERY</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Subgroup Discovery</td>
- <td>active</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>1471</td>
- <td>Class</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>Binomial test</td>
- <td>2</td>
- </tr>
- <tr>
- <th>8039</th>
- <td>56490</td>
- <td>TaskType.SUBGROUP_DISCOVERY</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Subgroup Discovery</td>
- <td>active</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>1471</td>
- <td>Class</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>Jaccard</td>
- <td>2</td>
- </tr>
- <tr>
- <th>8581</th>
- <td>75219</td>
- <td>TaskType.SUPERVISED_CLASSIFICATION</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Supervised Classification</td>
- <td>active</td>
- <td>33% Holdout set</td>
- <td>predictive_accuracy</td>
- <td>1471</td>
- <td>Class</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- </tr>
- <tr>
- <th>8663</th>
- <td>125901</td>
- <td>TaskType.LEARNING_CURVE</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Learning Curve</td>
- <td>active</td>
- <td>10-fold Learning Curve</td>
- <td>NaN</td>
- <td>1471</td>
- <td>Class</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>17</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- </tr>
- <tr>
- <th>9864</th>
- <td>127251</td>
- <td>TaskType.CLUSTERING</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Clustering</td>
- <td>active</td>
- <td>50 times Clustering</td>
- <td>NaN</td>
- <td>1471</td>
- <td>NaN</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- </tr>
- <tr>
- <th>11920</th>
- <td>146761</td>
- <td>TaskType.CLUSTERING</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Clustering</td>
- <td>active</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>1471</td>
- <td>Class</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- </tr>
- <tr>
- <th>13204</th>
- <td>148117</td>
- <td>TaskType.CLUSTERING</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Clustering</td>
- <td>active</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>1471</td>
- <td>NaN</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- </tr>
- <tr>
- <th>17309</th>
- <td>170139</td>
- <td>TaskType.CLUSTERING</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Clustering</td>
- <td>active</td>
- <td>50 times Clustering</td>
- <td>NaN</td>
- <td>1471</td>
- <td>NaN</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- </tr>
- <tr>
- <th>20760</th>
- <td>191646</td>
- <td>TaskType.CLUSTERING</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Clustering</td>
- <td>active</td>
- <td>50 times Clustering</td>
- <td>NaN</td>
- <td>1471</td>
- <td>NaN</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- </tr>
- <tr>
- <th>24446</th>
- <td>213320</td>
- <td>TaskType.CLUSTERING</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Clustering</td>
- <td>active</td>
- <td>50 times Clustering</td>
- <td>NaN</td>
- <td>1471</td>
- <td>NaN</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- </tr>
- <tr>
- <th>27655</th>
- <td>234435</td>
- <td>TaskType.CLUSTERING</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Clustering</td>
- <td>active</td>
- <td>50 times Clustering</td>
- <td>NaN</td>
- <td>1471</td>
- <td>NaN</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- </tr>
- <tr>
- <th>31979</th>
- <td>256739</td>
- <td>TaskType.CLUSTERING</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Clustering</td>
- <td>active</td>
- <td>50 times Clustering</td>
- <td>NaN</td>
- <td>1471</td>
- <td>NaN</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- </tr>
- <tr>
- <th>31989</th>
- <td>256750</td>
- <td>TaskType.CLUSTERING</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Clustering</td>
- <td>active</td>
- <td>50 times Clustering</td>
- <td>NaN</td>
- <td>1471</td>
- <td>NaN</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- </tr>
- <tr>
- <th>37166</th>
- <td>297708</td>
- <td>TaskType.CLUSTERING</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Clustering</td>
- <td>active</td>
- <td>50 times Clustering</td>
- <td>NaN</td>
- <td>1471</td>
- <td>NaN</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- </tr>
- <tr>
- <th>40370</th>
- <td>318835</td>
- <td>TaskType.CLUSTERING</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Clustering</td>
- <td>active</td>
- <td>50 times Clustering</td>
- <td>NaN</td>
- <td>1471</td>
- <td>NaN</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- </tr>
- <tr>
- <th>43587</th>
- <td>339990</td>
- <td>TaskType.CLUSTERING</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Clustering</td>
- <td>active</td>
- <td>50 times Clustering</td>
- <td>NaN</td>
- <td>1471</td>
- <td>NaN</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- </tr>
- <tr>
- <th>46851</th>
- <td>361459</td>
- <td>TaskType.SUPERVISED_CLASSIFICATION</td>
- <td>1471</td>
- <td>eeg-eye-state</td>
- <td>Supervised Classification</td>
- <td>active</td>
- <td>5 times 2-fold Crossvalidation</td>
- <td>NaN</td>
- <td>1471</td>
- <td>Class</td>
- <td>8257.0</td>
- <td>2.0</td>
- <td>6723.0</td>
- <td>2.0</td>
- <td>15.0</td>
- <td>14980.0</td>
- <td>0.0</td>
- <td>0.0</td>
- <td>14.0</td>
- <td>1.0</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- <td>NaN</td>
- </tr>
- </tbody>
-</table>
-</div>
-</div>
-<br />
-<br />
-
-## Downloading tasks
-
-We provide two functions to download tasks, one which downloads only a
-single task by its ID, and one which takes a list of IDs and downloads
-all of these tasks:
-
-``` default
-task_id = 31
-task = openml.tasks.get_task(task_id)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/tasks/functions.py:372: FutureWarning: Starting from Version 0.15.0 `download_splits` will default to ``False`` instead of ``True`` and be independent from `download_data`. To disable this message until version 0.15 explicitly set `download_splits` to a bool.
- warnings.warn(
-/code/openml/datasets/functions.py:438: FutureWarning: Starting from Version 0.15 `download_data`, `download_qualities`, and `download_features_meta_data` will all be ``False`` instead of ``True`` by default to enable lazy loading. To disable this message until version 0.15 explicitly set `download_data`, `download_qualities`, and `download_features_meta_data` to a bool while calling `get_dataset`.
- warnings.warn(
-```
-:::
-
-Properties of the task are stored as member variables:
-
-``` default
-print(task)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-OpenML Classification Task
-==========================
-Task Type Description: https://www.openml.org/tt/TaskType.SUPERVISED_CLASSIFICATION
-Task ID..............: 31
-Task URL.............: https://www.openml.org/t/31
-Estimation Procedure.: crossvalidation
-Target Feature.......: class
-# of Classes.........: 2
-Cost Matrix..........: Available
-```
-:::
-
-And:
-
-``` default
-ids = [2, 1891, 31, 9983]
-tasks = openml.tasks.get_tasks(ids)
-print(tasks[0])
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-OpenML Classification Task
-==========================
-Task Type Description: https://www.openml.org/tt/TaskType.SUPERVISED_CLASSIFICATION
-Task ID..............: 2
-Task URL.............: https://www.openml.org/t/2
-Estimation Procedure.: crossvalidation
-Evaluation Measure...: predictive_accuracy
-Target Feature.......: class
-# of Classes.........: 6
-Cost Matrix..........: Available
-```
-:::
-
-## Creating tasks
-
-You can also create new tasks. Take the following into account:
-
-- You can only create tasks on *active* datasets
-- For now, only the following tasks are supported: classification,
- regression, clustering, and learning curve analysis.
-- For now, tasks can only be created on a single dataset.
-- The exact same task must not already exist.
-
-Creating a task requires the following input:
-
-- task_type: The task type ID, required (see below). Required.
-- dataset_id: The dataset ID. Required.
-- target_name: The name of the attribute you aim to predict. Optional.
-- estimation_procedure_id : The ID of the estimation procedure used to
- create train-test splits. Optional.
-- evaluation_measure: The name of the evaluation measure. Optional.
-- Any additional inputs for specific tasks
-
-It is best to leave the evaluation measure open if there is no strong
-prerequisite for a specific measure. OpenML will always compute all
-appropriate measures and you can filter or sort results on your
-favourite measure afterwards. Only add an evaluation measure if
-necessary (e.g. when other measure make no sense), since it will create
-a new task, which scatters results across tasks.
-
-We\'ll use the test server for the rest of this tutorial.
-
-:::: warning
-::: title
-Warning
-:::
-
-.. include:: ../../test_server_usage_warning.txt
-::::
-
-``` default
-openml.config.start_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/config.py:184: UserWarning: Switching to the test server https://test.openml.org/api/v1/xml to not upload results to the live server. Using the test server may result in reduced performance of the API!
- warnings.warn(
-```
-:::
-
-### Example
-
-Let\'s create a classification task on a dataset. In this example we
-will do this on the Iris dataset (ID=128 (on test server)). We\'ll use
-10-fold cross-validation (ID=1), and *predictive accuracy* as the
-predefined measure (this can also be left open). If a task with these
-parameters exists, we will get an appropriate exception. If such a task
-doesn\'t exist, a task will be created and the corresponding task_id
-will be returned.
-
-``` default
-try:
- my_task = openml.tasks.create_task(
- task_type=TaskType.SUPERVISED_CLASSIFICATION,
- dataset_id=128,
- target_name="class",
- evaluation_measure="predictive_accuracy",
- estimation_procedure_id=1,
- )
- my_task.publish()
-except openml.exceptions.OpenMLServerException as e:
- # Error code for 'task already exists'
- if e.code == 614:
- # Lookup task
- tasks = openml.tasks.list_tasks(data_id=128, output_format="dataframe")
- tasks = tasks.query(
- 'task_type == "Supervised Classification" '
- 'and estimation_procedure == "10-fold Crossvalidation" '
- 'and evaluation_measures == "predictive_accuracy"'
- )
- task_id = tasks.loc[:, "tid"].values[0]
- print("Task already exists. Task ID is", task_id)
-
-# reverting to prod server
-openml.config.stop_using_configuration_for_example()
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-/code/openml/tasks/functions.py:112: RuntimeWarning: Could not create task type id for 10 due to error 10 is not a valid TaskType
- warnings.warn(
-/code/openml/tasks/functions.py:112: RuntimeWarning: Could not create task type id for 11 due to error 11 is not a valid TaskType
- warnings.warn(
-/code/openml/tasks/functions.py:263: RuntimeWarning: Could not create task type id for 10 due to error 10 is not a valid TaskType
- warnings.warn(
-Task already exists. Task ID is 1306
-```
-:::
-
-- [Complete list of task
- types](https://www.openml.org/search?type=task_type).
-- [Complete list of model estimation
- procedures](https://www.openml.org/search?q=%2520measure_type%3Aestimation_procedure&type=measure).
-- [Complete list of evaluation
- measures](https://www.openml.org/search?q=measure_type%3Aevaluation_measure&type=measure).
-
-::: rst-class
-sphx-glr-timing
-
-**Total running time of the script:** ( 0 minutes 47.382 seconds)
-:::
-
-::::::: {#sphx_glr_download_examples_30_extended_tasks_tutorial.py}
-:::::: only
-html
-
-::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
-::: {.container .sphx-glr-download .sphx-glr-download-python}
-`Download Python source code: tasks_tutorial.py <tasks_tutorial.py>`{.interpreted-text
-role="download"}
-:::
-
-::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
-`Download Jupyter notebook: tasks_tutorial.ipynb <tasks_tutorial.ipynb>`{.interpreted-text
-role="download"}
-:::
-:::::
-::::::
-:::::::
-
-:::: only
-html
-
-::: rst-class
-sphx-glr-signature
-
-[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
-:::
-::::
diff --git a/docs/examples/40_paper/2015_neurips_feurer_example.rst.md b/docs/examples/40_paper/2015_neurips_feurer_example.rst.md
deleted file mode 100644
index 5972b4b00..000000000
--- a/docs/examples/40_paper/2015_neurips_feurer_example.rst.md
+++ /dev/null
@@ -1,167 +0,0 @@
-::::: only
-html
-
-:::: {.note .sphx-glr-download-link-note}
-::: title
-Note
-:::
-
-`Go to the end <sphx_glr_download_examples_40_paper_2015_neurips_feurer_example.py>`{.interpreted-text
-role="ref"} to download the full example code
-::::
-:::::
-
-::: rst-class
-sphx-glr-example-title
-:::
-
-# Feurer et al. (2015) {#sphx_glr_examples_40_paper_2015_neurips_feurer_example.py}
-
-A tutorial on how to get the datasets used in the paper introducing
-*Auto-sklearn* by Feurer et al..
-
-Auto-sklearn website: <https://automl.github.io/auto-sklearn/>
-
-## Publication
-
-| Efficient and Robust Automated Machine Learning
-| Matthias Feurer, Aaron Klein, Katharina Eggensperger, Jost
- Springenberg, Manuel Blum and Frank Hutter
-| In *Advances in Neural Information Processing Systems 28*, 2015
-| Available at
- <https://papers.nips.cc/paper/5872-efficient-and-robust-automated-machine-learning.pdf>
-
-``` default
-# License: BSD 3-Clause
-
-import pandas as pd
-
-import openml
-```
-
-List of dataset IDs given in the supplementary material of Feurer et
-al.:
-<https://papers.nips.cc/paper/5872-efficient-and-robust-automated-machine-learning-supplemental.zip>
-fmt: off
-
-``` default
-dataset_ids = [
- 3, 6, 12, 14, 16, 18, 21, 22, 23, 24, 26, 28, 30, 31, 32, 36, 38, 44, 46,
- 57, 60, 179, 180, 181, 182, 184, 185, 273, 293, 300, 351, 354, 357, 389,
- 390, 391, 392, 393, 395, 396, 398, 399, 401, 554, 679, 715, 718, 720, 722,
- 723, 727, 728, 734, 735, 737, 740, 741, 743, 751, 752, 761, 772, 797, 799,
- 803, 806, 807, 813, 816, 819, 821, 822, 823, 833, 837, 843, 845, 846, 847,
- 849, 866, 871, 881, 897, 901, 903, 904, 910, 912, 913, 914, 917, 923, 930,
- 934, 953, 958, 959, 962, 966, 971, 976, 977, 978, 979, 980, 991, 993, 995,
- 1000, 1002, 1018, 1019, 1020, 1021, 1036, 1040, 1041, 1049, 1050, 1053,
- 1056, 1067, 1068, 1069, 1111, 1112, 1114, 1116, 1119, 1120, 1128, 1130,
- 1134, 1138, 1139, 1142, 1146, 1161, 1166,
-]
-# fmt: on
-```
-
-The dataset IDs could be used directly to load the dataset and split the
-data into a training set and a test set. However, to be reproducible, we
-will first obtain the respective tasks from OpenML, which define both
-the target feature and the train/test split.
-
-:::: note
-::: title
-Note
-:::
-
-It is discouraged to work directly on datasets and only provide dataset
-IDs in a paper as this does not allow reproducibility (unclear
-splitting). Please do not use datasets but the respective tasks as basis
-for a paper and publish task IDS. This example is only given to showcase
-the use of OpenML-Python for a published paper and as a warning on how
-not to do it. Please check the [OpenML documentation of
-tasks](https://docs.openml.org/#tasks) if you want to learn more about
-them.
-::::
-
-This lists both active and inactive tasks (because of `status='all'`).
-Unfortunately, this is necessary as some of the datasets contain issues
-found after the publication and became deactivated, which also
-deactivated the tasks on them. More information on active or inactive
-datasets can be found in the [online
-docs](https://docs.openml.org/#dataset-status).
-
-``` default
-tasks = openml.tasks.list_tasks(
- task_type=openml.tasks.TaskType.SUPERVISED_CLASSIFICATION,
- status="all",
- output_format="dataframe",
-)
-
-# Query only those with holdout as the resampling startegy.
-tasks = tasks.query('estimation_procedure == "33% Holdout set"')
-
-task_ids = []
-for did in dataset_ids:
- tasks_ = list(tasks.query("did == {}".format(did)).tid)
- if len(tasks_) >= 1: # if there are multiple task, take the one with lowest ID (oldest).
- task_id = min(tasks_)
- else:
- raise ValueError(did)
-
- # Optional - Check that the task has the same target attribute as the
- # dataset default target attribute
- # (disabled for this example as it needs to run fast to be rendered online)
- # task = openml.tasks.get_task(task_id)
- # dataset = task.get_dataset()
- # if task.target_name != dataset.default_target_attribute:
- # raise ValueError(
- # (task.target_name, dataset.default_target_attribute)
- # )
-
- task_ids.append(task_id)
-
-assert len(task_ids) == 140
-task_ids.sort()
-
-# These are the tasks to work with:
-print(task_ids)
-```
-
-::: rst-class
-sphx-glr-script-out
-
-``` none
-[233, 236, 242, 244, 246, 248, 251, 252, 253, 254, 256, 258, 260, 261, 262, 266, 273, 275, 288, 2117, 2118, 2119, 2120, 2122, 2123, 2350, 3043, 3044, 75090, 75092, 75093, 75098, 75099, 75100, 75103, 75104, 75105, 75106, 75107, 75108, 75111, 75112, 75113, 75114, 75115, 75116, 75117, 75119, 75120, 75121, 75122, 75125, 75126, 75129, 75131, 75133, 75136, 75137, 75138, 75139, 75140, 75142, 75143, 75146, 75147, 75148, 75149, 75150, 75151, 75152, 75153, 75155, 75157, 75159, 75160, 75161, 75162, 75163, 75164, 75165, 75166, 75168, 75169, 75170, 75171, 75172, 75173, 75174, 75175, 75176, 75179, 75180, 75182, 75183, 75184, 75185, 75186, 75188, 75189, 75190, 75191, 75192, 75194, 75195, 75196, 75197, 75198, 75199, 75200, 75201, 75202, 75203, 75204, 75205, 75206, 75207, 75208, 75209, 75210, 75212, 75213, 75216, 75218, 75220, 75222, 75224, 75226, 75228, 75229, 75233, 75238, 75240, 75244, 75245, 75246, 75247, 75248, 75249, 75251, 190400]
-```
-:::
-
-::: rst-class
-sphx-glr-timing
-
-**Total running time of the script:** ( 0 minutes 1.752 seconds)
-:::
-
-::::::: {#sphx_glr_download_examples_40_paper_2015_neurips_feurer_example.py}
-:::::: only
-html
-
-::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
-::: {.container .sphx-glr-download .sphx-glr-download-python}
-`Download Python source code: 2015_neurips_feurer_example.py <2015_neurips_feurer_example.py>`{.interpreted-text
-role="download"}
-:::
-
-::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
-`Download Jupyter notebook: 2015_neurips_feurer_example.ipynb <2015_neurips_feurer_example.ipynb>`{.interpreted-text
-role="download"}
-:::
-:::::
-::::::
-:::::::
-
-:::: only
-html
-
-::: rst-class
-sphx-glr-signature
-
-[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
-:::
-::::
diff --git a/docs/examples/40_paper/2018_ida_strang_example.rst.md b/docs/examples/40_paper/2018_ida_strang_example.rst.md
deleted file mode 100644
index 42f5cb09a..000000000
--- a/docs/examples/40_paper/2018_ida_strang_example.rst.md
+++ /dev/null
@@ -1,191 +0,0 @@
-::::: only
-html
-
-:::: {.note .sphx-glr-download-link-note}
-::: title
-Note
-:::
-
-`Go to the end <sphx_glr_download_examples_40_paper_2018_ida_strang_example.py>`{.interpreted-text
-role="ref"} to download the full example code
-::::
-:::::
-
-::: rst-class
-sphx-glr-example-title
-:::
-
-# Strang et al. (2018) {#sphx_glr_examples_40_paper_2018_ida_strang_example.py}
-
-A tutorial on how to reproduce the analysis conducted for *Don\'t Rule
-Out Simple Models Prematurely: A Large Scale Benchmark Comparing Linear
-and Non-linear Classifiers in OpenML*.
-
-## Publication
-
-| Don\'t Rule Out Simple Models Prematurely: A Large Scale Benchmark
- Comparing Linear and Non-linear Classifiers in OpenML
-| Benjamin Strang, Peter van der Putten, Jan N. van Rijn and Frank
- Hutter
-| In *Advances in Intelligent Data Analysis XVII 17th International
- Symposium*, 2018
-| Available at
- <https://link.springer.com/chapter/10.1007%2F978-3-030-01768-2_25>
-
-``` default
-# License: BSD 3-Clause
-
-import matplotlib.pyplot as plt
-import openml
-import pandas as pd
-```
-
-A basic step for each data-mining or machine learning task is to
-determine which model to choose based on the problem and the data at
-hand. In this work we investigate when non-linear classifiers outperform
-linear classifiers by means of a large scale experiment.
-
-The paper is accompanied with a study object, containing all relevant
-tasks and runs (`study_id=123`). The paper features three experiment
-classes: Support Vector Machines (SVM), Neural Networks (NN) and
-Decision Trees (DT). This example demonstrates how to reproduce the
-plots, comparing two classifiers given the OpenML flow ids. Note that
-this allows us to reproduce the SVM and NN experiment, but not the DT
-experiment, as this requires a bit more effort to distinguish the same
-flow with different hyperparameter values.
-
-``` default
-study_id = 123
-# for comparing svms: flow_ids = [7754, 7756]
-# for comparing nns: flow_ids = [7722, 7729]
-# for comparing dts: flow_ids = [7725], differentiate on hyper-parameter value
-classifier_family = "SVM"
-flow_ids = [7754, 7756]
-measure = "predictive_accuracy"
-meta_features = ["NumberOfInstances", "NumberOfFeatures"]
-class_values = ["non-linear better", "linear better", "equal"]
-
-# Downloads all evaluation records related to this study
-evaluations = openml.evaluations.list_evaluations(
- measure, size=None, flows=flow_ids, study=study_id, output_format="dataframe"
-)
-# gives us a table with columns data_id, flow1_value, flow2_value
-evaluations = evaluations.pivot(index="data_id", columns="flow_id", values="value").dropna()
-# downloads all data qualities (for scatter plot)
-data_qualities = openml.datasets.list_datasets(
- data_id=list(evaluations.index.values), output_format="dataframe"
-)
-# removes irrelevant data qualities
-data_qualities = data_qualities[meta_features]
-# makes a join between evaluation table and data qualities table,
-# now we have columns data_id, flow1_value, flow2_value, meta_feature_1,
-# meta_feature_2
-evaluations = evaluations.join(data_qualities, how="inner")
-
-# adds column that indicates the difference between the two classifiers
-evaluations["diff"] = evaluations[flow_ids[0]] - evaluations[flow_ids[1]]
-```
-
-makes the s-plot
-
-``` default
-fig_splot, ax_splot = plt.subplots()
-ax_splot.plot(range(len(evaluations)), sorted(evaluations["diff"]))
-ax_splot.set_title(classifier_family)
-ax_splot.set_xlabel("Dataset (sorted)")
-ax_splot.set_ylabel("difference between linear and non-linear classifier")
-ax_splot.grid(linestyle="--", axis="y")
-plt.show()
-```
-
-::: {.image-sg .sphx-glr-single-img alt="SVM" srcset="/examples/40_paper/images/sphx_glr_2018_ida_strang_example_001.png"}
-/examples/40_paper/images/sphx_glr_2018_ida_strang_example_001.png
-:::
-
-adds column that indicates the difference between the two classifiers,
-needed for the scatter plot
-
-``` default
-def determine_class(val_lin, val_nonlin):
- if val_lin < val_nonlin:
- return class_values[0]
- elif val_nonlin < val_lin:
- return class_values[1]
- else:
- return class_values[2]
-
-
-evaluations["class"] = evaluations.apply(
- lambda row: determine_class(row[flow_ids[0]], row[flow_ids[1]]), axis=1
-)
-
-# does the plotting and formatting
-fig_scatter, ax_scatter = plt.subplots()
-for class_val in class_values:
- df_class = evaluations[evaluations["class"] == class_val]
- plt.scatter(df_class[meta_features[0]], df_class[meta_features[1]], label=class_val)
-ax_scatter.set_title(classifier_family)
-ax_scatter.set_xlabel(meta_features[0])
-ax_scatter.set_ylabel(meta_features[1])
-ax_scatter.legend()
-ax_scatter.set_xscale("log")
-ax_scatter.set_yscale("log")
-plt.show()
-```
-
-::: {.image-sg .sphx-glr-single-img alt="SVM" srcset="/examples/40_paper/images/sphx_glr_2018_ida_strang_example_002.png"}
-/examples/40_paper/images/sphx_glr_2018_ida_strang_example_002.png
-:::
-
-makes a scatter plot where each data point represents the performance of
-the two algorithms on various axis (not in the paper)
-
-``` default
-fig_diagplot, ax_diagplot = plt.subplots()
-ax_diagplot.grid(linestyle="--")
-ax_diagplot.plot([0, 1], ls="-", color="black")
-ax_diagplot.plot([0.2, 1.2], ls="--", color="black")
-ax_diagplot.plot([-0.2, 0.8], ls="--", color="black")
-ax_diagplot.scatter(evaluations[flow_ids[0]], evaluations[flow_ids[1]])
-ax_diagplot.set_xlabel(measure)
-ax_diagplot.set_ylabel(measure)
-plt.show()
-```
-
-::: {.image-sg .sphx-glr-single-img alt="2018 ida strang example" srcset="/examples/40_paper/images/sphx_glr_2018_ida_strang_example_003.png"}
-/examples/40_paper/images/sphx_glr_2018_ida_strang_example_003.png
-:::
-
-::: rst-class
-sphx-glr-timing
-
-**Total running time of the script:** ( 0 minutes 1.507 seconds)
-:::
-
-::::::: {#sphx_glr_download_examples_40_paper_2018_ida_strang_example.py}
-:::::: only
-html
-
-::::: {.container .sphx-glr-footer .sphx-glr-footer-example}
-::: {.container .sphx-glr-download .sphx-glr-download-python}
-`Download Python source code: 2018_ida_strang_example.py <2018_ida_strang_example.py>`{.interpreted-text
-role="download"}
-:::
-
-::: {.container .sphx-glr-download .sphx-glr-download-jupyter}
-`Download Jupyter notebook: 2018_ida_strang_example.ipynb <2018_ida_strang_example.ipynb>`{.interpreted-text
-role="download"}
-:::
-:::::
-::::::
-:::::::
-
-:::: only
-html
-
-::: rst-class
-sphx-glr-signature
-
-[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)
-:::
-::::
diff --git a/docs/examples/index.md b/docs/examples/index.md
deleted file mode 100644
index 4650e144c..000000000
--- a/docs/examples/index.md
+++ /dev/null
@@ -1 +0,0 @@
-# Example Gallery
\ No newline at end of file
From 2cc23af6747b4c6d43346dd934d8759fcf16b635 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 29 Jul 2024 15:33:47 +0200
Subject: [PATCH 10/25] Render notebooks from example folder
---
examples/20_basic/introduction_tutorial.ipynb | 226 ++++++++++++++++++
mkdocs.yml | 2 +
pyproject.toml | 1 +
scripts/gen_ref_pages.py | 15 ++
4 files changed, 244 insertions(+)
create mode 100644 examples/20_basic/introduction_tutorial.ipynb
diff --git a/examples/20_basic/introduction_tutorial.ipynb b/examples/20_basic/introduction_tutorial.ipynb
new file mode 100644
index 000000000..b7c839654
--- /dev/null
+++ b/examples/20_basic/introduction_tutorial.ipynb
@@ -0,0 +1,226 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "source": [
+ "# Introduction tutorial & Setup\n",
+ "\n",
+ "An example how to set up OpenML-Python followed up by a simple example.\n",
+ "OpenML is an online collaboration platform for machine learning which allows\n",
+ "you to:\n",
+ "\n",
+ "* Find or share interesting, well-documented datasets\n",
+ "* Define research / modelling goals (tasks)\n",
+ "* Explore large amounts of machine learning algorithms, with APIs in Java, R, Python\n",
+ "* Log and share reproducible experiments, models, results\n",
+ "* Works seamlessly with scikit-learn and other libraries\n",
+ "* Large scale benchmarking, compare to state of the art"
+ ],
+ "metadata": {
+ "collapsed": false
+ },
+ "id": "3c295aed5545023a"
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "# Installation\n",
+ "installation is done via ``pip``: `pip install openml`\n",
+ "\n",
+ "for further information, please check out the installation guide at\n",
+ ":ref:`installation`."
+ ],
+ "metadata": {
+ "collapsed": false
+ },
+ "id": "48800928f4e5a6a6"
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "# Authentication\n",
+ "\n",
+ "the OpenML server can only be accessed by users who have signed up on the\n",
+ "openML platform. If you don’t have an account yet, sign up now.\n",
+ "you will receive an API key, which will authenticate you to the server\n",
+ "and allow you to download and upload datasets, tasks, runs and flows.\n",
+ "\n",
+ "* Create an OpenML account (free) on [https://www.openml.org](https://www.openml.org).\n",
+ "* After logging in, open your account page (avatar on the top right)\n",
+ "* Open 'Account Settings', then 'API authentication' to find your API key.\n",
+ "\n",
+ "there are two ways to permanently authenticate:\n",
+ "\n",
+ "* Use the ``openml`` CLI tool with ``openml configure apikey MYKEY``,\n",
+ " replacing **MYKEY** with your API key.\n",
+ "* Create a plain text file **~/.openml/config** with the line\n",
+ " **'apikey=MYKEY'**, replacing **MYKEY** with your API key. The config\n",
+ " file must be in the directory ~/.openml/config and exist prior to\n",
+ " importing the openml module.\n",
+ "\n",
+ "alternatively, by running the code below and replacing 'YOURKEY' with your API key,\n",
+ "you authenticate for the duration of the python process."
+ ],
+ "metadata": {
+ "collapsed": false
+ },
+ "id": "7d6e682752db0423"
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "outputs": [],
+ "source": [
+ "import openml\n",
+ "from sklearn import neighbors"
+ ],
+ "metadata": {
+ "collapsed": false
+ },
+ "id": "b2fae124f696eff0"
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "# .. warning::\n",
+ "# .. include:: ../../test_server_usage_warning.txt"
+ ],
+ "metadata": {
+ "collapsed": false
+ },
+ "id": "d0863018bf228f68"
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "outputs": [],
+ "source": [
+ "openml.config.start_using_configuration_for_example()"
+ ],
+ "metadata": {
+ "collapsed": false
+ },
+ "id": "690279c9d3c1b9db"
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "when using the main server instead, make sure your apikey is configured.\n",
+ "this can be done with the following line of code (uncomment it!). Never share your apikey with others."
+ ],
+ "metadata": {
+ "collapsed": false
+ },
+ "id": "11851df7a3d601db"
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "outputs": [],
+ "source": [
+ "# openml.config.apikey = 'YOURKEY'"
+ ],
+ "metadata": {
+ "collapsed": false
+ },
+ "id": "ce399cc2fd336599"
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "# Caching\n",
+ "when downloading datasets, tasks, runs and flows, they will be cached to\n",
+ "retrieve them without calling the server later. As with the API key,\n",
+ "the cache directory can be either specified through the config file or\n",
+ "through the API:\n",
+ "\n",
+ "* Add the line **cachedir = 'MYDIR'** to the config file, replacing\n",
+ " 'MYDIR' with the path to the cache directory. By default, OpenML\n",
+ " will use **~/.openml/cache** as the cache directory.\n",
+ "* Run the code below, replacing 'YOURDIR' with the path to the cache directory."
+ ],
+ "metadata": {
+ "collapsed": false
+ },
+ "id": "e13705e39146a74d"
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "outputs": [],
+ "source": [
+ "# Uncomment and set your OpenML cache directory\n",
+ "# import os\n",
+ "# openml.config.cache_directory = os.path.expanduser('YOURDIR')"
+ ],
+ "metadata": {
+ "collapsed": false
+ },
+ "id": "bd7c4fe7e60a5551"
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "# Simple Example\n",
+ "download the OpenML task for the eeg-eye-state."
+ ],
+ "metadata": {
+ "collapsed": false
+ },
+ "id": "e55589fdbc0576b9"
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "outputs": [],
+ "source": [
+ "task = openml.tasks.get_task(403)\n",
+ "data = openml.datasets.get_dataset(task.dataset_id)\n",
+ "clf = neighbors.KNeighborsClassifier(n_neighbors=5)\n",
+ "run = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False)\n",
+ "# Publish the experiment on OpenML (optional, requires an API key).\n",
+ "# For this tutorial, our configuration publishes to the test server\n",
+ "# as to not crowd the main server with runs created by examples.\n",
+ "myrun = run.publish()\n",
+ "print(f\"kNN on {data.name}: {myrun.openml_url}\")"
+ ],
+ "metadata": {
+ "collapsed": false
+ },
+ "id": "bf48de543d90a92c"
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "outputs": [],
+ "source": [
+ "openml.config.stop_using_configuration_for_example()"
+ ],
+ "metadata": {
+ "collapsed": false
+ },
+ "id": "e56c29b055269e1a"
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.6"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/mkdocs.yml b/mkdocs.yml
index 4355b5bb2..2d898e81f 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -6,6 +6,7 @@ theme:
nav:
- Code Reference: reference/
+ - Examples: examples/
markdown_extensions:
@@ -17,6 +18,7 @@ markdown_extensions:
alternate_style: true
plugins:
+ - mkdocs-jupyter
- search
- autorefs
- section-index
diff --git a/pyproject.toml b/pyproject.toml
index 7dedcdb01..cfd190f86 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -94,6 +94,7 @@ docs=[
"mkdocs-gen-files",
"mkdocs-literate-nav",
"mkdocs-section-index",
+ "mkdocs-jupyter",
]
[project.urls]
diff --git a/scripts/gen_ref_pages.py b/scripts/gen_ref_pages.py
index 608bb4993..0e477ffa5 100644
--- a/scripts/gen_ref_pages.py
+++ b/scripts/gen_ref_pages.py
@@ -6,6 +6,7 @@
"""
from pathlib import Path
+import shutil
import mkdocs_gen_files
@@ -38,3 +39,17 @@
with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as nav_file:
nav_file.writelines(nav.build_literate_nav())
+
+nav = mkdocs_gen_files.Nav()
+examples_dir = root / "examples"
+examples_doc_dir = root / "docs" / "examples"
+for path in sorted(examples_dir.rglob("*.ipynb")):
+ dest_path = (root / "docs") / path.relative_to(root)
+ with mkdocs_gen_files.open(dest_path, "w") as dest_file:
+ dest_file.write(path.read_text())
+
+ new_relative_location = dest_path.relative_to(examples_doc_dir)
+ nav[new_relative_location.parts] = new_relative_location.as_posix()
+
+ with mkdocs_gen_files.open("examples/SUMMARY.md", "w") as nav_file:
+ nav_file.writelines(nav.build_literate_nav())
From a4efa616d2b3a2a7334b8300015efecad958ff4a Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 29 Jul 2024 15:37:54 +0200
Subject: [PATCH 11/25] make clearer the symmetry
---
scripts/gen_ref_pages.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/scripts/gen_ref_pages.py b/scripts/gen_ref_pages.py
index 0e477ffa5..5d8e4d79d 100644
--- a/scripts/gen_ref_pages.py
+++ b/scripts/gen_ref_pages.py
@@ -44,7 +44,7 @@
examples_dir = root / "examples"
examples_doc_dir = root / "docs" / "examples"
for path in sorted(examples_dir.rglob("*.ipynb")):
- dest_path = (root / "docs") / path.relative_to(root)
+ dest_path = examples_doc_dir / path.relative_to(examples_dir)
with mkdocs_gen_files.open(dest_path, "w") as dest_file:
dest_file.write(path.read_text())
From 2bc16b833470936a01237a5ba44f8ca75f0b9c3b Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 29 Jul 2024 16:14:53 +0200
Subject: [PATCH 12/25] Put notebooks as virtual files in doc, reorder plugins
order plugin is important: first gen_files needs to generate
the virtual files for the jupyter extension to render them
---
examples/20_basic/introduction_tutorial.py | 44 +++++++++++-----------
mkdocs.yml | 2 +-
scripts/gen_ref_pages.py | 10 ++---
3 files changed, 28 insertions(+), 28 deletions(-)
diff --git a/examples/20_basic/introduction_tutorial.py b/examples/20_basic/introduction_tutorial.py
index 26d3143dd..ab505b046 100644
--- a/examples/20_basic/introduction_tutorial.py
+++ b/examples/20_basic/introduction_tutorial.py
@@ -1,12 +1,10 @@
-"""
-Introduction tutorial & Setup
-=============================
+# %% [markdown]
+# # Introduction tutorial & Setup
+# An example how to set up OpenML-Python followed up by a simple example.
-An example how to set up OpenML-Python followed up by a simple example.
-"""
-############################################################################
+# %% [markdown]
# OpenML is an online collaboration platform for machine learning which allows
-# you to:
+# you to: Pueter
#
# * Find or share interesting, well-documented datasets
# * Define research / modelling goals (tasks)
@@ -16,9 +14,8 @@
# * Large scale benchmarking, compare to state of the art
#
-############################################################################
-# Installation
-# ^^^^^^^^^^^^
+# %% [markdown]
+# # Installation
# Installation is done via ``pip``:
#
# .. code:: bash
@@ -29,7 +26,7 @@
# :ref:`installation`.
#
-############################################################################
+# %% [markdown]
# Authentication
# ^^^^^^^^^^^^^^
#
@@ -55,28 +52,29 @@
# you authenticate for the duration of the python process.
-############################################################################
-
+# %%
# License: BSD 3-Clause
import openml
from sklearn import neighbors
-############################################################################
+# %% [markdown]
# .. warning::
# .. include:: ../../test_server_usage_warning.txt
+
+# %%
openml.config.start_using_configuration_for_example()
-############################################################################
+# %% [markdown]
# When using the main server instead, make sure your apikey is configured.
# This can be done with the following line of code (uncomment it!).
# Never share your apikey with others.
+# %%
# openml.config.apikey = 'YOURKEY'
-############################################################################
-# Caching
-# ^^^^^^^
+# %% [markdown]
+# # Caching
# When downloading datasets, tasks, runs and flows, they will be cached to
# retrieve them without calling the server later. As with the API key,
# the cache directory can be either specified through the config file or
@@ -87,14 +85,16 @@
# will use **~/.openml/cache** as the cache directory.
# * Run the code below, replacing 'YOURDIR' with the path to the cache directory.
+# %%
# Uncomment and set your OpenML cache directory
# import os
# openml.config.cache_directory = os.path.expanduser('YOURDIR')
-############################################################################
-# Simple Example
-# ^^^^^^^^^^^^^^
+# %% [markdown]
+# # Simple Example
# Download the OpenML task for the eeg-eye-state.
+
+# %%
task = openml.tasks.get_task(403)
data = openml.datasets.get_dataset(task.dataset_id)
clf = neighbors.KNeighborsClassifier(n_neighbors=5)
@@ -105,5 +105,5 @@
myrun = run.publish()
print(f"kNN on {data.name}: {myrun.openml_url}")
-############################################################################
+# %%
openml.config.stop_using_configuration_for_example()
diff --git a/mkdocs.yml b/mkdocs.yml
index 2d898e81f..f90ff2dc5 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -18,7 +18,6 @@ markdown_extensions:
alternate_style: true
plugins:
- - mkdocs-jupyter
- search
- autorefs
- section-index
@@ -32,3 +31,4 @@ plugins:
- scripts/gen_ref_pages.py
- literate-nav:
nav_file: SUMMARY.md
+ - mkdocs-jupyter
diff --git a/scripts/gen_ref_pages.py b/scripts/gen_ref_pages.py
index 5d8e4d79d..730a98024 100644
--- a/scripts/gen_ref_pages.py
+++ b/scripts/gen_ref_pages.py
@@ -43,13 +43,13 @@
nav = mkdocs_gen_files.Nav()
examples_dir = root / "examples"
examples_doc_dir = root / "docs" / "examples"
-for path in sorted(examples_dir.rglob("*.ipynb")):
- dest_path = examples_doc_dir / path.relative_to(examples_dir)
+for path in sorted(examples_dir.rglob("*.py")):
+ dest_path = Path("examples") / path.relative_to(examples_dir)
with mkdocs_gen_files.open(dest_path, "w") as dest_file:
- dest_file.write(path.read_text())
+ print(path.read_text(), file=dest_file)
- new_relative_location = dest_path.relative_to(examples_doc_dir)
- nav[new_relative_location.parts] = new_relative_location.as_posix()
+ new_relative_location = Path("../") / dest_path
+ nav[new_relative_location.parts[2:]] = new_relative_location.as_posix()
with mkdocs_gen_files.open("examples/SUMMARY.md", "w") as nav_file:
nav_file.writelines(nav.build_literate_nav())
From 62c40d9e6e43c498c7bdc5c0a5363fcaa0265867 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Fri, 2 Aug 2024 19:55:37 +0200
Subject: [PATCH 13/25] Start converting to jupytext
---
examples/20_basic/simple_datasets_tutorial.py | 38 +++++++++----------
.../simple_flows_and_runs_tutorial.py | 6 +++
examples/20_basic/simple_suites_tutorial.py | 6 +++
examples/30_extended/configure_logging.py | 4 ++
.../30_extended/create_upload_tutorial.py | 16 +++++++-
examples/30_extended/custom_flow_.py | 9 +++++
examples/30_extended/datasets_tutorial.py | 11 ++++++
.../30_extended/fetch_evaluations_tutorial.py | 7 ++++
.../30_extended/fetch_runtimes_tutorial.py | 22 ++++++++++-
examples/30_extended/flow_id_tutorial.py | 8 ++++
.../30_extended/flows_and_runs_tutorial.py | 12 ++++++
.../plot_svm_hyperparameters_tutorial.py | 6 +++
examples/30_extended/run_setup_tutorial.py | 6 +++
examples/30_extended/study_tutorial.py | 9 +++++
examples/30_extended/suites_tutorial.py | 11 +++++-
.../task_manual_iteration_tutorial.py | 12 ++++++
examples/30_extended/tasks_tutorial.py | 14 +++++++
.../40_paper/2015_neurips_feurer_example.py | 4 ++
examples/40_paper/2018_ida_strang_example.py | 5 +++
examples/40_paper/2018_kdd_rijn_example.py | 3 ++
.../40_paper/2018_neurips_perrone_example.py | 5 +++
21 files changed, 190 insertions(+), 24 deletions(-)
diff --git a/examples/20_basic/simple_datasets_tutorial.py b/examples/20_basic/simple_datasets_tutorial.py
index c525a3ef9..18493f684 100644
--- a/examples/20_basic/simple_datasets_tutorial.py
+++ b/examples/20_basic/simple_datasets_tutorial.py
@@ -1,31 +1,28 @@
-"""
-========
-Datasets
-========
-
-A basic tutorial on how to list, load and visualize datasets.
-"""
-############################################################################
+# %% [markdown]
+# # Datasets
+# A basic tutorial on how to list, load and visualize datasets.
+#
# In general, we recommend working with tasks, so that the results can
# be easily reproduced. Furthermore, the results can be compared to existing results
# at OpenML. However, for the purposes of this tutorial, we are going to work with
# the datasets directly.
+# %%
# License: BSD 3-Clause
import openml
-############################################################################
-# List datasets
-# =============
+# %% [markdown]
+# ## List datasets
+# %%
datasets_df = openml.datasets.list_datasets(output_format="dataframe")
print(datasets_df.head(n=10))
-############################################################################
-# Download a dataset
-# ==================
+# %% [markdown]
+# ## Download a dataset
+# %%
# Iris dataset https://www.openml.org/d/61
dataset = openml.datasets.get_dataset(61)
@@ -37,23 +34,24 @@
print(f"URL: {dataset.url}")
print(dataset.description[:500])
-############################################################################
-# Load a dataset
-# ==============
-
+# %% [markdown]
+# ## Load a dataset
# X - An array/dataframe where each row represents one example with
# the corresponding feature values.
# y - the classes for each example
# categorical_indicator - an array that indicates which feature is categorical
# attribute_names - the names of the features for the examples (X) and
# target feature (y)
+
+# %%
X, y, categorical_indicator, attribute_names = dataset.get_data(
dataset_format="dataframe", target=dataset.default_target_attribute
)
-############################################################################
+
+# %% [markdown]
# Visualize the dataset
-# =====================
+# %%
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
diff --git a/examples/20_basic/simple_flows_and_runs_tutorial.py b/examples/20_basic/simple_flows_and_runs_tutorial.py
index 0176328b6..ca5cd4412 100644
--- a/examples/20_basic/simple_flows_and_runs_tutorial.py
+++ b/examples/20_basic/simple_flows_and_runs_tutorial.py
@@ -1,4 +1,5 @@
"""
+# %% [markdown]
Flows and Runs
==============
@@ -12,11 +13,13 @@
############################################################################
+# %% [markdown]
# .. warning::
# .. include:: ../../test_server_usage_warning.txt
openml.config.start_using_configuration_for_example()
############################################################################
+# %% [markdown]
# Train a machine learning model
# ==============================
@@ -29,6 +32,7 @@
clf.fit(X, y)
############################################################################
+# %% [markdown]
# Running a model on a task
# =========================
@@ -38,6 +42,7 @@
print(run)
############################################################################
+# %% [markdown]
# Publishing the run
# ==================
@@ -46,4 +51,5 @@
print(f"The flow can be found at {myrun.flow.openml_url}")
############################################################################
+# %% [markdown]
openml.config.stop_using_configuration_for_example()
diff --git a/examples/20_basic/simple_suites_tutorial.py b/examples/20_basic/simple_suites_tutorial.py
index 92dfb3c04..10e834dcc 100644
--- a/examples/20_basic/simple_suites_tutorial.py
+++ b/examples/20_basic/simple_suites_tutorial.py
@@ -1,4 +1,5 @@
"""
+# %% [markdown]
================
Benchmark suites
================
@@ -14,6 +15,7 @@
import openml
####################################################################################################
+# %% [markdown]
# OpenML-CC18
# ===========
#
@@ -36,6 +38,7 @@
# In this example we'll focus on how to use benchmark suites in practice.
####################################################################################################
+# %% [markdown]
# Downloading benchmark suites
# ============================
@@ -43,6 +46,7 @@
print(suite)
####################################################################################################
+# %% [markdown]
# The benchmark suite does not download the included tasks and datasets itself, but only contains
# a list of which tasks constitute the study.
#
@@ -52,6 +56,7 @@
print(tasks)
####################################################################################################
+# %% [markdown]
# and iterated over for benchmarking. For speed reasons we only iterate over the first three tasks:
for task_id in tasks[:3]:
@@ -59,6 +64,7 @@
print(task)
####################################################################################################
+# %% [markdown]
# Further examples
# ================
#
diff --git a/examples/30_extended/configure_logging.py b/examples/30_extended/configure_logging.py
index 3d33f1546..c13140a95 100644
--- a/examples/30_extended/configure_logging.py
+++ b/examples/30_extended/configure_logging.py
@@ -1,4 +1,5 @@
"""
+# %% [markdown]
========
Logging
========
@@ -6,6 +7,7 @@
Explains openml-python logging, and shows how to configure it.
"""
##################################################################################
+# %% [markdown]
# Openml-python uses the `Python logging module <https://docs.python.org/3/library/logging.html>`_
# to provide users with log messages. Each log message is assigned a level of importance, see
# the table in Python's logging tutorial
@@ -26,6 +28,7 @@
openml.datasets.get_dataset("iris")
+# %% [markdown]
# With default configuration, the above example will show no output to console.
# However, in your cache directory you should find a file named 'openml_python.log',
# which has a DEBUG message written to it. It should be either like
@@ -41,6 +44,7 @@
openml.config.set_file_log_level(logging.WARNING)
openml.datasets.get_dataset("iris")
+# %% [markdown]
# Now the log level that was previously written to file should also be shown in the console.
# The message is now no longer written to file as the `file_log` was set to level `WARNING`.
#
diff --git a/examples/30_extended/create_upload_tutorial.py b/examples/30_extended/create_upload_tutorial.py
index 7825d8cf7..95ae4d468 100644
--- a/examples/30_extended/create_upload_tutorial.py
+++ b/examples/30_extended/create_upload_tutorial.py
@@ -1,4 +1,5 @@
"""
+# %% [markdown]
Dataset upload tutorial
=======================
@@ -15,14 +16,17 @@
import openml
from openml.datasets.functions import create_dataset
-############################################################################
+#
+# %% [markdown]
+##########################################################################
# .. warning::
# .. include:: ../../test_server_usage_warning.txt
openml.config.start_using_configuration_for_example()
############################################################################
-############################################################################
+##############################################################################
+# %% [markdown]
# Below we will cover the following cases of the dataset object:
#
# * A numpy array
@@ -32,6 +36,7 @@
# * A pandas sparse dataframe
############################################################################
+# %% [markdown]
# Dataset is a numpy array
# ========================
# A numpy array can contain lists in the case of dense data or it can contain
@@ -50,6 +55,7 @@
description = diabetes.DESCR
############################################################################
+# %% [markdown]
# OpenML does not distinguish between the attributes and targets on the data
# level and stores all data in a single matrix.
#
@@ -69,6 +75,7 @@
paper_url = "https://web.stanford.edu/~hastie/Papers/LARS/LeastAngle_2002.pdf"
############################################################################
+# %% [markdown]
# Create the dataset object
# ^^^^^^^^^^^^^^^^^^^^^^^^^
# The definition of all fields can be found in the XSD files describing the
@@ -119,6 +126,7 @@
print(f"URL for dataset: {diabetes_dataset.openml_url}")
############################################################################
+# %% [markdown]
# Dataset is a list
# =================
# A list can contain lists in the case of dense data or it can contain
@@ -192,6 +200,7 @@
print(f"URL for dataset: {weather_dataset.openml_url}")
############################################################################
+# %% [markdown]
# Dataset is a pandas DataFrame
# =============================
# It might happen that your dataset is made of heterogeneous data which can usually
@@ -210,6 +219,7 @@
print(df.info())
############################################################################
+# %% [markdown]
# We enforce the column 'outlook' and 'play' to be a categorical
# dtype while the column 'windy' is kept as a boolean column. 'temperature'
# and 'humidity' are kept as numeric columns. Then, we can
@@ -239,6 +249,7 @@
print(f"URL for dataset: {weather_dataset.openml_url}")
############################################################################
+# %% [markdown]
# Dataset is a sparse matrix
# ==========================
@@ -276,6 +287,7 @@
############################################################################
+# %% [markdown]
# Dataset is a pandas dataframe with sparse columns
# =================================================
diff --git a/examples/30_extended/custom_flow_.py b/examples/30_extended/custom_flow_.py
index 241f3e6eb..d48efcd09 100644
--- a/examples/30_extended/custom_flow_.py
+++ b/examples/30_extended/custom_flow_.py
@@ -1,4 +1,5 @@
"""
+# %% [markdown]
================================
Creating and Using a Custom Flow
================================
@@ -23,11 +24,13 @@
from openml.runs.functions import format_prediction
####################################################################################################
+# %% [markdown]
# .. warning::
# .. include:: ../../test_server_usage_warning.txt
openml.config.start_using_configuration_for_example()
####################################################################################################
+# %% [markdown]
# 1. Defining the flow
# ====================
# The first step is to define all the hyperparameters of your flow.
@@ -56,6 +59,7 @@
)
####################################################################################################
+# %% [markdown]
# Next we define the flow hyperparameters. We define their name and default value in `parameters`,
# and provide meta-data for each hyperparameter through `parameters_meta_info`.
# Note that even though the argument name is `parameters` they describe the hyperparameters.
@@ -71,6 +75,7 @@
)
####################################################################################################
+# %% [markdown]
# It is possible to build a flow which uses other flows.
# For example, the Random Forest Classifier is a flow, but you could also construct a flow
# which uses a Random Forest Classifier in a ML pipeline. When constructing the pipeline flow,
@@ -94,6 +99,7 @@
)
####################################################################################################
+# %% [markdown]
# With all parameters of the flow defined, we can now initialize the OpenMLFlow and publish.
# Because we provided all the details already, we do not need to provide a `model` to the flow.
#
@@ -113,6 +119,7 @@
print(f"autosklearn flow created: {autosklearn_amlb_flow.flow_id}")
####################################################################################################
+# %% [markdown]
# 2. Using the flow
# ====================
# This Section will show how to upload run data for your custom flow.
@@ -134,6 +141,7 @@
####################################################################################################
+# %% [markdown]
# The last bit of information for the run we need are the predicted values.
# The exact format of the predictions will depend on the task.
#
@@ -194,6 +202,7 @@
predictions.append(prediction)
####################################################################################################
+# %% [markdown]
# Finally we can create the OpenMLRun object and upload.
# We use the argument setup_string because the used flow was a script.
diff --git a/examples/30_extended/datasets_tutorial.py b/examples/30_extended/datasets_tutorial.py
index 764cb8f36..64f3d5d0c 100644
--- a/examples/30_extended/datasets_tutorial.py
+++ b/examples/30_extended/datasets_tutorial.py
@@ -1,4 +1,5 @@
"""
+# %% [markdown]
========
Datasets
========
@@ -13,6 +14,7 @@
from openml.datasets import edit_dataset, fork_dataset, get_dataset
############################################################################
+# %% [markdown]
# Exercise 0
# **********
#
@@ -34,6 +36,7 @@
openml_df.head(n=10)
############################################################################
+# %% [markdown]
# Exercise 1
# **********
#
@@ -47,6 +50,7 @@
datalist.query("NumberOfClasses > 50")
############################################################################
+# %% [markdown]
# Download datasets
# =================
@@ -62,6 +66,7 @@
print(dataset.description[:500])
############################################################################
+# %% [markdown]
# Get the actual data.
#
# openml-python returns data as pandas dataframes (stored in the `eeg` variable below),
@@ -69,6 +74,7 @@
eeg, *_ = dataset.get_data()
############################################################################
+# %% [markdown]
# You can optionally choose to have openml separate out a column from the
# dataset. In particular, many datasets for supervised problems have a set
# `default_target_attribute` which may help identify the target variable.
@@ -79,6 +85,7 @@
print(X.info())
############################################################################
+# %% [markdown]
# Sometimes you only need access to a dataset's metadata.
# In those cases, you can download the dataset without downloading the
# data file. The dataset object can be used as normal.
@@ -90,6 +97,7 @@
dataset = openml.datasets.get_dataset(1471, download_data=False)
############################################################################
+# %% [markdown]
# Exercise 2
# **********
# * Explore the data visually.
@@ -106,6 +114,7 @@
############################################################################
+# %% [markdown]
# Edit a created dataset
# ======================
# This example uses the test server, to avoid editing a dataset on the main server.
@@ -136,6 +145,7 @@
############################################################################
+# %% [markdown]
# Editing critical fields (default_target_attribute, row_id_attribute, ignore_attribute) is allowed
# only for the dataset owner. Further, critical fields cannot be edited if the dataset has any
# tasks associated with it. To edit critical fields of a dataset (without tasks) owned by you,
@@ -148,6 +158,7 @@
print(e)
############################################################################
+# %% [markdown]
# Fork dataset
# ============
# Used to create a copy of the dataset with you as the owner.
diff --git a/examples/30_extended/fetch_evaluations_tutorial.py b/examples/30_extended/fetch_evaluations_tutorial.py
index 86302e2d1..215fbef23 100644
--- a/examples/30_extended/fetch_evaluations_tutorial.py
+++ b/examples/30_extended/fetch_evaluations_tutorial.py
@@ -1,4 +1,5 @@
"""
+# %% [markdown]
====================
Fetching Evaluations
====================
@@ -26,6 +27,7 @@
import openml
############################################################################
+# %% [markdown]
# Listing evaluations
# *******************
# Evaluations can be retrieved from the database in the chosen output format.
@@ -45,6 +47,7 @@
print(evals[evals.value > 0.98])
#############################################################################
+# %% [markdown]
# Viewing a sample task
# =====================
# Over here we shall briefly take a look at the details of the task.
@@ -55,6 +58,7 @@
print(task)
#############################################################################
+# %% [markdown]
# Obtaining all the evaluations for the task
# ==========================================
# We'll now obtain all the evaluations that were uploaded for the task
@@ -73,6 +77,7 @@
print(evals.head())
#############################################################################
+# %% [markdown]
# Obtaining CDF of metric for chosen task
# ***************************************
# We shall now analyse how the performance of various flows have been on this task,
@@ -104,6 +109,7 @@ def plot_cdf(values, metric="predictive_accuracy"):
# with non-zero probability. While the maximum accuracy seen till now is 96.5%.
#############################################################################
+# %% [markdown]
# Comparing top 10 performing flows
# *********************************
# Let us now try to see which flows generally performed the best for this task.
@@ -155,6 +161,7 @@ def plot_flow_compare(evaluations, top_n=10, metric="predictive_accuracy"):
print((flow_ids[i], flow_names[i]))
#############################################################################
+# %% [markdown]
# Obtaining evaluations with hyperparameter settings
# ==================================================
# We'll now obtain the evaluations of a task and a flow with the hyperparameters
diff --git a/examples/30_extended/fetch_runtimes_tutorial.py b/examples/30_extended/fetch_runtimes_tutorial.py
index 107adee79..39995e25f 100644
--- a/examples/30_extended/fetch_runtimes_tutorial.py
+++ b/examples/30_extended/fetch_runtimes_tutorial.py
@@ -1,4 +1,5 @@
"""
+# %% [markdown]
==========================================
Measuring runtimes for Scikit-learn models
@@ -60,6 +61,7 @@
############################################################################
+# %% [markdown]
# Preparing tasks and scikit-learn models
# ***************************************
@@ -92,6 +94,7 @@ def print_compare_runtimes(measures):
############################################################################
+# %% [markdown]
# Case 1: Running a Random Forest model on an OpenML task
# *******************************************************
# We'll run a Random Forest model and obtain an OpenML run object. We can
@@ -123,6 +126,7 @@ def print_compare_runtimes(measures):
print()
################################################################################
+# %% [markdown]
# The remaining entries recorded in `measures` are the runtime records
# related as:
#
@@ -141,6 +145,7 @@ def print_compare_runtimes(measures):
print_compare_runtimes(measures)
######################################################################
+# %% [markdown]
# Case 2: Running Scikit-learn model on an OpenML task in parallel
# ****************************************************************
# Redefining the model to allow parallelism with `n_jobs=2` (2 cores)
@@ -155,6 +160,7 @@ def print_compare_runtimes(measures):
print_compare_runtimes(measures)
####################################################################################
+# %% [markdown]
# Running a Random Forest model on an OpenML task in parallel (all cores available):
# Redefining the model to use all available cores with `n_jobs=-1`
@@ -170,6 +176,7 @@ def print_compare_runtimes(measures):
print_compare_runtimes(measures)
####################################################################################
+# %% [markdown]
# We can now observe that the ratio of CPU time to wallclock time is lower
# than in case 1. This happens because joblib by default spawns subprocesses
# for the workloads for which CPU time cannot be tracked. Therefore, interpreting
@@ -177,6 +184,7 @@ def print_compare_runtimes(measures):
# applied at runtime.
####################################################################################
+# %% [markdown]
# Running the same task with a different parallel backend. Joblib provides multiple
# backends: {`loky` (default), `multiprocessing`, `dask`, `threading`, `sequential`}.
# The backend can be explicitly set using a joblib context manager. The behaviour of
@@ -190,6 +198,7 @@ def print_compare_runtimes(measures):
print_compare_runtimes(measures)
####################################################################################
+# %% [markdown]
# The CPU time interpretation becomes ambiguous when jobs are distributed over an
# unknown number of cores or when subprocesses are spawned for which the CPU time
# cannot be tracked, as in the examples above. It is impossible for OpenML-Python
@@ -199,6 +208,7 @@ def print_compare_runtimes(measures):
# the final interpretation of the runtimes is left to the `user`.
#####################################################################
+# %% [markdown]
# Case 3: Running and benchmarking HPO algorithms with their runtimes
# *******************************************************************
# We shall now optimize a similar RandomForest model for the same task using
@@ -229,6 +239,7 @@ def print_compare_runtimes(measures):
print_compare_runtimes(measures)
##################################################################################
+# %% [markdown]
# Like any optimisation problem, scikit-learn's HPO estimators also generate
# a sequence of configurations which are evaluated, using which the best found
# configuration is tracked throughout the trace.
@@ -248,6 +259,7 @@ def print_compare_runtimes(measures):
print(run4.fold_evaluations["wall_clock_time_millis_training"][0][0])
##################################################################################
+# %% [markdown]
# To extract the training runtime of the 1-st repeat, 4-th (outer) fold and also
# to fetch the parameters and performance of the evaluations made during
# the 1-st repeat, 4-th fold evaluation by the Grid Search model.
@@ -269,6 +281,7 @@ def print_compare_runtimes(measures):
)
##################################################################################
+# %% [markdown]
# Scikit-learn's HPO estimators also come with an argument `refit=True` as a default.
# In our previous model definition it was set to True by default, which meant that the best
# found hyperparameter configuration was used to refit or retrain the model without any inner
@@ -300,7 +313,8 @@ def extract_refit_time(run, repeat, fold):
)
)
-############################################################################
+#########################################################
+# %% [markdown]
# Along with the GridSearchCV already used above, we demonstrate how such
# optimisation traces can be retrieved by showing an application of these
# traces - comparing the speed of finding the best configuration using
@@ -321,6 +335,7 @@ def extract_refit_time(run, repeat, fold):
)
################################################################################
+# %% [markdown]
# Since for the call to ``openml.runs.run_model_on_task`` the parameter
# ``n_jobs`` is set to its default ``None``, the evaluations across the OpenML folds
# are not parallelized. Hence, the time recorded is agnostic to the ``n_jobs``
@@ -377,6 +392,7 @@ def get_incumbent_trace(trace):
plt.show()
################################################################################
+# %% [markdown]
# Case 4: Running models that scikit-learn doesn't parallelize
# *************************************************************
# Both scikit-learn and OpenML depend on parallelism implemented through `joblib`.
@@ -395,6 +411,7 @@ def get_incumbent_trace(trace):
print_compare_runtimes(measures)
################################################################################
+# %% [markdown]
# Although the decision tree does not run in parallel, it can release the
# `Python GIL <https://docs.python.org/dev/glossary.html#term-global-interpreter-lock>`_.
# This can result in surprising runtime measures as demonstrated below:
@@ -407,6 +424,7 @@ def get_incumbent_trace(trace):
print_compare_runtimes(measures)
################################################################################
+# %% [markdown]
# Running a Neural Network from scikit-learn that uses scikit-learn independent
# parallelism using libraries such as `MKL, OpenBLAS or BLIS
# <https://scikit-learn.org/stable/computing/parallelism.html#parallel-numpy-and-scipy-routines-from-numerical-libraries>`_.
@@ -420,6 +438,7 @@ def get_incumbent_trace(trace):
print_compare_runtimes(measures)
################################################################################
+# %% [markdown]
# Case 5: Running Scikit-learn models that don't release GIL
# **********************************************************
# Certain Scikit-learn models do not release the `Python GIL
@@ -438,6 +457,7 @@ def get_incumbent_trace(trace):
print_compare_runtimes(measures)
################################################################################
+# %% [markdown]
# Summmary
# *********
# The scikit-learn extension for OpenML-Python records model runtimes for the
diff --git a/examples/30_extended/flow_id_tutorial.py b/examples/30_extended/flow_id_tutorial.py
index 137f8d14e..5abab7704 100644
--- a/examples/30_extended/flow_id_tutorial.py
+++ b/examples/30_extended/flow_id_tutorial.py
@@ -1,4 +1,5 @@
"""
+# %% [markdown]
==================
Obtaining Flow IDs
==================
@@ -17,6 +18,7 @@
############################################################################
+# %% [markdown]
# .. warning::
# .. include:: ../../test_server_usage_warning.txt
openml.config.start_using_configuration_for_example()
@@ -27,6 +29,7 @@
clf = sklearn.tree.DecisionTreeClassifier()
####################################################################################################
+# %% [markdown]
# 1. Obtaining a flow given a classifier
# ======================================
#
@@ -36,6 +39,7 @@
print(flow_id)
####################################################################################################
+# %% [markdown]
# This piece of code is rather involved. First, it retrieves a
# :class:`~openml.extensions.Extension` which is registered and can handle the given model,
# in our case it is :class:`openml.extensions.sklearn.SklearnExtension`. Second, the extension
@@ -50,6 +54,7 @@
print(flow_id)
####################################################################################################
+# %% [markdown]
# 2. Obtaining a flow given its name
# ==================================
# The schema of a flow is given in XSD (`here
@@ -63,6 +68,7 @@
print(flow.name, flow.external_version)
####################################################################################################
+# %% [markdown]
# The name and external version are automatically added to a flow when constructing it from a
# model. We can then use them to retrieve the flow id as follows:
@@ -70,11 +76,13 @@
print(flow_id)
####################################################################################################
+# %% [markdown]
# We can also retrieve all flows for a given name:
flow_ids = openml.flows.get_flow_id(name=flow.name)
print(flow_ids)
####################################################################################################
+# %% [markdown]
# This also works with the actual model (generalizing the first part of this example):
flow_ids = openml.flows.get_flow_id(model=clf, exact_version=False)
print(flow_ids)
diff --git a/examples/30_extended/flows_and_runs_tutorial.py b/examples/30_extended/flows_and_runs_tutorial.py
index 38b0d23cf..2b23196ae 100644
--- a/examples/30_extended/flows_and_runs_tutorial.py
+++ b/examples/30_extended/flows_and_runs_tutorial.py
@@ -1,4 +1,5 @@
"""
+# %% [markdown]
Flows and Runs
==============
@@ -12,6 +13,7 @@
############################################################################
+# %% [markdown]
# We'll use the test server for the rest of this tutorial.
#
# .. warning::
@@ -19,6 +21,7 @@
openml.config.start_using_configuration_for_example()
############################################################################
+# %% [markdown]
# Train machine learning models
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
#
@@ -33,6 +36,7 @@
clf.fit(X, y)
############################################################################
+# %% [markdown]
# You can also ask for meta-data to automatically preprocess the data.
#
# * e.g. categorical features -> do feature encoding
@@ -48,6 +52,7 @@
clf.fit(X, y)
############################################################################
+# %% [markdown]
# Runs: Easily explore models
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^
# We can run (many) scikit-learn algorithms on (many) OpenML tasks.
@@ -64,6 +69,7 @@
print(run)
############################################################################
+# %% [markdown]
# Share the run on the OpenML server
#
# So far the run is only available locally. By calling the publish function,
@@ -75,12 +81,14 @@
print(f"Uploaded to {myrun.openml_url}")
############################################################################
+# %% [markdown]
# We can now also inspect the flow object which was automatically created:
flow = openml.flows.get_flow(run.flow_id)
print(flow)
############################################################################
+# %% [markdown]
# It also works with pipelines
# ############################
#
@@ -121,6 +129,7 @@
print(f"Uploaded to {myrun.openml_url}")
+# %% [markdown]
# The above pipeline works with the helper functions that internally deal with pandas DataFrame.
# In the case, pandas is not available, or a NumPy based data processing is the requirement, the
# above pipeline is presented below to work with NumPy.
@@ -165,6 +174,7 @@
print(f"Uploaded to {myrun.openml_url}")
###############################################################################
+# %% [markdown]
# Running flows on tasks offline for later upload
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# For those scenarios where there is no access to internet, it is possible to run
@@ -193,6 +203,7 @@
# it does not yet exist on the server.
############################################################################
+# %% [markdown]
# Alternatively, one can also directly run flows.
# Get a task
@@ -209,6 +220,7 @@
run = openml.runs.run_flow_on_task(flow, task)
############################################################################
+# %% [markdown]
# Challenge
# ^^^^^^^^^
#
diff --git a/examples/30_extended/plot_svm_hyperparameters_tutorial.py b/examples/30_extended/plot_svm_hyperparameters_tutorial.py
index e366c56df..96e13ed70 100644
--- a/examples/30_extended/plot_svm_hyperparameters_tutorial.py
+++ b/examples/30_extended/plot_svm_hyperparameters_tutorial.py
@@ -1,4 +1,5 @@
"""
+# %% [markdown]
================================
Plotting hyperparameter surfaces
================================
@@ -10,6 +11,7 @@
import numpy as np
####################################################################################################
+# %% [markdown]
# First step - obtaining the data
# ===============================
# First, we need to choose an SVM flow, for example 8353, and a task. Finding the IDs of them are
@@ -30,17 +32,20 @@
print(df.head(n=10))
####################################################################################################
+# %% [markdown]
# We can see all the hyperparameter names in the columns of the dataframe:
for name in df.columns:
print(name)
####################################################################################################
+# %% [markdown]
# Next, we cast and transform the hyperparameters of interest (``C`` and ``gamma``) so that we
# can nicely plot them.
hyperparameters = ["sklearn.svm.classes.SVC(16)_C", "sklearn.svm.classes.SVC(16)_gamma"]
df[hyperparameters] = df[hyperparameters].astype(float).apply(np.log10)
####################################################################################################
+# %% [markdown]
# Option 1 - plotting via the pandas helper functions
# ===================================================
#
@@ -54,6 +59,7 @@
)
####################################################################################################
+# %% [markdown]
# Option 2 - plotting via matplotlib
# ==================================
#
diff --git a/examples/30_extended/run_setup_tutorial.py b/examples/30_extended/run_setup_tutorial.py
index a2bc3a4df..087be849f 100644
--- a/examples/30_extended/run_setup_tutorial.py
+++ b/examples/30_extended/run_setup_tutorial.py
@@ -1,4 +1,5 @@
"""
+# %% [markdown]
=========
Run Setup
=========
@@ -40,11 +41,13 @@
from sklearn.decomposition import TruncatedSVD
############################################################################
+# %% [markdown]
# .. warning::
# .. include:: ../../test_server_usage_warning.txt
openml.config.start_using_configuration_for_example()
###############################################################################
+# %% [markdown]
# 1) Create a flow and use it to solve a task
###############################################################################
@@ -70,6 +73,7 @@
]
)
+# %% [markdown]
# Let's change some hyperparameters. Of course, in any good application we
# would tune them using, e.g., Random Search or Bayesian Optimization, but for
# the purpose of this tutorial we set them to some specific values that might
@@ -87,6 +91,7 @@
run_original = run.publish() # this implicitly uploads the flow
###############################################################################
+# %% [markdown]
# 2) Download the flow and solve the same task again.
###############################################################################
@@ -104,6 +109,7 @@
###############################################################################
+# %% [markdown]
# 3) We will verify that the obtained results are exactly the same.
###############################################################################
diff --git a/examples/30_extended/study_tutorial.py b/examples/30_extended/study_tutorial.py
index d5bfcd88a..392eb9abc 100644
--- a/examples/30_extended/study_tutorial.py
+++ b/examples/30_extended/study_tutorial.py
@@ -1,4 +1,5 @@
"""
+# %% [markdown]
=================
Benchmark studies
=================
@@ -19,6 +20,7 @@
############################################################################
+# %% [markdown]
# Listing studies
# ***************
#
@@ -31,23 +33,28 @@
############################################################################
+# %% [markdown]
# Downloading studies
# ===================
############################################################################
+# %% [markdown]
# This is done based on the study ID.
study = openml.study.get_study(123)
print(study)
############################################################################
+# %% [markdown]
# Studies also features a description:
print(study.description)
############################################################################
+# %% [markdown]
# Studies are a container for runs:
print(study.runs)
############################################################################
+# %% [markdown]
# And we can use the evaluation listing functionality to learn more about
# the evaluations available for the conducted runs:
evaluations = openml.evaluations.list_evaluations(
@@ -58,6 +65,7 @@
print(evaluations.head())
############################################################################
+# %% [markdown]
# We'll use the test server for the rest of this tutorial.
#
# .. warning::
@@ -65,6 +73,7 @@
openml.config.start_using_configuration_for_example()
############################################################################
+# %% [markdown]
# Uploading studies
# =================
#
diff --git a/examples/30_extended/suites_tutorial.py b/examples/30_extended/suites_tutorial.py
index ff9902356..1e1786cbb 100644
--- a/examples/30_extended/suites_tutorial.py
+++ b/examples/30_extended/suites_tutorial.py
@@ -1,4 +1,5 @@
"""
+# %% [markdown]
================
Benchmark suites
================
@@ -21,6 +22,7 @@
############################################################################
+# %% [markdown]
# Listing suites
# **************
#
@@ -32,23 +34,28 @@
print(suites.head(n=10))
############################################################################
+# %% [markdown]
# Downloading suites
# ==================
############################################################################
+# %% [markdown]
# This is done based on the dataset ID.
suite = openml.study.get_suite(99)
print(suite)
-############################################################################
+#############################################################################
+# %% [markdown]
# Suites also feature a description:
print(suite.description)
############################################################################
+# %% [markdown]
# Suites are a container for tasks:
print(suite.tasks)
############################################################################
+# %% [markdown]
# And we can use the task listing functionality to learn more about them:
tasks = openml.tasks.list_tasks(output_format="dataframe")
@@ -59,6 +66,7 @@
print(tasks.describe().transpose())
############################################################################
+# %% [markdown]
# We'll use the test server for the rest of this tutorial.
#
# .. warning::
@@ -66,6 +74,7 @@
openml.config.start_using_configuration_for_example()
############################################################################
+# %% [markdown]
# Uploading suites
# ================
#
diff --git a/examples/30_extended/task_manual_iteration_tutorial.py b/examples/30_extended/task_manual_iteration_tutorial.py
index 676a742a1..ae7f9bf01 100644
--- a/examples/30_extended/task_manual_iteration_tutorial.py
+++ b/examples/30_extended/task_manual_iteration_tutorial.py
@@ -1,4 +1,5 @@
"""
+# %% [markdown]
Tasks: retrieving splits
========================
@@ -15,6 +16,7 @@
import openml
####################################################################################################
+# %% [markdown]
# For this tutorial we will use the famous King+Rook versus King+Pawn on A7 dataset, which has
# the dataset ID 3 (`dataset on OpenML <https://www.openml.org/d/3>`_), and for which there exist
# tasks with all important estimation procedures. It is small enough (less than 5000 samples) to
@@ -26,12 +28,14 @@
task = openml.tasks.get_task(task_id)
####################################################################################################
+# %% [markdown]
# Now that we have a task object we can obtain the number of repetitions, folds and samples as
# defined by the task:
n_repeats, n_folds, n_samples = task.get_split_dimensions()
####################################################################################################
+# %% [markdown]
# * ``n_repeats``: Number of times the model quality estimation is performed
# * ``n_folds``: Number of folds per repeat
# * ``n_samples``: How many data points to use. This is only relevant for learning curve tasks
@@ -52,6 +56,7 @@
)
####################################################################################################
+# %% [markdown]
# We can now retrieve the train/test split for this combination of repeats, folds and number of
# samples (indexing is zero-based). Usually, one would loop over all repeats, folds and sample
# sizes, but we can neglect this here as there is only a single repetition.
@@ -66,6 +71,7 @@
print(test_indices.shape, test_indices.dtype)
####################################################################################################
+# %% [markdown]
# And then split the data based on this:
X, y = task.get_X_and_y(dataset_format="dataframe")
@@ -84,6 +90,7 @@
)
####################################################################################################
+# %% [markdown]
# Obviously, we can also retrieve cross-validation versions of the dataset used in task ``233``:
task_id = 3
@@ -100,6 +107,7 @@
)
####################################################################################################
+# %% [markdown]
# And then perform the aforementioned iteration over all splits:
for repeat_idx in range(n_repeats):
for fold_idx in range(n_folds):
@@ -128,6 +136,7 @@
)
####################################################################################################
+# %% [markdown]
# And also versions with multiple repeats:
task_id = 1767
@@ -144,6 +153,7 @@
)
####################################################################################################
+# %% [markdown]
# And then again perform the aforementioned iteration over all splits:
for repeat_idx in range(n_repeats):
for fold_idx in range(n_folds):
@@ -172,6 +182,7 @@
)
####################################################################################################
+# %% [markdown]
# And finally a task based on learning curves:
task_id = 1702
@@ -188,6 +199,7 @@
)
####################################################################################################
+# %% [markdown]
# And then again perform the aforementioned iteration over all splits:
for repeat_idx in range(n_repeats):
for fold_idx in range(n_folds):
diff --git a/examples/30_extended/tasks_tutorial.py b/examples/30_extended/tasks_tutorial.py
index 19a7e542c..42ed313e1 100644
--- a/examples/30_extended/tasks_tutorial.py
+++ b/examples/30_extended/tasks_tutorial.py
@@ -1,4 +1,5 @@
"""
+# %% [markdown]
Tasks
=====
@@ -12,6 +13,7 @@
import pandas as pd
############################################################################
+# %% [markdown]
#
# Tasks are identified by IDs and can be accessed in two different ways:
#
@@ -26,6 +28,7 @@
# splits in a useful manner.
############################################################################
+# %% [markdown]
# Listing tasks
# ^^^^^^^^^^^^^
#
@@ -43,6 +46,7 @@
print(tasks.head())
############################################################################
+# %% [markdown]
# We can filter the list of tasks to only contain datasets with more than
# 500 samples, but less than 1000 samples:
@@ -55,6 +59,8 @@
print(len(filtered_tasks))
############################################################################
+# %% [markdown]
+# %% [markdown]
# Then, we can further restrict the tasks to all have the same resampling strategy:
filtered_tasks = filtered_tasks.query('estimation_procedure == "10-fold Crossvalidation"')
@@ -66,6 +72,7 @@
print(len(filtered_tasks))
############################################################################
+# %% [markdown]
# Resampling strategies can be found on the
# `OpenML Website <https://www.openml.org/search?type=measure&q=estimation%20procedure>`_.
#
@@ -89,6 +96,7 @@
print(tasks)
############################################################################
+# %% [markdown]
#
# **OpenML 100**
# is a curated list of 100 tasks to start using OpenML. They are all
@@ -105,6 +113,7 @@
print(len(tasks))
############################################################################
+# %% [markdown]
# Exercise
# ########
#
@@ -113,6 +122,7 @@
tasks.query('name=="eeg-eye-state"')
############################################################################
+# %% [markdown]
# Downloading tasks
# ^^^^^^^^^^^^^^^^^
#
@@ -136,6 +146,7 @@
print(tasks[0])
############################################################################
+# %% [markdown]
# Creating tasks
# ^^^^^^^^^^^^^^
#
@@ -164,6 +175,7 @@
# scatters results across tasks.
############################################################################
+# %% [markdown]
# We'll use the test server for the rest of this tutorial.
#
# .. warning::
@@ -171,6 +183,7 @@
openml.config.start_using_configuration_for_example()
############################################################################
+# %% [markdown]
# Example
# #######
#
@@ -209,6 +222,7 @@
############################################################################
+# %% [markdown]
# * `Complete list of task types <https://www.openml.org/search?type=task_type>`_.
# * `Complete list of model estimation procedures <https://www.openml.org/search?q=%2520measure_type%3Aestimation_procedure&type=measure>`_.
# * `Complete list of evaluation measures <https://www.openml.org/search?q=measure_type%3Aevaluation_measure&type=measure>`_.
diff --git a/examples/40_paper/2015_neurips_feurer_example.py b/examples/40_paper/2015_neurips_feurer_example.py
index 3960c3852..24ca3832c 100644
--- a/examples/40_paper/2015_neurips_feurer_example.py
+++ b/examples/40_paper/2015_neurips_feurer_example.py
@@ -1,4 +1,5 @@
"""
+# %% [markdown]
Feurer et al. (2015)
====================
@@ -22,6 +23,7 @@
import openml
####################################################################################################
+# %% [markdown]
# List of dataset IDs given in the supplementary material of Feurer et al.:
# https://papers.nips.cc/paper/5872-efficient-and-robust-automated-machine-learning-supplemental.zip
# fmt: off
@@ -40,6 +42,7 @@
# fmt: on
####################################################################################################
+# %% [markdown]
# The dataset IDs could be used directly to load the dataset and split the data into a training set
# and a test set. However, to be reproducible, we will first obtain the respective tasks from
# OpenML, which define both the target feature and the train/test split.
@@ -53,6 +56,7 @@
# want to learn more about them.
####################################################################################################
+# %% [markdown]
# This lists both active and inactive tasks (because of ``status='all'``). Unfortunately,
# this is necessary as some of the datasets contain issues found after the publication and became
# deactivated, which also deactivated the tasks on them. More information on active or inactive
diff --git a/examples/40_paper/2018_ida_strang_example.py b/examples/40_paper/2018_ida_strang_example.py
index 8b225125b..255f461d5 100644
--- a/examples/40_paper/2018_ida_strang_example.py
+++ b/examples/40_paper/2018_ida_strang_example.py
@@ -1,4 +1,5 @@
"""
+# %% [markdown]
Strang et al. (2018)
====================
@@ -21,6 +22,7 @@
import pandas as pd
##############################################################################
+# %% [markdown]
# A basic step for each data-mining or machine learning task is to determine
# which model to choose based on the problem and the data at hand. In this
# work we investigate when non-linear classifiers outperform linear
@@ -67,6 +69,7 @@
##############################################################################
+# %% [markdown]
# makes the s-plot
fig_splot, ax_splot = plt.subplots()
@@ -79,6 +82,7 @@
##############################################################################
+# %% [markdown]
# adds column that indicates the difference between the two classifiers,
# needed for the scatter plot
@@ -110,6 +114,7 @@ def determine_class(val_lin, val_nonlin):
plt.show()
##############################################################################
+# %% [markdown]
# makes a scatter plot where each data point represents the performance of the
# two algorithms on various axis (not in the paper)
diff --git a/examples/40_paper/2018_kdd_rijn_example.py b/examples/40_paper/2018_kdd_rijn_example.py
index d3ce59f35..7733be6a0 100644
--- a/examples/40_paper/2018_kdd_rijn_example.py
+++ b/examples/40_paper/2018_kdd_rijn_example.py
@@ -1,4 +1,5 @@
"""
+# %% [markdown]
van Rijn and Hutter (2018)
==========================
@@ -36,6 +37,7 @@
##############################################################################
+# %% [markdown]
# With the advent of automated machine learning, automated hyperparameter
# optimization methods are by now routinely used in data mining. However, this
# progress is not yet matched by equal progress on automatic analyses that
@@ -160,6 +162,7 @@
fanova_results = pd.DataFrame(fanova_results)
##############################################################################
+# %% [markdown]
# make the boxplot of the variance contribution. Obviously, we can also use
# this data to make the Nemenyi plot, but this relies on the rather complex
# ``Orange`` dependency (``pip install Orange3``). For the complete example,
diff --git a/examples/40_paper/2018_neurips_perrone_example.py b/examples/40_paper/2018_neurips_perrone_example.py
index 0d72846ac..4e7dd136a 100644
--- a/examples/40_paper/2018_neurips_perrone_example.py
+++ b/examples/40_paper/2018_neurips_perrone_example.py
@@ -1,4 +1,5 @@
"""
+# %% [markdown]
Perrone et al. (2018)
=====================
@@ -40,6 +41,7 @@
flow_type = "svm" # this example will use the smaller svm flow evaluations
############################################################################
+# %% [markdown]
# The subsequent functions are defined to fetch tasks, flows, evaluations and preprocess them into
# a tabular format that can be used to build models.
@@ -155,6 +157,7 @@ def list_categorical_attributes(flow_type="svm"):
#############################################################################
+# %% [markdown]
# Fetching the data from OpenML
# *****************************
# Now, we read all the tasks and evaluations for them and collate into a table.
@@ -167,6 +170,7 @@ def list_categorical_attributes(flow_type="svm"):
print("Y : ", y[:5])
#############################################################################
+# %% [markdown]
# Creating pre-processing and modelling pipelines
# ***********************************************
# The two primary tasks are to impute the missing values, that is, account for the hyperparameters
@@ -192,6 +196,7 @@ def list_categorical_attributes(flow_type="svm"):
#############################################################################
+# %% [markdown]
# Building a surrogate model on a task's evaluation
# *************************************************
# The same set of functions can be used for a single task to retrieve a singular table which can
From 7d2901631d43b57531e61542273d88dddc3a1d3e Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Fri, 18 Oct 2024 13:15:48 +0200
Subject: [PATCH 14/25] Add other files to toc
---
docs/api.md | 222 ----------------------------------------------------
mkdocs.yml | 6 +-
2 files changed, 5 insertions(+), 223 deletions(-)
delete mode 100644 docs/api.md
diff --git a/docs/api.md b/docs/api.md
deleted file mode 100644
index 2b9f62374..000000000
--- a/docs/api.md
+++ /dev/null
@@ -1,222 +0,0 @@
-# API
-
-## Modules
-
-### `openml.datasets`
-
-::: openml.datasets.OpenMLDataset
-
-[//]: # (::: {.automodule no-members="" no-inherited-members=""})
-[//]: # (openml.datasets)
-[//]: # (:::)
-
-#### Dataset Classes
-
-[//]: # (::: currentmodule)
-[//]: # (openml.datasets)
-[//]: # (:::)
-[//]: # ()
-[//]: # (::: {.autosummary toctree="generated/" template="class.rst"})
-[//]: # (OpenMLDataFeature OpenMLDataset)
-[//]: # (:::)
-
-#### Dataset Functions
-
-[//]: # (::: currentmodule)
-[//]: # (openml.datasets)
-[//]: # (:::)
-[//]: # ()
-[//]: # (::: {.autosummary toctree="generated/" template="function.rst"})
-[//]: # (attributes_arff_from_df check_datasets_active create_dataset)
-[//]: # (delete_dataset get_dataset get_datasets list_datasets list_qualities)
-[//]: # (status_update edit_dataset fork_dataset)
-[//]: # (:::)
-
-### `openml.evaluations`{.interpreted-text role="mod"}
-
-[//]: # (::: {.automodule no-members="" no-inherited-members=""})
-[//]: # (openml.evaluations)
-[//]: # (:::)
-
-#### Evaluations Classes
-
-[//]: # (::: currentmodule)
-[//]: # (openml.evaluations)
-[//]: # (:::)
-[//]: # ()
-[//]: # (::: {.autosummary toctree="generated/" template="class.rst"})
-[//]: # (OpenMLEvaluation)
-[//]: # (:::)
-[//]: # ()
-#### Evaluations Functions
-
-[//]: # (::: currentmodule)
-[//]: # (openml.evaluations)
-[//]: # (:::)
-[//]: # ()
-[//]: # (::: {.autosummary toctree="generated/" template="function.rst"})
-[//]: # (list_evaluations list_evaluation_measures list_evaluations_setups)
-[//]: # (:::)
-
-### `openml.flows`{.interpreted-text role="mod"}: Flow Functions
-
-[//]: # (::: {.automodule no-members="" no-inherited-members=""})
-[//]: # (openml.flows)
-[//]: # (:::)
-
-#### Flow Classes
-
-[//]: # (::: currentmodule)
-[//]: # (openml.flows)
-[//]: # (:::)
-[//]: # ()
-[//]: # (::: {.autosummary toctree="generated/" template="class.rst"})
-[//]: # (OpenMLFlow)
-[//]: # (:::)
-[//]: # ()
-#### Flow Functions
-
-[//]: # (::: currentmodule)
-[//]: # (openml.flows)
-[//]: # (:::)
-[//]: # ()
-[//]: # (::: {.autosummary toctree="generated/" template="function.rst"})
-[//]: # (assert_flows_equal delete_flow flow_exists get_flow list_flows)
-[//]: # (:::)
-
-### `openml.runs`{.interpreted-text role="mod"}: Run Functions
-
-[//]: # (::: {.automodule no-members="" no-inherited-members=""})
-[//]: # (openml.runs)
-[//]: # (:::)
-
-#### Run Classes
-
-[//]: # (::: currentmodule)
-[//]: # (openml.runs)
-[//]: # (:::)
-[//]: # ()
-[//]: # (::: {.autosummary toctree="generated/" template="class.rst"})
-[//]: # (OpenMLRun)
-[//]: # (:::)
-
-#### Run Functions
-
-[//]: # (::: currentmodule)
-[//]: # (openml.runs)
-[//]: # (:::)
-[//]: # ()
-[//]: # (::: {.autosummary toctree="generated/" template="function.rst"})
-[//]: # (delete_run get_run get_runs get_run_trace initialize_model_from_run)
-[//]: # (initialize_model_from_trace list_runs run_model_on_task run_flow_on_task)
-[//]: # (run_exists)
-[//]: # (:::)
-
-### `openml.setups`{.interpreted-text role="mod"}: Setup Functions
-
-[//]: # (::: {.automodule no-members="" no-inherited-members=""})
-[//]: # (openml.setups)
-[//]: # (:::)
-
-#### Setup Classes
-
-[//]: # (::: currentmodule)
-[//]: # (openml.setups)
-[//]: # (:::)
-[//]: # ()
-[//]: # (::: {.autosummary toctree="generated/" template="class.rst"})
-[//]: # (OpenMLParameter OpenMLSetup)
-[//]: # (:::)
-
-#### Setup Functions
-
-[//]: # (::: currentmodule)
-[//]: # (openml.setups)
-[//]: # (:::)
-[//]: # ()
-[//]: # (::: {.autosummary toctree="generated/" template="function.rst"})
-[//]: # (get_setup initialize_model list_setups setup_exists)
-[//]: # (:::)
-
-### `openml.study`{.interpreted-text role="mod"}: Study Functions
-
-[//]: # (::: {.automodule no-members="" no-inherited-members=""})
-[//]: # (openml.study)
-[//]: # (:::)
-
-#### Study Classes
-
-[//]: # (::: currentmodule)
-[//]: # (openml.study)
-[//]: # (:::)
-[//]: # ()
-[//]: # (::: {.autosummary toctree="generated/" template="class.rst"})
-[//]: # (OpenMLBenchmarkSuite OpenMLStudy)
-[//]: # (:::)
-
-#### Study Functions
-
-[//]: # (::: currentmodule)
-[//]: # (openml.study)
-[//]: # (:::)
-[//]: # ()
-[//]: # (::: {.autosummary toctree="generated/" template="function.rst"})
-[//]: # (attach_to_study attach_to_suite create_benchmark_suite create_study)
-[//]: # (delete_study delete_suite detach_from_study detach_from_suite get_study)
-[//]: # (get_suite list_studies list_suites update_study_status)
-[//]: # (update_suite_status)
-[//]: # (:::)
-[//]: # ()
-### `openml.tasks`{.interpreted-text role="mod"}: Task Functions
-
-[//]: # (::: {.automodule no-members="" no-inherited-members=""})
-[//]: # (openml.tasks)
-[//]: # (:::)
-
-#### Task Classes
-
-[//]: # (::: currentmodule)
-[//]: # (openml.tasks)
-[//]: # (:::)
-
-[//]: # (::: {.autosummary toctree="generated/" template="class.rst"})
-[//]: # (OpenMLClassificationTask OpenMLClusteringTask OpenMLLearningCurveTask)
-[//]: # (OpenMLRegressionTask OpenMLSplit OpenMLSupervisedTask OpenMLTask)
-[//]: # (TaskType)
-[//]: # (:::)
-
-#### Task Functions
-
-[//]: # (::: currentmodule)
-[//]: # (openml.tasks)
-[//]: # (:::)
-[//]: # ()
-[//]: # (::: {.autosummary toctree="generated/" template="function.rst"})
-[//]: # (create_task delete_task get_task get_tasks list_tasks)
-[//]: # (:::)
-
-## Extensions {#api_extensions}
-
-[//]: # (::: {.automodule no-members="" no-inherited-members=""})
-[//]: # (openml.extensions)
-[//]: # (:::)
-
-### Extension Classes
-
-[//]: # (::: currentmodule)
-[//]: # (openml.extensions)
-[//]: # (:::)
-[//]: # ()
-[//]: # (::: {.autosummary toctree="generated/" template="class.rst"})
-[//]: # (Extension sklearn.SklearnExtension)
-[//]: # (:::)
-[//]: # ()
-### Extension Functions
-
-[//]: # (::: currentmodule)
-[//]: # (openml.extensions)
-[//]: # (:::)
-[//]: # ()
-[//]: # (::: {.autosummary toctree="generated/" template="function.rst"})
-[//]: # (get_extension_by_flow get_extension_by_model register_extension)
-[//]: # (:::)
diff --git a/mkdocs.yml b/mkdocs.yml
index f90ff2dc5..58b53ca54 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -5,9 +5,13 @@ theme:
- content.code.copy
nav:
+ - index.md
- Code Reference: reference/
- Examples: examples/
-
+ - Usage: usage.md
+ - Contributing: contributing.md
+ - Extensions: extensions.md
+ - Changelog: progress.md
markdown_extensions:
- pymdownx.highlight:
From 978839d563bd60b9f24fe7e322281780ca8cdfc7 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Fri, 18 Oct 2024 13:19:04 +0200
Subject: [PATCH 15/25] Fix formatting and links
---
docs/contributing.md | 39 +++++++++++++++++++--------------------
1 file changed, 19 insertions(+), 20 deletions(-)
diff --git a/docs/contributing.md b/docs/contributing.md
index 7329e2345..c18de3ccc 100644
--- a/docs/contributing.md
+++ b/docs/contributing.md
@@ -3,23 +3,22 @@
Contribution to the OpenML package is highly appreciated in all forms.
In particular, a few ways to contribute to openml-python are:
-> - A direct contribution to the package, by means of improving the
-> code, documentation or examples. To get started, see [this
-> file](https://github.com/openml/openml-python/blob/main/CONTRIBUTING.md)
-> with details on how to set up your environment to develop for
-> openml-python.
-> - A contribution to an openml-python extension. An extension package
-> allows OpenML to interface with a machine learning package (such
-> as scikit-learn or keras). These extensions are hosted in separate
-> repositories and may have their own guidelines. For more
-> information, see the `extensions`{.interpreted-text role="ref"}
-> below.
-> - Bug reports. If something doesn\'t work for you or is cumbersome,
-> please open a new issue to let us know about the problem. See
-> [this
-> section](https://github.com/openml/openml-python/blob/main/CONTRIBUTING.md).
-> - [Cite OpenML](https://www.openml.org/cite) if you use it in a
-> scientific publication.
-> - Visit one of our [hackathons](https://www.openml.org/meet).
-> - Contribute to another OpenML project, such as [the main OpenML
-> project](https://github.com/openml/OpenML/blob/master/CONTRIBUTING.md).
+- A direct contribution to the package, by means of improving the
+ code, documentation or examples. To get started, see [this
+ file](https://github.com/openml/openml-python/blob/main/CONTRIBUTING.md)
+ with details on how to set up your environment to develop for
+ openml-python.
+- A contribution to an openml-python extension. An extension package
+ allows OpenML to interface with a machine learning package (such
+ as scikit-learn or keras). These extensions are hosted in separate
+ repositories and may have their own guidelines. For more
+ information, see also [extensions](extensions.md).
+- Bug reports. If something doesn't work for you or is cumbersome,
+ please open a new issue to let us know about the problem. See
+ [this
+ section](https://github.com/openml/openml-python/blob/main/CONTRIBUTING.md).
+- [Cite OpenML](https://www.openml.org/cite) if you use it in a
+ scientific publication.
+- Visit one of our [hackathons](https://www.openml.org/meet).
+- Contribute to another OpenML project, such as [the main OpenML
+ project](https://github.com/openml/OpenML/blob/master/CONTRIBUTING.md).
From 142887232d60c90e190da38640da8c0d2aaecaab Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Fri, 18 Oct 2024 16:04:17 +0200
Subject: [PATCH 16/25] Fix formatting
---
docs/usage.md | 128 +++++++++++++++-----------------------------------
1 file changed, 38 insertions(+), 90 deletions(-)
diff --git a/docs/usage.md b/docs/usage.md
index eefbb0532..7c733fedc 100644
--- a/docs/usage.md
+++ b/docs/usage.md
@@ -5,7 +5,6 @@ functions and classes in the OpenML Python API. Throughout this
document, we will use [pandas](https://pandas.pydata.org/) to format and
filter tables.
-[](){#installation-guide}
## Installation
The OpenML Python package is a connector to
@@ -13,12 +12,9 @@ The OpenML Python package is a connector to
datasets and tasks, run machine learning algorithms on them and then
share the results online.
-The following tutorial gives a short introduction on how to install and
+The ["intruduction tutorial and setup"][intro] tutorial gives a short introduction on how to install and
set up the OpenML Python connector, followed up by a simple example.
-- `sphx_glr_examples_20_basic_introduction_tutorial.py`{.interpreted-text
- role="ref"}
-
## Configuration
The configuration file resides in a directory `.config/openml` in the
@@ -29,73 +25,37 @@ Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-
It consists of `key = value` pairs which are separated by newlines. The
following keys are defined:
--
-
- apikey:
-
- : - required to access the server. The
- `sphx_glr_examples_20_basic_introduction_tutorial.py`{.interpreted-text
- role="ref"} describes how to obtain an API key.
-
--
-
- server:
-
- : - default: `http://www.openml.org`. Alternatively, use
- `test.openml.org` for the test server.
-
--
-
- cachedir:
-
- : - if not given, will default to `~/.openml/cache`
-
--
-
- avoid_duplicate_runs:
-
- : - if set to `True`, when `run_flow_on_task` or similar methods
+- apikey: required to access the server. The [introduction tutorial][intro] describes how to obtain an API key.
+- server: the server to connect to (default: `http://www.openml.org`).
+ For connection to the test server, set this to `test.openml.org`.
+- cachedir: the root folder where the cache file directories should be created.
+ If not given, will default to `~/.openml/cache`
+- avoid_duplicate_runs: if set to `True` (default), when `run_flow_on_task` or similar methods
are called a lookup is performed to see if there already
exists such a run on the server. If so, download those
results instead.
- - if not given, will default to `True`.
-
--
-
- retry_policy:
-
- : - Defines how to react when the server is unavailable or
+- retry_policy: Defines how to react when the server is unavailable or
experiencing high load. It determines both how often to
- attempt to reconnect and how quickly to do so. Please don\'t
+ attempt to reconnect and how quickly to do so. Please don't
use `human` in an automated script that you run more than
one instance of, it might increase the time to complete your
- jobs and that of others.
- - human (default): For people running openml in interactive
- fashion. Try only a few times, but in quick succession.
- - robot: For people using openml in an automated fashion. Keep
- trying to reconnect for a longer time, quickly increasing
- the time between retries.
-
--
-
- connection_n_retries:
-
- : - number of connection retries
- - default depends on retry_policy (5 for `human`, 50 for
- `robot`)
-
--
-
- verbosity:
-
- : - 0: normal output
- - 1: info output
- - 2: debug output
+ jobs and that of others. One of:
+ - human (default): For people running openml in interactive
+ fashion. Try only a few times, but in quick succession.
+ - robot: For people using openml in an automated fashion. Keep
+ trying to reconnect for a longer time, quickly increasing
+ the time between retries.
+
+- connection_n_retries: number of times to retry a request if they fail.
+Default depends on retry_policy (5 for `human`, 50 for `robot`)
+- verbosity: the level of output:
+ - 0: normal output
+ - 1: info output
+ - 2: debug output
This file is easily configurable by the `openml` command line interface.
-To see where the file is stored, and what its values are, use [openml
-configure none]{.title-ref}. Set any field with `openml configure FIELD`
-or even all fields with just `openml configure`.
+To see where the file is stored, and what its values are, use openml
+configure none.
## Docker
@@ -139,22 +99,17 @@ train and test splits (cross-validation) and whether this should be
repeated several times. Also, the task defines a target metric for which
a flow should be optimized.
-Below you can find our tutorial regarding tasks and if you want to know
-more you can read the [OpenML guide](https://docs.openml.org/#tasks):
-
-- `sphx_glr_examples_30_extended_tasks_tutorial.py`{.interpreted-text
- role="ref"}
+If you want to know more about tasks, try the ["Task tutorial"](../examples/30_extended/tasks_tutorial)
## Running machine learning algorithms and uploading results
In order to upload and share results of running a machine learning
algorithm on a task, we need to create an
-`~openml.OpenMLRun`{.interpreted-text role="class"}. A run object can be
-created by running a `~openml.OpenMLFlow`{.interpreted-text
-role="class"} or a scikit-learn compatible model on a task. We will
+[openml.runs.OpenMLRun][]. A run object can be
+created by running a [openml.flows.OpenMLFlow][] or a scikit-learn compatible model on a task. We will
focus on the simpler example of running a scikit-learn model.
-Flows are descriptions of something runable which does the machine
+Flows are descriptions of something runnable which does the machine
learning. A flow contains all information to set up the necessary
machine learning library and its dependencies as well as all possible
parameters.
@@ -171,37 +126,30 @@ API](https://scikit-learn.org/stable/developers/develop.html#apis-of-scikit-lear
Those can be directly run on a task, and a flow will automatically be
created or downloaded from the server if it already exists.
-The next tutorial covers how to train different machine learning models,
+See ["Simple Flows and Runs"](../examples/20_basic/simple_flows_and_runs_tutorial) for a tutorial covers how to train different machine learning models,
how to run machine learning models on OpenML data and how to share the
-results:
-
-- `sphx_glr_examples_20_basic_simple_flows_and_runs_tutorial.py`{.interpreted-text
- role="ref"}
+results.
## Datasets
OpenML provides a large collection of datasets and the benchmark
-\"[OpenML100](https://docs.openml.org/benchmark/)\" which consists of a
+[OpenML100](https://docs.openml.org/benchmark/) which consists of a
curated list of datasets.
You can find the dataset that best fits your requirements by making use
-of the available metadata. The tutorial which follows explains how to
+of the available metadata. The tutorial ["extended datasets"](../examples/30_extended/datasets_tutorial) which follows explains how to
get a list of datasets, how to filter the list to find the dataset that
-suits your requirements and how to download a dataset:
-
-- `sphx_glr_examples_30_extended_datasets_tutorial.py`{.interpreted-text
- role="ref"}
+suits your requirements and how to download a dataset.
OpenML is about sharing machine learning results and the datasets they
were obtained on. Learn how to share your datasets in the following
-tutorial:
-
-- `sphx_glr_examples_30_extended_create_upload_tutorial.py`{.interpreted-text
- role="ref"}
+tutorial ["Upload"](../examples/30_extended/create_upload_tutorial) tutorial.
# Extending OpenML-Python
OpenML-Python provides an extension interface to connect machine
learning libraries directly to the API and ships a `scikit-learn`
-extension. You can find more information in the Section
-`extensions`{.interpreted-text role="ref"}\'
+extension. Read more about them in the ["Extensions"](extensions.md) section.
+
+[intro]: examples/20_basic/introduction_tutorial/
+
From f38af1f4b205f66d8d4ad6a745e9be132f231732 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 21 Oct 2024 09:24:17 +0200
Subject: [PATCH 17/25] fix broken links
---
docs/index.md | 19 ++++++++++---------
1 file changed, 10 insertions(+), 9 deletions(-)
diff --git a/docs/index.md b/docs/index.md
index d9247ed54..cda5bcb4b 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -32,7 +32,7 @@ run.publish()
print(f'View the run online: {run.openml_url}')
```
-Find more examples in our ["Example Gallery"][example-gallery].
+Find more examples in the sidebar on the left.
## How to get OpenML for python
@@ -43,7 +43,8 @@ python -m pip install openml
```
For more advanced installation information, please see the
-["Installation"][installation-guide] section.
+["Introduction"](../examples/20_basic/introduction_tutorial.py) example.
+
## Further information
@@ -66,12 +67,6 @@ If you use OpenML-Python in a scientific publication, we would
appreciate a reference to our JMLR-MLOSS paper
["OpenML-Python: an extensible Python API for OpenML"](https://www.jmlr.org/papers/v22/19-920.html):
-=== "MLA"
-
- Feurer, Matthias, et al.
- "OpenML-Python: an extensible Python API for OpenML."
- _Journal of Machine Learning Research_ 22.100 (2021):1−5.
-
=== "Bibtex"
```bibtex
@@ -85,4 +80,10 @@ appreciate a reference to our JMLR-MLOSS paper
pages = {1--5},
url = {http://jmlr.org/papers/v22/19-920.html}
}
- ```
\ No newline at end of file
+ ```
+
+=== "MLA"
+
+ Feurer, Matthias, et al.
+ "OpenML-Python: an extensible Python API for OpenML."
+ _Journal of Machine Learning Research_ 22.100 (2021):1−5.
From 177faf15c7b43af91a1e8b57a4c85d69c639b953 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 21 Oct 2024 10:40:18 +0200
Subject: [PATCH 18/25] Add more notebooks
---
examples/20_basic/introduction_tutorial.ipynb | 180 +++++++++++-------
examples/20_basic/introduction_tutorial.py | 2 +-
.../20_basic/simple_datasets_tutorial.ipynb | 165 ++++++++++++++++
.../20_basic/simple_suites_tutorial.ipynb | 142 ++++++++++++++
4 files changed, 418 insertions(+), 71 deletions(-)
create mode 100644 examples/20_basic/simple_datasets_tutorial.ipynb
create mode 100644 examples/20_basic/simple_suites_tutorial.ipynb
diff --git a/examples/20_basic/introduction_tutorial.ipynb b/examples/20_basic/introduction_tutorial.ipynb
index b7c839654..c940966d2 100644
--- a/examples/20_basic/introduction_tutorial.ipynb
+++ b/examples/20_basic/introduction_tutorial.ipynb
@@ -2,6 +2,13 @@
"cells": [
{
"cell_type": "markdown",
+ "id": "3c295aed5545023a",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
+ },
"source": [
"# Introduction tutorial & Setup\n",
"\n",
@@ -15,28 +22,34 @@
"* Log and share reproducible experiments, models, results\n",
"* Works seamlessly with scikit-learn and other libraries\n",
"* Large scale benchmarking, compare to state of the art"
- ],
- "metadata": {
- "collapsed": false
- },
- "id": "3c295aed5545023a"
+ ]
},
{
"cell_type": "markdown",
+ "id": "48800928f4e5a6a6",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
+ },
"source": [
"# Installation\n",
"installation is done via ``pip``: `pip install openml`\n",
"\n",
"for further information, please check out the installation guide at\n",
":ref:`installation`."
- ],
- "metadata": {
- "collapsed": false
- },
- "id": "48800928f4e5a6a6"
+ ]
},
{
"cell_type": "markdown",
+ "id": "7d6e682752db0423",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
+ },
"source": [
"# Authentication\n",
"\n",
@@ -60,73 +73,92 @@
"\n",
"alternatively, by running the code below and replacing 'YOURKEY' with your API key,\n",
"you authenticate for the duration of the python process."
- ],
- "metadata": {
- "collapsed": false
- },
- "id": "7d6e682752db0423"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "id": "b2fae124f696eff0",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
+ },
"outputs": [],
"source": [
"import openml\n",
"from sklearn import neighbors"
- ],
- "metadata": {
- "collapsed": false
- },
- "id": "b2fae124f696eff0"
+ ]
},
{
"cell_type": "markdown",
- "source": [
- "# .. warning::\n",
- "# .. include:: ../../test_server_usage_warning.txt"
- ],
+ "id": "d0863018bf228f68",
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
- "id": "d0863018bf228f68"
+ "source": [
+ ".. warning::\n",
+ "\n",
+ ".. include:: ../../test_server_usage_warning.txt"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "id": "690279c9d3c1b9db",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
+ },
"outputs": [],
"source": [
"openml.config.start_using_configuration_for_example()"
- ],
- "metadata": {
- "collapsed": false
- },
- "id": "690279c9d3c1b9db"
+ ]
},
{
"cell_type": "markdown",
+ "id": "11851df7a3d601db",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
+ },
"source": [
"when using the main server instead, make sure your apikey is configured.\n",
"this can be done with the following line of code (uncomment it!). Never share your apikey with others."
- ],
- "metadata": {
- "collapsed": false
- },
- "id": "11851df7a3d601db"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "id": "ce399cc2fd336599",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
+ },
"outputs": [],
"source": [
"# openml.config.apikey = 'YOURKEY'"
- ],
- "metadata": {
- "collapsed": false
- },
- "id": "ce399cc2fd336599"
+ ]
},
{
"cell_type": "markdown",
+ "id": "e13705e39146a74d",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
+ },
"source": [
"# Caching\n",
"when downloading datasets, tasks, runs and flows, they will be cached to\n",
@@ -138,40 +170,49 @@
" 'MYDIR' with the path to the cache directory. By default, OpenML\n",
" will use **~/.openml/cache** as the cache directory.\n",
"* Run the code below, replacing 'YOURDIR' with the path to the cache directory."
- ],
- "metadata": {
- "collapsed": false
- },
- "id": "e13705e39146a74d"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "id": "bd7c4fe7e60a5551",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
+ },
"outputs": [],
"source": [
"# Uncomment and set your OpenML cache directory\n",
"# import os\n",
"# openml.config.cache_directory = os.path.expanduser('YOURDIR')"
- ],
- "metadata": {
- "collapsed": false
- },
- "id": "bd7c4fe7e60a5551"
+ ]
},
{
"cell_type": "markdown",
+ "id": "e55589fdbc0576b9",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
+ },
"source": [
"# Simple Example\n",
"download the OpenML task for the eeg-eye-state."
- ],
- "metadata": {
- "collapsed": false
- },
- "id": "e55589fdbc0576b9"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "id": "bf48de543d90a92c",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
+ },
"outputs": [],
"source": [
"task = openml.tasks.get_task(403)\n",
@@ -183,42 +224,41 @@
"# as to not crowd the main server with runs created by examples.\n",
"myrun = run.publish()\n",
"print(f\"kNN on {data.name}: {myrun.openml_url}\")"
- ],
- "metadata": {
- "collapsed": false
- },
- "id": "bf48de543d90a92c"
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "id": "e56c29b055269e1a",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
+ },
"outputs": [],
"source": [
"openml.config.stop_using_configuration_for_example()"
- ],
- "metadata": {
- "collapsed": false
- },
- "id": "e56c29b055269e1a"
+ ]
}
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3",
+ "display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
- "version": 2
+ "version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
- "pygments_lexer": "ipython2",
- "version": "2.7.6"
+ "pygments_lexer": "ipython3",
+ "version": "3.11.3"
}
},
"nbformat": 4,
diff --git a/examples/20_basic/introduction_tutorial.py b/examples/20_basic/introduction_tutorial.py
index ab505b046..ae75a4dff 100644
--- a/examples/20_basic/introduction_tutorial.py
+++ b/examples/20_basic/introduction_tutorial.py
@@ -4,7 +4,7 @@
# %% [markdown]
# OpenML is an online collaboration platform for machine learning which allows
-# you to: Pueter
+# you to:
#
# * Find or share interesting, well-documented datasets
# * Define research / modelling goals (tasks)
diff --git a/examples/20_basic/simple_datasets_tutorial.ipynb b/examples/20_basic/simple_datasets_tutorial.ipynb
new file mode 100644
index 000000000..e0b67edb3
--- /dev/null
+++ b/examples/20_basic/simple_datasets_tutorial.ipynb
@@ -0,0 +1,165 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "40c8509e-4bd4-408f-a48a-b599643241d8",
+ "metadata": {},
+ "source": [
+ "# Datasets\n",
+ "This is a tutorial on how to list, load and visualize datasets.\n",
+ "In general, we recommend working with tasks, so that the results can\n",
+ "be easily reproduced. Furthermore, the results can be compared to existing results\n",
+ "at OpenML. However, for the purposes of this tutorial, we are going to work with\n",
+ "the datasets directly."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "87d8acb0-7238-4945-80ac-fc272b8ae4fb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import openml"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b78c2345-33e9-4c2a-9a93-dee249516e87",
+ "metadata": {},
+ "source": [
+ "To see what datasets are available on OpenML:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "df62d174-e292-494b-802c-bfea731fd30f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "datasets_df = openml.datasets.list_datasets()\n",
+ "print(datasets_df.head(n=10))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5259c7dc-2a6b-487a-a467-b39865ba4f80",
+ "metadata": {},
+ "source": [
+ "Download a dataset"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "eb0ff822-11d9-4911-b8ee-dd7c86e152a8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dataset = openml.datasets.get_dataset(61)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "43669467-0978-4ed7-bff9-54a9030e00e5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(\n",
+ " f\"This is dataset '{dataset.name}', the target feature is \"\n",
+ " f\"'{dataset.default_target_attribute}'\"\n",
+ ")\n",
+ "print(f\"URL: {dataset.url}\")\n",
+ "print(dataset.description[:500])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b72841b0-b028-4ab0-a125-22bdee59f3ee",
+ "metadata": {},
+ "source": [
+ "## Load a dataset\n",
+ "X - An array/datafraX, y, categorical_indicator, attribute_names = dataset.get_data(\n",
+ " dataset_format=\"dataframe\", target=dataset.default_target_attribute\n",
+ ")me where each row represents one example with\n",
+ "the corresponding feature values.\n",
+ "\n",
+ "y - the classes for each example\n",
+ "\n",
+ "categorical_indicator - an array that indicates which feature is categorical\n",
+ "\n",
+ "attribute_names - the names of the features for the examples (X) and\n",
+ "target feature (y)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8555d82d-922c-412d-b362-689d2e50b952",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "X, y, categorical_indicator, attribute_names = dataset.get_data(\n",
+ " dataset_format=\"dataframe\", target=dataset.default_target_attribute\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8c3ab47e-9c6e-4b2c-b775-1f73c976573a",
+ "metadata": {},
+ "source": [
+ "Visualize the dataset"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8f96dcfd-9083-4b3a-868f-2b456ac1a0a1",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "import seaborn as sns\n",
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "sns.set_style(\"darkgrid\")\n",
+ "\n",
+ "\n",
+ "def hide_current_axis(*args, **kwds):\n",
+ " plt.gca().set_visible(False)\n",
+ "\n",
+ "\n",
+ "# We combine all the data so that we can map the different\n",
+ "# examples to different colors according to the classes.\n",
+ "combined_data = pd.concat([X, y], axis=1)\n",
+ "iris_plot = sns.pairplot(combined_data, hue=\"class\")\n",
+ "iris_plot.map_upper(hide_current_axis)\n",
+ "plt.show()"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.11.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/examples/20_basic/simple_suites_tutorial.ipynb b/examples/20_basic/simple_suites_tutorial.ipynb
new file mode 100644
index 000000000..f6d8b569f
--- /dev/null
+++ b/examples/20_basic/simple_suites_tutorial.ipynb
@@ -0,0 +1,142 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "51328ccc-c991-4f13-a233-d3ae33cd95a8",
+ "metadata": {},
+ "source": [
+ "# Benchmark suites\n",
+ "\n",
+ "This is a brief showcase of OpenML benchmark suites, which were introduced by\n",
+ "[Bischl et al. (2019)](https://arxiv.org/abs/1708.03731v2). Benchmark suites standardize the\n",
+ "datasets and splits to be used in an experiment or paper. They are fully integrated into OpenML\n",
+ "and simplify both the sharing of the setup and the results."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b2c6e3ce-ca6a-4355-b834-9add5c6ed9c8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import openml"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a4dd492b-ba0f-4696-bebf-670ecd04113e",
+ "metadata": {},
+ "source": [
+ "# OpenML-CC18\n",
+ "As an example we have a look at the OpenML-CC18, which is a suite of 72 classification datasets\n",
+ "from OpenML which were carefully selected to be usable by many algorithms and also represent\n",
+ "datasets commonly used in machine learning research. These are all datasets from mid-2018 that\n",
+ "satisfy a large set of clear requirements for thorough yet practical benchmarking:\n",
+ "\n",
+ "1. the number of observations are between 500 and 100,000 to focus on medium-sized datasets,\n",
+ "2. the number of features does not exceed 5,000 features to keep the runtime of the algorithms low\n",
+ "3. the target attribute has at least two classes with no class having less than 20 observations\n",
+ "4. the ratio of the minority class and the majority class is above 0.05 (to eliminate highly imbalanced datasets which require special treatment for both algorithms and evaluation measures).\n",
+ "\n",
+ "A full description can be found in the [OpenML benchmarking docs](https://docs.openml.org/benchmark/#openml-cc18).\n",
+ "\n",
+ "In this example we'll focus on how to use benchmark suites in practice.\n",
+ "\n",
+ "# Downloading benchmark suites"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a1562cca-caef-4f31-b2d5-c45c8738c6cd",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "suite = openml.study.get_suite(99)\n",
+ "print(suite)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3578c88d-6c38-4ad5-8bdc-688e32e9cded",
+ "metadata": {},
+ "source": [
+ "The benchmark suite does not download the included tasks and datasets itself, but only contains a list of which tasks constitute the study.\n",
+ "\n",
+ "Tasks can then be accessed via"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4b23b960-2df7-444e-9751-bfb1add540cf",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "tasks = suite.tasks\n",
+ "print(tasks)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5aa0aec7-16e7-46e5-8c76-0a31a1513c52",
+ "metadata": {},
+ "source": [
+ "and iterated over for benchmarking. For speed reasons we only iterate over the first three tasks:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "20a13a2c-807f-4692-a7d6-16c2cfee8b36",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for task_id in tasks[:3]:\n",
+ " task = openml.tasks.get_task(task_id)\n",
+ " print(task)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "cc8183d8-8ca1-4248-8c31-b18b7f7ca72f",
+ "metadata": {},
+ "source": [
+ "# Further examples\n",
+ "* [Extended Suites Tutorial](../30_extended/suites_tutorial)\n",
+ "* :ref:`sphx_glr_examples_30_extended_study_tutorial.py`\n",
+ "* :ref:`sphx_glr_examples_40_paper_2018_ida_strang_example.py`"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "77db0c7d-eb13-4b28-937b-9c02c2dcce70",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.11.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
From ce24b44141e58c664b2ff3b701b4697ecd48374a Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 21 Oct 2024 10:47:59 +0200
Subject: [PATCH 19/25] Remove notebooks
---
examples/20_basic/introduction_tutorial.ipynb | 266 ------------------
.../20_basic/simple_datasets_tutorial.ipynb | 165 -----------
.../20_basic/simple_suites_tutorial.ipynb | 142 ----------
3 files changed, 573 deletions(-)
delete mode 100644 examples/20_basic/introduction_tutorial.ipynb
delete mode 100644 examples/20_basic/simple_datasets_tutorial.ipynb
delete mode 100644 examples/20_basic/simple_suites_tutorial.ipynb
diff --git a/examples/20_basic/introduction_tutorial.ipynb b/examples/20_basic/introduction_tutorial.ipynb
deleted file mode 100644
index c940966d2..000000000
--- a/examples/20_basic/introduction_tutorial.ipynb
+++ /dev/null
@@ -1,266 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "id": "3c295aed5545023a",
- "metadata": {
- "collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- }
- },
- "source": [
- "# Introduction tutorial & Setup\n",
- "\n",
- "An example how to set up OpenML-Python followed up by a simple example.\n",
- "OpenML is an online collaboration platform for machine learning which allows\n",
- "you to:\n",
- "\n",
- "* Find or share interesting, well-documented datasets\n",
- "* Define research / modelling goals (tasks)\n",
- "* Explore large amounts of machine learning algorithms, with APIs in Java, R, Python\n",
- "* Log and share reproducible experiments, models, results\n",
- "* Works seamlessly with scikit-learn and other libraries\n",
- "* Large scale benchmarking, compare to state of the art"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "48800928f4e5a6a6",
- "metadata": {
- "collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- }
- },
- "source": [
- "# Installation\n",
- "installation is done via ``pip``: `pip install openml`\n",
- "\n",
- "for further information, please check out the installation guide at\n",
- ":ref:`installation`."
- ]
- },
- {
- "cell_type": "markdown",
- "id": "7d6e682752db0423",
- "metadata": {
- "collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- }
- },
- "source": [
- "# Authentication\n",
- "\n",
- "the OpenML server can only be accessed by users who have signed up on the\n",
- "openML platform. If you don’t have an account yet, sign up now.\n",
- "you will receive an API key, which will authenticate you to the server\n",
- "and allow you to download and upload datasets, tasks, runs and flows.\n",
- "\n",
- "* Create an OpenML account (free) on [https://www.openml.org](https://www.openml.org).\n",
- "* After logging in, open your account page (avatar on the top right)\n",
- "* Open 'Account Settings', then 'API authentication' to find your API key.\n",
- "\n",
- "there are two ways to permanently authenticate:\n",
- "\n",
- "* Use the ``openml`` CLI tool with ``openml configure apikey MYKEY``,\n",
- " replacing **MYKEY** with your API key.\n",
- "* Create a plain text file **~/.openml/config** with the line\n",
- " **'apikey=MYKEY'**, replacing **MYKEY** with your API key. The config\n",
- " file must be in the directory ~/.openml/config and exist prior to\n",
- " importing the openml module.\n",
- "\n",
- "alternatively, by running the code below and replacing 'YOURKEY' with your API key,\n",
- "you authenticate for the duration of the python process."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "b2fae124f696eff0",
- "metadata": {
- "collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- }
- },
- "outputs": [],
- "source": [
- "import openml\n",
- "from sklearn import neighbors"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "d0863018bf228f68",
- "metadata": {
- "collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- }
- },
- "source": [
- ".. warning::\n",
- "\n",
- ".. include:: ../../test_server_usage_warning.txt"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "690279c9d3c1b9db",
- "metadata": {
- "collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- }
- },
- "outputs": [],
- "source": [
- "openml.config.start_using_configuration_for_example()"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "11851df7a3d601db",
- "metadata": {
- "collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- }
- },
- "source": [
- "when using the main server instead, make sure your apikey is configured.\n",
- "this can be done with the following line of code (uncomment it!). Never share your apikey with others."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "ce399cc2fd336599",
- "metadata": {
- "collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- }
- },
- "outputs": [],
- "source": [
- "# openml.config.apikey = 'YOURKEY'"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "e13705e39146a74d",
- "metadata": {
- "collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- }
- },
- "source": [
- "# Caching\n",
- "when downloading datasets, tasks, runs and flows, they will be cached to\n",
- "retrieve them without calling the server later. As with the API key,\n",
- "the cache directory can be either specified through the config file or\n",
- "through the API:\n",
- "\n",
- "* Add the line **cachedir = 'MYDIR'** to the config file, replacing\n",
- " 'MYDIR' with the path to the cache directory. By default, OpenML\n",
- " will use **~/.openml/cache** as the cache directory.\n",
- "* Run the code below, replacing 'YOURDIR' with the path to the cache directory."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "bd7c4fe7e60a5551",
- "metadata": {
- "collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- }
- },
- "outputs": [],
- "source": [
- "# Uncomment and set your OpenML cache directory\n",
- "# import os\n",
- "# openml.config.cache_directory = os.path.expanduser('YOURDIR')"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "e55589fdbc0576b9",
- "metadata": {
- "collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- }
- },
- "source": [
- "# Simple Example\n",
- "download the OpenML task for the eeg-eye-state."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "bf48de543d90a92c",
- "metadata": {
- "collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- }
- },
- "outputs": [],
- "source": [
- "task = openml.tasks.get_task(403)\n",
- "data = openml.datasets.get_dataset(task.dataset_id)\n",
- "clf = neighbors.KNeighborsClassifier(n_neighbors=5)\n",
- "run = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False)\n",
- "# Publish the experiment on OpenML (optional, requires an API key).\n",
- "# For this tutorial, our configuration publishes to the test server\n",
- "# as to not crowd the main server with runs created by examples.\n",
- "myrun = run.publish()\n",
- "print(f\"kNN on {data.name}: {myrun.openml_url}\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "e56c29b055269e1a",
- "metadata": {
- "collapsed": false,
- "jupyter": {
- "outputs_hidden": false
- }
- },
- "outputs": [],
- "source": [
- "openml.config.stop_using_configuration_for_example()"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3 (ipykernel)",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.11.3"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}
diff --git a/examples/20_basic/simple_datasets_tutorial.ipynb b/examples/20_basic/simple_datasets_tutorial.ipynb
deleted file mode 100644
index e0b67edb3..000000000
--- a/examples/20_basic/simple_datasets_tutorial.ipynb
+++ /dev/null
@@ -1,165 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "id": "40c8509e-4bd4-408f-a48a-b599643241d8",
- "metadata": {},
- "source": [
- "# Datasets\n",
- "This is a tutorial on how to list, load and visualize datasets.\n",
- "In general, we recommend working with tasks, so that the results can\n",
- "be easily reproduced. Furthermore, the results can be compared to existing results\n",
- "at OpenML. However, for the purposes of this tutorial, we are going to work with\n",
- "the datasets directly."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "87d8acb0-7238-4945-80ac-fc272b8ae4fb",
- "metadata": {},
- "outputs": [],
- "source": [
- "import openml"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "b78c2345-33e9-4c2a-9a93-dee249516e87",
- "metadata": {},
- "source": [
- "To see what datasets are available on OpenML:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "df62d174-e292-494b-802c-bfea731fd30f",
- "metadata": {},
- "outputs": [],
- "source": [
- "datasets_df = openml.datasets.list_datasets()\n",
- "print(datasets_df.head(n=10))"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "5259c7dc-2a6b-487a-a467-b39865ba4f80",
- "metadata": {},
- "source": [
- "Download a dataset"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "eb0ff822-11d9-4911-b8ee-dd7c86e152a8",
- "metadata": {},
- "outputs": [],
- "source": [
- "dataset = openml.datasets.get_dataset(61)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "43669467-0978-4ed7-bff9-54a9030e00e5",
- "metadata": {},
- "outputs": [],
- "source": [
- "print(\n",
- " f\"This is dataset '{dataset.name}', the target feature is \"\n",
- " f\"'{dataset.default_target_attribute}'\"\n",
- ")\n",
- "print(f\"URL: {dataset.url}\")\n",
- "print(dataset.description[:500])"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "b72841b0-b028-4ab0-a125-22bdee59f3ee",
- "metadata": {},
- "source": [
- "## Load a dataset\n",
- "X - An array/datafraX, y, categorical_indicator, attribute_names = dataset.get_data(\n",
- " dataset_format=\"dataframe\", target=dataset.default_target_attribute\n",
- ")me where each row represents one example with\n",
- "the corresponding feature values.\n",
- "\n",
- "y - the classes for each example\n",
- "\n",
- "categorical_indicator - an array that indicates which feature is categorical\n",
- "\n",
- "attribute_names - the names of the features for the examples (X) and\n",
- "target feature (y)\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "8555d82d-922c-412d-b362-689d2e50b952",
- "metadata": {},
- "outputs": [],
- "source": [
- "X, y, categorical_indicator, attribute_names = dataset.get_data(\n",
- " dataset_format=\"dataframe\", target=dataset.default_target_attribute\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "8c3ab47e-9c6e-4b2c-b775-1f73c976573a",
- "metadata": {},
- "source": [
- "Visualize the dataset"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "8f96dcfd-9083-4b3a-868f-2b456ac1a0a1",
- "metadata": {},
- "outputs": [],
- "source": [
- "import pandas as pd\n",
- "import seaborn as sns\n",
- "import matplotlib.pyplot as plt\n",
- "\n",
- "sns.set_style(\"darkgrid\")\n",
- "\n",
- "\n",
- "def hide_current_axis(*args, **kwds):\n",
- " plt.gca().set_visible(False)\n",
- "\n",
- "\n",
- "# We combine all the data so that we can map the different\n",
- "# examples to different colors according to the classes.\n",
- "combined_data = pd.concat([X, y], axis=1)\n",
- "iris_plot = sns.pairplot(combined_data, hue=\"class\")\n",
- "iris_plot.map_upper(hide_current_axis)\n",
- "plt.show()"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3 (ipykernel)",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.11.3"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}
diff --git a/examples/20_basic/simple_suites_tutorial.ipynb b/examples/20_basic/simple_suites_tutorial.ipynb
deleted file mode 100644
index f6d8b569f..000000000
--- a/examples/20_basic/simple_suites_tutorial.ipynb
+++ /dev/null
@@ -1,142 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "id": "51328ccc-c991-4f13-a233-d3ae33cd95a8",
- "metadata": {},
- "source": [
- "# Benchmark suites\n",
- "\n",
- "This is a brief showcase of OpenML benchmark suites, which were introduced by\n",
- "[Bischl et al. (2019)](https://arxiv.org/abs/1708.03731v2). Benchmark suites standardize the\n",
- "datasets and splits to be used in an experiment or paper. They are fully integrated into OpenML\n",
- "and simplify both the sharing of the setup and the results."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "b2c6e3ce-ca6a-4355-b834-9add5c6ed9c8",
- "metadata": {},
- "outputs": [],
- "source": [
- "import openml"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "a4dd492b-ba0f-4696-bebf-670ecd04113e",
- "metadata": {},
- "source": [
- "# OpenML-CC18\n",
- "As an example we have a look at the OpenML-CC18, which is a suite of 72 classification datasets\n",
- "from OpenML which were carefully selected to be usable by many algorithms and also represent\n",
- "datasets commonly used in machine learning research. These are all datasets from mid-2018 that\n",
- "satisfy a large set of clear requirements for thorough yet practical benchmarking:\n",
- "\n",
- "1. the number of observations are between 500 and 100,000 to focus on medium-sized datasets,\n",
- "2. the number of features does not exceed 5,000 features to keep the runtime of the algorithms low\n",
- "3. the target attribute has at least two classes with no class having less than 20 observations\n",
- "4. the ratio of the minority class and the majority class is above 0.05 (to eliminate highly imbalanced datasets which require special treatment for both algorithms and evaluation measures).\n",
- "\n",
- "A full description can be found in the [OpenML benchmarking docs](https://docs.openml.org/benchmark/#openml-cc18).\n",
- "\n",
- "In this example we'll focus on how to use benchmark suites in practice.\n",
- "\n",
- "# Downloading benchmark suites"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "a1562cca-caef-4f31-b2d5-c45c8738c6cd",
- "metadata": {},
- "outputs": [],
- "source": [
- "suite = openml.study.get_suite(99)\n",
- "print(suite)"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "3578c88d-6c38-4ad5-8bdc-688e32e9cded",
- "metadata": {},
- "source": [
- "The benchmark suite does not download the included tasks and datasets itself, but only contains a list of which tasks constitute the study.\n",
- "\n",
- "Tasks can then be accessed via"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "4b23b960-2df7-444e-9751-bfb1add540cf",
- "metadata": {},
- "outputs": [],
- "source": [
- "tasks = suite.tasks\n",
- "print(tasks)"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "5aa0aec7-16e7-46e5-8c76-0a31a1513c52",
- "metadata": {},
- "source": [
- "and iterated over for benchmarking. For speed reasons we only iterate over the first three tasks:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "20a13a2c-807f-4692-a7d6-16c2cfee8b36",
- "metadata": {},
- "outputs": [],
- "source": [
- "for task_id in tasks[:3]:\n",
- " task = openml.tasks.get_task(task_id)\n",
- " print(task)"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "cc8183d8-8ca1-4248-8c31-b18b7f7ca72f",
- "metadata": {},
- "source": [
- "# Further examples\n",
- "* [Extended Suites Tutorial](../30_extended/suites_tutorial)\n",
- "* :ref:`sphx_glr_examples_30_extended_study_tutorial.py`\n",
- "* :ref:`sphx_glr_examples_40_paper_2018_ida_strang_example.py`"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "77db0c7d-eb13-4b28-937b-9c02c2dcce70",
- "metadata": {},
- "outputs": [],
- "source": []
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3 (ipykernel)",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.11.3"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}
From e5f60f5be0f5ea5e45d03691e3b3aac3cdb66c33 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 21 Oct 2024 11:20:49 +0200
Subject: [PATCH 20/25] Convert more examples to jupytext
---
examples/20_basic/introduction_tutorial.py | 15 +--
examples/20_basic/simple_datasets_tutorial.py | 3 +-
.../simple_flows_and_runs_tutorial.py | 28 ++----
examples/20_basic/simple_suites_tutorial.py | 32 +++----
examples/30_extended/configure_logging.py | 23 ++---
.../30_extended/create_upload_tutorial.py | 62 +++++-------
examples/30_extended/custom_flow_.py | 49 +++++-----
examples/30_extended/datasets_tutorial.py | 81 ++++++++--------
.../30_extended/fetch_evaluations_tutorial.py | 94 +++++++++----------
mkdocs.yml | 3 +-
10 files changed, 170 insertions(+), 220 deletions(-)
diff --git a/examples/20_basic/introduction_tutorial.py b/examples/20_basic/introduction_tutorial.py
index ae75a4dff..4bf5fd45d 100644
--- a/examples/20_basic/introduction_tutorial.py
+++ b/examples/20_basic/introduction_tutorial.py
@@ -18,17 +18,12 @@
# # Installation
# Installation is done via ``pip``:
#
-# .. code:: bash
-#
-# pip install openml
-#
-# For further information, please check out the installation guide at
-# :ref:`installation`.
-#
+# ```bash
+# pip install openml
+# ```
# %% [markdown]
-# Authentication
-# ^^^^^^^^^^^^^^
+# # Authentication
#
# The OpenML server can only be accessed by users who have signed up on the
# OpenML platform. If you don’t have an account yet, sign up now.
@@ -53,7 +48,6 @@
# %%
-# License: BSD 3-Clause
import openml
from sklearn import neighbors
@@ -107,3 +101,4 @@
# %%
openml.config.stop_using_configuration_for_example()
+# License: BSD 3-Clause
diff --git a/examples/20_basic/simple_datasets_tutorial.py b/examples/20_basic/simple_datasets_tutorial.py
index 18493f684..b0381533c 100644
--- a/examples/20_basic/simple_datasets_tutorial.py
+++ b/examples/20_basic/simple_datasets_tutorial.py
@@ -8,7 +8,6 @@
# the datasets directly.
# %%
-# License: BSD 3-Clause
import openml
@@ -69,3 +68,5 @@ def hide_current_axis(*args, **kwds):
iris_plot = sns.pairplot(combined_data, hue="class")
iris_plot.map_upper(hide_current_axis)
plt.show()
+
+# License: BSD 3-Clause
diff --git a/examples/20_basic/simple_flows_and_runs_tutorial.py b/examples/20_basic/simple_flows_and_runs_tutorial.py
index ca5cd4412..79c69245c 100644
--- a/examples/20_basic/simple_flows_and_runs_tutorial.py
+++ b/examples/20_basic/simple_flows_and_runs_tutorial.py
@@ -1,27 +1,21 @@
-"""
# %% [markdown]
-Flows and Runs
-==============
-
-A simple tutorial on how to train/run a model and how to upload the results.
-"""
-
-# License: BSD 3-Clause
+# # Flows and Runs
+# A simple tutorial on how to train/run a model and how to upload the results.
+# %%
import openml
from sklearn import ensemble, neighbors
-############################################################################
# %% [markdown]
# .. warning::
# .. include:: ../../test_server_usage_warning.txt
+
+# %%
openml.config.start_using_configuration_for_example()
-############################################################################
# %% [markdown]
-# Train a machine learning model
-# ==============================
+# ## Train a machine learning model
# NOTE: We are using dataset 20 from the test server: https://test.openml.org/d/20
dataset = openml.datasets.get_dataset(20)
@@ -31,25 +25,21 @@
clf = neighbors.KNeighborsClassifier(n_neighbors=3)
clf.fit(X, y)
-############################################################################
# %% [markdown]
-# Running a model on a task
-# =========================
+# ## Running a model on a task
task = openml.tasks.get_task(119)
clf = ensemble.RandomForestClassifier()
run = openml.runs.run_model_on_task(clf, task)
print(run)
-############################################################################
# %% [markdown]
-# Publishing the run
-# ==================
+# ## Publishing the run
myrun = run.publish()
print(f"Run was uploaded to {myrun.openml_url}")
print(f"The flow can be found at {myrun.flow.openml_url}")
-############################################################################
# %% [markdown]
openml.config.stop_using_configuration_for_example()
+# License: BSD 3-Clause
diff --git a/examples/20_basic/simple_suites_tutorial.py b/examples/20_basic/simple_suites_tutorial.py
index 10e834dcc..34aefc913 100644
--- a/examples/20_basic/simple_suites_tutorial.py
+++ b/examples/20_basic/simple_suites_tutorial.py
@@ -1,17 +1,11 @@
-"""
# %% [markdown]
-================
-Benchmark suites
-================
-
-This is a brief showcase of OpenML benchmark suites, which were introduced by
-`Bischl et al. (2019) <https://arxiv.org/abs/1708.03731v2>`_. Benchmark suites standardize the
-datasets and splits to be used in an experiment or paper. They are fully integrated into OpenML
-and simplify both the sharing of the setup and the results.
-"""
-
-# License: BSD 3-Clause
+# # Benchmark suites
+# This is a brief showcase of OpenML benchmark suites, which were introduced by
+# [Bischl et al. (2019)](https://arxiv.org/abs/1708.03731v2). Benchmark suites standardize the
+# datasets and splits to be used in an experiment or paper. They are fully integrated into OpenML
+# and simplify both the sharing of the setup and the results.
+# %%
import openml
####################################################################################################
@@ -32,42 +26,44 @@
# imbalanced datasets which require special treatment for both algorithms and evaluation
# measures).
#
-# A full description can be found in the `OpenML benchmarking docs
-# <https://docs.openml.org/benchmark/#openml-cc18>`_.
+# A full description can be found in the
+# [OpenML benchmarking docs](https://docs.openml.org/benchmark/#openml-cc18).
#
# In this example we'll focus on how to use benchmark suites in practice.
-####################################################################################################
# %% [markdown]
# Downloading benchmark suites
# ============================
+# %%
suite = openml.study.get_suite(99)
print(suite)
-####################################################################################################
# %% [markdown]
# The benchmark suite does not download the included tasks and datasets itself, but only contains
# a list of which tasks constitute the study.
#
# Tasks can then be accessed via
+# %%
tasks = suite.tasks
print(tasks)
-####################################################################################################
# %% [markdown]
# and iterated over for benchmarking. For speed reasons we only iterate over the first three tasks:
+# %%
for task_id in tasks[:3]:
task = openml.tasks.get_task(task_id)
print(task)
-####################################################################################################
# %% [markdown]
# Further examples
# ================
#
+# * [ex](../../30_extended/suites_tutorial)
# * :ref:`sphx_glr_examples_30_extended_suites_tutorial.py`
# * :ref:`sphx_glr_examples_30_extended_study_tutorial.py`
# * :ref:`sphx_glr_examples_40_paper_2018_ida_strang_example.py`
+
+# License: BSD 3-Clause
diff --git a/examples/30_extended/configure_logging.py b/examples/30_extended/configure_logging.py
index c13140a95..bb7fc22a3 100644
--- a/examples/30_extended/configure_logging.py
+++ b/examples/30_extended/configure_logging.py
@@ -1,29 +1,21 @@
-"""
# %% [markdown]
-========
-Logging
-========
-
-Explains openml-python logging, and shows how to configure it.
-"""
-##################################################################################
-# %% [markdown]
-# Openml-python uses the `Python logging module <https://docs.python.org/3/library/logging.html>`_
+# # Logging
+# This tutorial explains openml-python logging, and shows how to configure it.
+# Openml-python uses the [Python logging module[(https://docs.python.org/3/library/logging.html)
# to provide users with log messages. Each log message is assigned a level of importance, see
# the table in Python's logging tutorial
-# `here <https://docs.python.org/3/howto/logging.html#when-to-use-logging>`_.
+# [here](https://docs.python.org/3/howto/logging.html#when-to-use-logging).
#
# By default, openml-python will print log messages of level `WARNING` and above to console.
# All log messages (including `DEBUG` and `INFO`) are also saved in a file, which can be
# found in your cache directory (see also the
-# :ref:`sphx_glr_examples_20_basic_introduction_tutorial.py`).
+# [introduction tutorial](../20_basic/introduction_tutorial).
# These file logs are automatically deleted if needed, and use at most 2MB of space.
#
# It is possible to configure what log levels to send to console and file.
# When downloading a dataset from OpenML, a `DEBUG`-level message is written:
-# License: BSD 3-Clause
-
+# %%
import openml
openml.datasets.get_dataset("iris")
@@ -38,6 +30,7 @@
# , depending on whether or not you had downloaded iris before.
# The processed log levels can be configured programmatically:
+# %%
import logging
openml.config.set_console_log_level(logging.DEBUG)
@@ -56,3 +49,5 @@
# * 0: `logging.WARNING` and up.
# * 1: `logging.INFO` and up.
# * 2: `logging.DEBUG` and up (i.e. all messages).
+#
+# License: BSD 3-Clause
diff --git a/examples/30_extended/create_upload_tutorial.py b/examples/30_extended/create_upload_tutorial.py
index 95ae4d468..2b010401c 100644
--- a/examples/30_extended/create_upload_tutorial.py
+++ b/examples/30_extended/create_upload_tutorial.py
@@ -1,13 +1,8 @@
-"""
# %% [markdown]
-Dataset upload tutorial
-=======================
-
-A tutorial on how to create and upload a dataset to OpenML.
-"""
-
-# License: BSD 3-Clause
+# # Dataset upload tutorial
+# A tutorial on how to create and upload a dataset to OpenML.
+# %%
import numpy as np
import pandas as pd
import sklearn.datasets
@@ -16,16 +11,13 @@
import openml
from openml.datasets.functions import create_dataset
-#
# %% [markdown]
-##########################################################################
# .. warning::
# .. include:: ../../test_server_usage_warning.txt
+# %%
openml.config.start_using_configuration_for_example()
-############################################################################
-##############################################################################
# %% [markdown]
# Below we will cover the following cases of the dataset object:
#
@@ -35,18 +27,17 @@
# * A sparse matrix
# * A pandas sparse dataframe
-############################################################################
# %% [markdown]
# Dataset is a numpy array
# ========================
# A numpy array can contain lists in the case of dense data or it can contain
# OrderedDicts in the case of sparse data.
#
-# Prepare dataset
-# ^^^^^^^^^^^^^^^
+# # Prepare dataset
# Load an example dataset from scikit-learn which we will upload to OpenML.org
# via the API.
+# %%
diabetes = sklearn.datasets.load_diabetes()
name = "Diabetes(scikit-learn)"
X = diabetes.data
@@ -54,7 +45,6 @@
attribute_names = diabetes.feature_names
description = diabetes.DESCR
-############################################################################
# %% [markdown]
# OpenML does not distinguish between the attributes and targets on the data
# level and stores all data in a single matrix.
@@ -62,6 +52,7 @@
# The target feature is indicated as meta-data of the dataset (and tasks on
# that data).
+# %%
data = np.concatenate((X, y.reshape((-1, 1))), axis=1)
attribute_names = list(attribute_names)
attributes = [(attribute_name, "REAL") for attribute_name in attribute_names] + [
@@ -74,15 +65,14 @@
)
paper_url = "https://web.stanford.edu/~hastie/Papers/LARS/LeastAngle_2002.pdf"
-############################################################################
# %% [markdown]
-# Create the dataset object
-# ^^^^^^^^^^^^^^^^^^^^^^^^^
+# # Create the dataset object
# The definition of all fields can be found in the XSD files describing the
# expected format:
#
# https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/openml.data.upload.xsd
+# %%
diabetes_dataset = create_dataset(
# The name of the dataset (needs to be unique).
# Must not be longer than 128 characters and only contain
@@ -120,21 +110,20 @@
paper_url=paper_url,
)
-############################################################################
+# %%
diabetes_dataset.publish()
print(f"URL for dataset: {diabetes_dataset.openml_url}")
-############################################################################
# %% [markdown]
-# Dataset is a list
-# =================
+# ## Dataset is a list
# A list can contain lists in the case of dense data or it can contain
# OrderedDicts in the case of sparse data.
#
# Weather dataset:
# https://storm.cis.fordham.edu/~gweiss/data-mining/datasets.html
+# %%
data = [
["sunny", 85, 85, "FALSE", "no"],
["sunny", 80, 90, "TRUE", "no"],
@@ -194,15 +183,13 @@
version_label="example",
)
-############################################################################
+# %%
weather_dataset.publish()
print(f"URL for dataset: {weather_dataset.openml_url}")
-############################################################################
# %% [markdown]
-# Dataset is a pandas DataFrame
-# =============================
+# ## Dataset is a pandas DataFrame
# It might happen that your dataset is made of heterogeneous data which can usually
# be stored as a Pandas DataFrame. DataFrames offer the advantage of
# storing the type of data for each column as well as the attribute names.
@@ -211,14 +198,15 @@
# function :func:`openml.datasets.create_dataset`. In this regard, you only
# need to pass ``'auto'`` to the ``attributes`` parameter.
+# %%
df = pd.DataFrame(data, columns=[col_name for col_name, _ in attribute_names])
+
# enforce the categorical column to have a categorical dtype
df["outlook"] = df["outlook"].astype("category")
df["windy"] = df["windy"].astype("bool")
df["play"] = df["play"].astype("category")
print(df.info())
-############################################################################
# %% [markdown]
# We enforce the column 'outlook' and 'play' to be a categorical
# dtype while the column 'windy' is kept as a boolean column. 'temperature'
@@ -226,6 +214,7 @@
# call :func:`openml.datasets.create_dataset` by passing the dataframe and
# fixing the parameter ``attributes`` to ``'auto'``.
+# %%
weather_dataset = create_dataset(
name="Weather",
description=description,
@@ -243,16 +232,15 @@
version_label="example",
)
-############################################################################
-
+# %%
weather_dataset.publish()
print(f"URL for dataset: {weather_dataset.openml_url}")
-############################################################################
# %% [markdown]
# Dataset is a sparse matrix
# ==========================
+# %%
sparse_data = coo_matrix(
([0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], ([0, 1, 1, 2, 2, 3, 3], [0, 1, 2, 0, 2, 0, 1]))
)
@@ -280,16 +268,14 @@
version_label="example",
)
-############################################################################
+# %%
xor_dataset.publish()
print(f"URL for dataset: {xor_dataset.openml_url}")
-############################################################################
# %% [markdown]
-# Dataset is a pandas dataframe with sparse columns
-# =================================================
+# ## Dataset is a pandas dataframe with sparse columns
sparse_data = coo_matrix(
([1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0], ([0, 1, 1, 2, 2, 3, 3], [0, 1, 2, 0, 2, 0, 1]))
@@ -315,11 +301,11 @@
version_label="example",
)
-############################################################################
+# %%
xor_dataset.publish()
print(f"URL for dataset: {xor_dataset.openml_url}")
-
-############################################################################
+# %%
openml.config.stop_using_configuration_for_example()
+# License: BSD 3-Clause
diff --git a/examples/30_extended/custom_flow_.py b/examples/30_extended/custom_flow_.py
index d48efcd09..7e28a92ca 100644
--- a/examples/30_extended/custom_flow_.py
+++ b/examples/30_extended/custom_flow_.py
@@ -1,21 +1,16 @@
-"""
# %% [markdown]
-================================
-Creating and Using a Custom Flow
-================================
+# Creating and Using a Custom Flow
-The most convenient way to create a flow for your machine learning workflow is to generate it
-automatically as described in the :ref:`sphx_glr_examples_30_extended_flow_id_tutorial.py` tutorial.
-However, there are scenarios where this is not possible, such
-as when the flow uses a framework without an extension or when the flow is described by a script.
+# The most convenient way to create a flow for your machine learning workflow is to generate it
+# automatically as described in the :ref:`sphx_glr_examples_30_extended_flow_id_tutorial.py` tutorial.
+# However, there are scenarios where this is not possible, such
+# as when the flow uses a framework without an extension or when the flow is described by a script.
-In those cases you can still create a custom flow by following the steps of this tutorial.
-As an example we will use the flows generated for the `AutoML Benchmark <https://openml.github.io/automlbenchmark/>`_,
-and also show how to link runs to the custom flow.
-"""
-
-# License: BSD 3-Clause
+# In those cases you can still create a custom flow by following the steps of this tutorial.
+# As an example we will use the flows generated for the `AutoML Benchmark <https://openml.github.io/automlbenchmark/>`_,
+# and also show how to link runs to the custom flow.
+# %%
from collections import OrderedDict
import numpy as np
@@ -23,16 +18,15 @@
from openml import OpenMLClassificationTask
from openml.runs.functions import format_prediction
-####################################################################################################
# %% [markdown]
# .. warning::
# .. include:: ../../test_server_usage_warning.txt
+
+# %%
openml.config.start_using_configuration_for_example()
-####################################################################################################
# %% [markdown]
-# 1. Defining the flow
-# ====================
+# ## 1. Defining the flow
# The first step is to define all the hyperparameters of your flow.
# The API pages feature a descriptions of each variable of the :class:`openml.flows.OpenMLFlow`.
# Note that `external version` and `name` together uniquely identify a flow.
@@ -46,6 +40,7 @@
# Make sure to leave enough information so others can determine exactly which
# version of the package/script is used. Use tags so users can find your flow easily.
+# %%
general = dict(
name="automlbenchmark_autosklearn",
description=(
@@ -58,13 +53,13 @@
dependencies="amlb==0.9",
)
-####################################################################################################
# %% [markdown]
# Next we define the flow hyperparameters. We define their name and default value in `parameters`,
# and provide meta-data for each hyperparameter through `parameters_meta_info`.
# Note that even though the argument name is `parameters` they describe the hyperparameters.
# The use of ordered dicts is required.
+# %%
flow_hyperparameters = dict(
parameters=OrderedDict(time="240", memory="32", cores="8"),
parameters_meta_info=OrderedDict(
@@ -74,7 +69,6 @@
),
)
-####################################################################################################
# %% [markdown]
# It is possible to build a flow which uses other flows.
# For example, the Random Forest Classifier is a flow, but you could also construct a flow
@@ -91,6 +85,7 @@
# Note: flow 9313 is not actually the right flow on the test server,
# but that does not matter for this demonstration.
+# %%
autosklearn_flow = openml.flows.get_flow(9313) # auto-sklearn 0.5.1
subflow = dict(
components=OrderedDict(automl_tool=autosklearn_flow),
@@ -98,7 +93,6 @@
# components=OrderedDict(),
)
-####################################################################################################
# %% [markdown]
# With all parameters of the flow defined, we can now initialize the OpenMLFlow and publish.
# Because we provided all the details already, we do not need to provide a `model` to the flow.
@@ -109,6 +103,7 @@
# So whether you have a model with no extension or no model at all, explicitly set
# the model of the flow to `None`.
+# %%
autosklearn_amlb_flow = openml.flows.OpenMLFlow(
**general,
**flow_hyperparameters,
@@ -118,15 +113,14 @@
autosklearn_amlb_flow.publish()
print(f"autosklearn flow created: {autosklearn_amlb_flow.flow_id}")
-####################################################################################################
# %% [markdown]
-# 2. Using the flow
-# ====================
+# ## 2. Using the flow
# This Section will show how to upload run data for your custom flow.
# Take care to change the values of parameters as well as the task id,
# to reflect the actual run.
# Task and parameter values in the example are fictional.
+# %%
flow_id = autosklearn_amlb_flow.flow_id
parameters = [
@@ -140,7 +134,6 @@
dataset_id = task.get_dataset().dataset_id
-####################################################################################################
# %% [markdown]
# The last bit of information for the run we need are the predicted values.
# The exact format of the predictions will depend on the task.
@@ -166,6 +159,8 @@
# You can ignore this code, or use it to better understand the formatting of the predictions.
#
# Find the repeats/folds for this task:
+
+# %%
n_repeats, n_folds, _ = task.get_split_dimensions()
all_test_indices = [
(repeat, fold, index)
@@ -201,11 +196,11 @@
)
predictions.append(prediction)
-####################################################################################################
# %% [markdown]
# Finally we can create the OpenMLRun object and upload.
# We use the argument setup_string because the used flow was a script.
+# %%
benchmark_command = f"python3 runbenchmark.py auto-sklearn medium -m aws -t 119"
my_run = openml.runs.OpenMLRun(
task_id=task_id,
@@ -220,4 +215,6 @@
my_run.publish()
print("run created:", my_run.run_id)
+# %%
openml.config.stop_using_configuration_for_example()
+# License: BSD 3-Clause
diff --git a/examples/30_extended/datasets_tutorial.py b/examples/30_extended/datasets_tutorial.py
index 64f3d5d0c..4cd04de0c 100644
--- a/examples/30_extended/datasets_tutorial.py
+++ b/examples/30_extended/datasets_tutorial.py
@@ -1,58 +1,46 @@
-"""
# %% [markdown]
-========
-Datasets
-========
-
-How to list and download datasets.
-"""
-
-# License: BSD 3-Clauses
+# # Datasets
+# How to list and download datasets.
+# %%
import openml
import pandas as pd
from openml.datasets import edit_dataset, fork_dataset, get_dataset
-############################################################################
# %% [markdown]
-# Exercise 0
-# **********
+# ## Exercise 0
#
# * List datasets
-#
-# * Use the output_format parameter to select output type
-# * Default gives 'dict' (other option: 'dataframe', see below)
-#
-# Note: list_datasets will return a pandas dataframe by default from 0.15. When using
-# openml-python 0.14, `list_datasets` will warn you to use output_format='dataframe'.
-datalist = openml.datasets.list_datasets(output_format="dataframe")
+
+# %%
+datalist = openml.datasets.list_datasets()
datalist = datalist[["did", "name", "NumberOfInstances", "NumberOfFeatures", "NumberOfClasses"]]
print(f"First 10 of {len(datalist)} datasets...")
datalist.head(n=10)
# The same can be done with lesser lines of code
-openml_df = openml.datasets.list_datasets(output_format="dataframe")
+openml_df = openml.datasets.list_datasets()
openml_df.head(n=10)
-############################################################################
# %% [markdown]
-# Exercise 1
-# **********
+# ## Exercise 1
#
# * Find datasets with more than 10000 examples.
# * Find a dataset called 'eeg_eye_state'.
# * Find all datasets with more than 50 classes.
+
+# %%
datalist[datalist.NumberOfInstances > 10000].sort_values(["NumberOfInstances"]).head(n=20)
-""
+
+# %%
datalist.query('name == "eeg-eye-state"')
-""
+
+# %%
datalist.query("NumberOfClasses > 50")
-############################################################################
# %% [markdown]
-# Download datasets
-# =================
+# ## Download datasets
# This is done based on the dataset ID.
dataset = openml.datasets.get_dataset(1471)
@@ -65,26 +53,27 @@
print(f"URL: {dataset.url}")
print(dataset.description[:500])
-############################################################################
# %% [markdown]
# Get the actual data.
#
# openml-python returns data as pandas dataframes (stored in the `eeg` variable below),
# and also some additional metadata that we don't care about right now.
+
+# %%
eeg, *_ = dataset.get_data()
-############################################################################
# %% [markdown]
# You can optionally choose to have openml separate out a column from the
# dataset. In particular, many datasets for supervised problems have a set
# `default_target_attribute` which may help identify the target variable.
+
+# %%
X, y, categorical_indicator, attribute_names = dataset.get_data(
target=dataset.default_target_attribute
)
print(X.head())
print(X.info())
-############################################################################
# %% [markdown]
# Sometimes you only need access to a dataset's metadata.
# In those cases, you can download the dataset without downloading the
@@ -94,13 +83,15 @@
# Starting from 0.15, not downloading data will be the default behavior instead.
# The data will be downloading automatically when you try to access it through
# openml objects, e.g., using `dataset.features`.
-dataset = openml.datasets.get_dataset(1471, download_data=False)
-############################################################################
+# %%
+dataset = openml.datasets.get_dataset(1471)
+
# %% [markdown]
-# Exercise 2
-# **********
+# ## Exercise 2
# * Explore the data visually.
+
+# %%
eegs = eeg.sample(n=1000)
_ = pd.plotting.scatter_matrix(
X.iloc[:100, :4],
@@ -113,19 +104,21 @@
)
-############################################################################
# %% [markdown]
-# Edit a created dataset
-# ======================
+# ## Edit a created dataset
# This example uses the test server, to avoid editing a dataset on the main server.
#
# .. warning::
# .. include:: ../../test_server_usage_warning.txt
+
+# %%
openml.config.start_using_configuration_for_example()
-############################################################################
+# %% [markdown]
# Edit non-critical fields, allowed for all authorized users:
# description, creator, contributor, collection_date, language, citation,
# original_data_url, paper_url
+
+# %%
desc = (
"This data sets consists of 3 different types of irises' "
"(Setosa, Versicolour, and Virginica) petal and sepal length,"
@@ -144,7 +137,6 @@
print(f"Edited dataset ID: {data_id}")
-############################################################################
# %% [markdown]
# Editing critical fields (default_target_attribute, row_id_attribute, ignore_attribute) is allowed
# only for the dataset owner. Further, critical fields cannot be edited if the dataset has any
@@ -152,23 +144,26 @@
# configure the API key:
# openml.config.apikey = 'FILL_IN_OPENML_API_KEY'
# This example here only shows a failure when trying to work on a dataset not owned by you:
+
+# %%
try:
data_id = edit_dataset(1, default_target_attribute="shape")
except openml.exceptions.OpenMLServerException as e:
print(e)
-############################################################################
# %% [markdown]
-# Fork dataset
-# ============
+# ## Fork dataset
# Used to create a copy of the dataset with you as the owner.
# Use this API only if you are unable to edit the critical fields (default_target_attribute,
# ignore_attribute, row_id_attribute) of a dataset through the edit_dataset API.
# After the dataset is forked, you can edit the new version of the dataset using edit_dataset.
+# %%
data_id = fork_dataset(1)
print(data_id)
data_id = edit_dataset(data_id, default_target_attribute="shape")
print(f"Forked dataset ID: {data_id}")
+# %%
openml.config.stop_using_configuration_for_example()
+# License: BSD 3-Clauses
diff --git a/examples/30_extended/fetch_evaluations_tutorial.py b/examples/30_extended/fetch_evaluations_tutorial.py
index 215fbef23..bb0228430 100644
--- a/examples/30_extended/fetch_evaluations_tutorial.py
+++ b/examples/30_extended/fetch_evaluations_tutorial.py
@@ -1,70 +1,61 @@
-"""
# %% [markdown]
-====================
-Fetching Evaluations
-====================
-
-Evaluations contain a concise summary of the results of all runs made. Each evaluation
-provides information on the dataset used, the flow applied, the setup used, the metric
-evaluated, and the result obtained on the metric, for each such run made. These collection
-of results can be used for efficient benchmarking of an algorithm and also allow transparent
-reuse of results from previous experiments on similar parameters.
-
-In this example, we shall do the following:
-
-* Retrieve evaluations based on different metrics
-* Fetch evaluations pertaining to a specific task
-* Sort the obtained results in descending order of the metric
-* Plot a cumulative distribution function for the evaluations
-* Compare the top 10 performing flows based on the evaluation performance
-* Retrieve evaluations with hyperparameter settings
-"""
-
-############################################################################
-
-# License: BSD 3-Clause
-
+# Fetching Evaluations
+
+# Evaluations contain a concise summary of the results of all runs made. Each evaluation
+# provides information on the dataset used, the flow applied, the setup used, the metric
+# evaluated, and the result obtained on the metric, for each such run made. These collection
+# of results can be used for efficient benchmarking of an algorithm and also allow transparent
+# reuse of results from previous experiments on similar parameters.
+#
+# In this example, we shall do the following:
+#
+# * Retrieve evaluations based on different metrics
+# * Fetch evaluations pertaining to a specific task
+# * Sort the obtained results in descending order of the metric
+# * Plot a cumulative distribution function for the evaluations
+# * Compare the top 10 performing flows based on the evaluation performance
+# * Retrieve evaluations with hyperparameter settings
+
+# %%
import openml
-############################################################################
# %% [markdown]
-# Listing evaluations
-# *******************
+# ## Listing evaluations
# Evaluations can be retrieved from the database in the chosen output format.
# Required filters can be applied to retrieve results from runs as required.
# We shall retrieve a small set (only 10 entries) to test the listing function for evaluations
+
+# %%
openml.evaluations.list_evaluations(
- function="predictive_accuracy", size=10, output_format="dataframe"
+ function="predictive_accuracy", size=10
)
# Using other evaluation metrics, 'precision' in this case
evals = openml.evaluations.list_evaluations(
- function="precision", size=10, output_format="dataframe"
+ function="precision", size=10
)
# Querying the returned results for precision above 0.98
print(evals[evals.value > 0.98])
-#############################################################################
# %% [markdown]
-# Viewing a sample task
-# =====================
+# ## Viewing a sample task
# Over here we shall briefly take a look at the details of the task.
-
# We will start by displaying a simple *supervised classification* task:
+
+# %%
task_id = 167140 # https://www.openml.org/t/167140
task = openml.tasks.get_task(task_id)
print(task)
-#############################################################################
# %% [markdown]
-# Obtaining all the evaluations for the task
-# ==========================================
+# ## Obtaining all the evaluations for the task
# We'll now obtain all the evaluations that were uploaded for the task
# we displayed previously.
# Note that we now filter the evaluations based on another parameter 'task'.
+# %%
metric = "predictive_accuracy"
evals = openml.evaluations.list_evaluations(
function=metric, tasks=[task_id], output_format="dataframe"
@@ -76,14 +67,13 @@
print("\nDisplaying head of sorted dataframe: ")
print(evals.head())
-#############################################################################
# %% [markdown]
-# Obtaining CDF of metric for chosen task
-# ***************************************
+# ## Obtaining CDF of metric for chosen task
# We shall now analyse how the performance of various flows have been on this task,
# by seeing the likelihood of the accuracy obtained across all runs.
# We shall now plot a cumulative distributive function (CDF) for the accuracies obtained.
+# %%
from matplotlib import pyplot as plt
@@ -104,17 +94,18 @@ def plot_cdf(values, metric="predictive_accuracy"):
plot_cdf(evals.value, metric)
+
+# %% [markdown]
# This CDF plot shows that for the given task, based on the results of the
# runs uploaded, it is almost certain to achieve an accuracy above 52%, i.e.,
# with non-zero probability. While the maximum accuracy seen till now is 96.5%.
-#############################################################################
# %% [markdown]
-# Comparing top 10 performing flows
-# *********************************
+# ## Comparing top 10 performing flows
# Let us now try to see which flows generally performed the best for this task.
# For this, we shall compare the top performing flows.
+# %%
import numpy as np
import pandas as pd
@@ -147,6 +138,8 @@ def plot_flow_compare(evaluations, top_n=10, metric="predictive_accuracy"):
plot_flow_compare(evals, metric=metric, top_n=10)
+
+# %% [markdown]
# The boxplots below show how the flows perform across multiple runs on the chosen
# task. The green horizontal lines represent the median accuracy of all the runs for
# that flow (number of runs denoted at the bottom of the boxplots). The higher the
@@ -154,36 +147,37 @@ def plot_flow_compare(evaluations, top_n=10, metric="predictive_accuracy"):
# are in the descending order of the higest accuracy value seen under that flow.
# Printing the corresponding flow names for the top 10 performing flow IDs
+
+# %%
top_n = 10
flow_ids = evals.flow_id.unique()[:top_n]
flow_names = evals.flow_name.unique()[:top_n]
for i in range(top_n):
print((flow_ids[i], flow_names[i]))
-#############################################################################
# %% [markdown]
-# Obtaining evaluations with hyperparameter settings
-# ==================================================
+# ## Obtaining evaluations with hyperparameter settings
# We'll now obtain the evaluations of a task and a flow with the hyperparameters
# List evaluations in descending order based on predictive_accuracy with
# hyperparameters
+# %%
evals_setups = openml.evaluations.list_evaluations_setups(
function="predictive_accuracy", tasks=[31], size=100, sort_order="desc"
)
-""
print(evals_setups.head())
-""
+# %% [markdown]
# Return evaluations for flow_id in descending order based on predictive_accuracy
# with hyperparameters. parameters_in_separate_columns returns parameters in
# separate columns
+
+# %%
evals_setups = openml.evaluations.list_evaluations_setups(
function="predictive_accuracy", flows=[6767], size=100, parameters_in_separate_columns=True
)
-""
print(evals_setups.head(10))
-""
+# License: BSD 3-Clause
diff --git a/mkdocs.yml b/mkdocs.yml
index 58b53ca54..a8ab9735e 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -35,4 +35,5 @@ plugins:
- scripts/gen_ref_pages.py
- literate-nav:
nav_file: SUMMARY.md
- - mkdocs-jupyter
+ - mkdocs-jupyter:
+ theme: light
From 957c0bd6224f6d4a5dc9a1ede8e463af661c9572 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 21 Oct 2024 12:56:38 +0200
Subject: [PATCH 21/25] Convert remaining files to jupytext
---
.../30_extended/fetch_evaluations_tutorial.py | 1 +
.../30_extended/fetch_runtimes_tutorial.py | 157 ++++++++----------
examples/30_extended/flow_id_tutorial.py | 50 +++---
.../30_extended/flows_and_runs_tutorial.py | 54 +++---
.../plot_svm_hyperparameters_tutorial.py | 36 ++--
examples/30_extended/run_setup_tutorial.py | 80 +++++----
examples/30_extended/study_tutorial.py | 50 +++---
examples/30_extended/suites_tutorial.py | 63 ++++---
.../task_manual_iteration_tutorial.py | 45 ++---
examples/30_extended/tasks_tutorial.py | 78 ++++-----
.../40_paper/2015_neurips_feurer_example.py | 40 +++--
examples/40_paper/2018_ida_strang_example.py | 39 ++---
examples/40_paper/2018_kdd_rijn_example.py | 38 ++---
.../40_paper/2018_neurips_perrone_example.py | 73 ++++----
14 files changed, 370 insertions(+), 434 deletions(-)
diff --git a/examples/30_extended/fetch_evaluations_tutorial.py b/examples/30_extended/fetch_evaluations_tutorial.py
index bb0228430..0c735c235 100644
--- a/examples/30_extended/fetch_evaluations_tutorial.py
+++ b/examples/30_extended/fetch_evaluations_tutorial.py
@@ -161,6 +161,7 @@ def plot_flow_compare(evaluations, top_n=10, metric="predictive_accuracy"):
# List evaluations in descending order based on predictive_accuracy with
# hyperparameters
+
# %%
evals_setups = openml.evaluations.list_evaluations_setups(
function="predictive_accuracy", tasks=[31], size=100, sort_order="desc"
diff --git a/examples/30_extended/fetch_runtimes_tutorial.py b/examples/30_extended/fetch_runtimes_tutorial.py
index 39995e25f..c4111a324 100644
--- a/examples/30_extended/fetch_runtimes_tutorial.py
+++ b/examples/30_extended/fetch_runtimes_tutorial.py
@@ -1,52 +1,43 @@
-"""
# %% [markdown]
-
-==========================================
-Measuring runtimes for Scikit-learn models
-==========================================
-
-The runtime of machine learning models on specific datasets can be a deciding
-factor on the choice of algorithms, especially for benchmarking and comparison
-purposes. OpenML's scikit-learn extension provides runtime data from runs of
-model fit and prediction on tasks or datasets, for both the CPU-clock as well
-as the actual wallclock-time incurred. The objective of this example is to
-illustrate how to retrieve such timing measures, and also offer some potential
-means of usage and interpretation of the same.
-
-It should be noted that there are multiple levels at which parallelism can occur.
-
-* At the outermost level, OpenML tasks contain fixed data splits, on which the
- defined model/flow is executed. Thus, a model can be fit on each OpenML dataset fold
- in parallel using the `n_jobs` parameter to `run_model_on_task` or `run_flow_on_task`
- (illustrated under Case 2 & 3 below).
-
-* The model/flow specified can also include scikit-learn models that perform their own
- parallelization. For instance, by specifying `n_jobs` in a Random Forest model definition
- (covered under Case 2 below).
-
-* The sklearn model can further be an HPO estimator and contain it's own parallelization.
- If the base estimator used also supports `parallelization`, then there's at least a 2-level nested
- definition for parallelization possible (covered under Case 3 below).
-
-We shall cover these 5 representative scenarios for:
-
-* (Case 1) Retrieving runtimes for Random Forest training and prediction on each of the
- cross-validation folds
-
-* (Case 2) Testing the above setting in a parallel setup and monitor the difference using
- runtimes retrieved
-
-* (Case 3) Comparing RandomSearchCV and GridSearchCV on the above task based on runtimes
-
-* (Case 4) Running models that don't run in parallel or models which scikit-learn doesn't
- parallelize
-
-* (Case 5) Running models that do not release the Python Global Interpreter Lock (GIL)
-"""
-
-############################################################################
-
-# License: BSD 3-Clause
+# Measuring runtimes for Scikit-learn models
+#
+# The runtime of machine learning models on specific datasets can be a deciding
+# factor on the choice of algorithms, especially for benchmarking and comparison
+# purposes. OpenML's scikit-learn extension provides runtime data from runs of
+# model fit and prediction on tasks or datasets, for both the CPU-clock as well
+# as the actual wallclock-time incurred. The objective of this example is to
+# illustrate how to retrieve such timing measures, and also offer some potential
+# means of usage and interpretation of the same.
+#
+# It should be noted that there are multiple levels at which parallelism can occur.
+#
+# * At the outermost level, OpenML tasks contain fixed data splits, on which the
+# defined model/flow is executed. Thus, a model can be fit on each OpenML dataset fold
+# in parallel using the `n_jobs` parameter to `run_model_on_task` or `run_flow_on_task`
+# (illustrated under Case 2 & 3 below).
+#
+# * The model/flow specified can also include scikit-learn models that perform their own
+# parallelization. For instance, by specifying `n_jobs` in a Random Forest model definition
+# (covered under Case 2 below).
+#
+# * The sklearn model can further be an HPO estimator and contain it's own parallelization.
+# If the base estimator used also supports `parallelization`, then there's at least a 2-level nested
+# definition for parallelization possible (covered under Case 3 below).
+#
+# We shall cover these 5 representative scenarios for:
+#
+# * (Case 1) Retrieving runtimes for Random Forest training and prediction on each of the
+# cross-validation folds
+#
+# * (Case 2) Testing the above setting in a parallel setup and monitor the difference using
+# runtimes retrieved
+#
+# * (Case 3) Comparing RandomSearchCV and GridSearchCV on the above task based on runtimes
+#
+# * (Case 4) Running models that don't run in parallel or models which scikit-learn doesn't
+# parallelize
+#
+# * (Case 5) Running models that do not release the Python Global Interpreter Lock (GIL)
import openml
import numpy as np
@@ -60,11 +51,10 @@
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
-############################################################################
# %% [markdown]
-# Preparing tasks and scikit-learn models
-# ***************************************
+# # Preparing tasks and scikit-learn models
+# %%
task_id = 167119
task = openml.tasks.get_task(task_id)
@@ -93,14 +83,13 @@ def print_compare_runtimes(measures):
)
-############################################################################
# %% [markdown]
-# Case 1: Running a Random Forest model on an OpenML task
-# *******************************************************
+# # Case 1: Running a Random Forest model on an OpenML task
# We'll run a Random Forest model and obtain an OpenML run object. We can
# see the evaluations recorded per fold for the dataset and the information
# available for this run.
+# %%
clf = RandomForestClassifier(n_estimators=10)
run1 = openml.runs.run_model_on_task(
@@ -125,7 +114,6 @@ def print_compare_runtimes(measures):
print("Repeat #{}-Fold #{}: {:.4f}".format(repeat, fold, val2))
print()
-################################################################################
# %% [markdown]
# The remaining entries recorded in `measures` are the runtime records
# related as:
@@ -142,14 +130,15 @@ def print_compare_runtimes(measures):
# follows the same procedure but for time taken for the `.predict()` procedure.
# Comparing the CPU and wall-clock training times of the Random Forest model
+
+# %%
print_compare_runtimes(measures)
-######################################################################
# %% [markdown]
-# Case 2: Running Scikit-learn model on an OpenML task in parallel
-# ****************************************************************
+# ## Case 2: Running Scikit-learn model on an OpenML task in parallel
# Redefining the model to allow parallelism with `n_jobs=2` (2 cores)
+# %%
clf = RandomForestClassifier(n_estimators=10, n_jobs=2)
run2 = openml.runs.run_model_on_task(
@@ -159,10 +148,10 @@ def print_compare_runtimes(measures):
# The wall-clock time recorded per fold should be lesser than Case 1 above
print_compare_runtimes(measures)
-####################################################################################
# %% [markdown]
# Running a Random Forest model on an OpenML task in parallel (all cores available):
+# %%
# Redefining the model to use all available cores with `n_jobs=-1`
clf = RandomForestClassifier(n_estimators=10, n_jobs=-1)
@@ -170,12 +159,13 @@ def print_compare_runtimes(measures):
model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False
)
measures = run3.fold_evaluations
+
+# %% [markdown]
# The wall-clock time recorded per fold should be lesser than the case above,
# if more than 2 CPU cores are available. The speed-up is more pronounced for
# larger datasets.
print_compare_runtimes(measures)
-####################################################################################
# %% [markdown]
# We can now observe that the ratio of CPU time to wallclock time is lower
# than in case 1. This happens because joblib by default spawns subprocesses
@@ -183,13 +173,13 @@ def print_compare_runtimes(measures):
# the reported CPU and wallclock time requires knowledge of the parallelization
# applied at runtime.
-####################################################################################
# %% [markdown]
# Running the same task with a different parallel backend. Joblib provides multiple
# backends: {`loky` (default), `multiprocessing`, `dask`, `threading`, `sequential`}.
# The backend can be explicitly set using a joblib context manager. The behaviour of
# the job distribution can change and therefore the scale of runtimes recorded too.
+# %%
with parallel_backend(backend="multiprocessing", n_jobs=-1):
run3_ = openml.runs.run_model_on_task(
model=clf, task=task, upload_flow=False, avoid_duplicate_runs=False
@@ -197,7 +187,6 @@ def print_compare_runtimes(measures):
measures = run3_.fold_evaluations
print_compare_runtimes(measures)
-####################################################################################
# %% [markdown]
# The CPU time interpretation becomes ambiguous when jobs are distributed over an
# unknown number of cores or when subprocesses are spawned for which the CPU time
@@ -207,10 +196,8 @@ def print_compare_runtimes(measures):
# cases that can arise as demonstrated in the rest of the example. Therefore,
# the final interpretation of the runtimes is left to the `user`.
-#####################################################################
# %% [markdown]
-# Case 3: Running and benchmarking HPO algorithms with their runtimes
-# *******************************************************************
+# ## Case 3: Running and benchmarking HPO algorithms with their runtimes
# We shall now optimize a similar RandomForest model for the same task using
# scikit-learn's HPO support by using GridSearchCV to optimize our earlier
# RandomForest model's hyperparameter `n_estimators`. Scikit-learn also provides a
@@ -218,9 +205,9 @@ def print_compare_runtimes(measures):
# and evaluating the model on the best found parameter setting. This is
# included in the `wall_clock_time_millis_training` measure recorded.
+# %%
from sklearn.model_selection import GridSearchCV
-
clf = RandomForestClassifier(n_estimators=10, n_jobs=2)
# GridSearchCV model
@@ -238,7 +225,6 @@ def print_compare_runtimes(measures):
measures = run4.fold_evaluations
print_compare_runtimes(measures)
-##################################################################################
# %% [markdown]
# Like any optimisation problem, scikit-learn's HPO estimators also generate
# a sequence of configurations which are evaluated, using which the best found
@@ -252,18 +238,19 @@ def print_compare_runtimes(measures):
# is for the entire `fit()` procedure of GridSearchCV thus subsuming the runtimes of
# the 2-fold (inner) CV search performed.
+# %%
# We earlier extracted the number of repeats and folds for this task:
print("# repeats: {}\n# folds: {}".format(n_repeats, n_folds))
# To extract the training runtime of the first repeat, first fold:
print(run4.fold_evaluations["wall_clock_time_millis_training"][0][0])
-##################################################################################
# %% [markdown]
# To extract the training runtime of the 1-st repeat, 4-th (outer) fold and also
# to fetch the parameters and performance of the evaluations made during
# the 1-st repeat, 4-th fold evaluation by the Grid Search model.
+# %%
_repeat = 0
_fold = 3
print(
@@ -280,7 +267,6 @@ def print_compare_runtimes(measures):
)
)
-##################################################################################
# %% [markdown]
# Scikit-learn's HPO estimators also come with an argument `refit=True` as a default.
# In our previous model definition it was set to True by default, which meant that the best
@@ -296,6 +282,8 @@ def print_compare_runtimes(measures):
# This refit time can therefore be explicitly extracted in this manner:
+# %%
+
def extract_refit_time(run, repeat, fold):
refit_time = (
run.fold_evaluations["wall_clock_time_millis"][repeat][fold]
@@ -313,13 +301,13 @@ def extract_refit_time(run, repeat, fold):
)
)
-#########################################################
# %% [markdown]
# Along with the GridSearchCV already used above, we demonstrate how such
# optimisation traces can be retrieved by showing an application of these
# traces - comparing the speed of finding the best configuration using
# RandomizedSearchCV and GridSearchCV available with scikit-learn.
+# %%
# RandomizedSearchCV model
rs_pipe = RandomizedSearchCV(
estimator=clf,
@@ -334,7 +322,6 @@ def extract_refit_time(run, repeat, fold):
model=rs_pipe, task=task, upload_flow=False, avoid_duplicate_runs=False, n_jobs=2
)
-################################################################################
# %% [markdown]
# Since for the call to ``openml.runs.run_model_on_task`` the parameter
# ``n_jobs`` is set to its default ``None``, the evaluations across the OpenML folds
@@ -349,6 +336,7 @@ def extract_refit_time(run, repeat, fold):
# the runtimes per fold can be cumulatively added to plot the trace against time.
+# %%
def extract_trace_data(run, n_repeats, n_folds, n_iter, key=None):
key = "wall_clock_time_millis_training" if key is None else key
data = {"score": [], "runtime": []}
@@ -391,10 +379,8 @@ def get_incumbent_trace(trace):
plt.legend()
plt.show()
-################################################################################
# %% [markdown]
-# Case 4: Running models that scikit-learn doesn't parallelize
-# *************************************************************
+# ## Case 4: Running models that scikit-learn doesn't parallelize
# Both scikit-learn and OpenML depend on parallelism implemented through `joblib`.
# However, there can be cases where either models cannot be parallelized or don't
# depend on joblib for its parallelism. 2 such cases are illustrated below.
@@ -402,6 +388,7 @@ def get_incumbent_trace(trace):
# Running a Decision Tree model that doesn't support parallelism implicitly, but
# using OpenML to parallelize evaluations for the outer-cross validation folds.
+# %%
dt = DecisionTreeClassifier()
run6 = openml.runs.run_model_on_task(
@@ -410,12 +397,12 @@ def get_incumbent_trace(trace):
measures = run6.fold_evaluations
print_compare_runtimes(measures)
-################################################################################
# %% [markdown]
# Although the decision tree does not run in parallel, it can release the
# `Python GIL <https://docs.python.org/dev/glossary.html#term-global-interpreter-lock>`_.
# This can result in surprising runtime measures as demonstrated below:
+# %%
with parallel_backend("threading", n_jobs=-1):
run7 = openml.runs.run_model_on_task(
model=dt, task=task, upload_flow=False, avoid_duplicate_runs=False
@@ -423,12 +410,12 @@ def get_incumbent_trace(trace):
measures = run7.fold_evaluations
print_compare_runtimes(measures)
-################################################################################
# %% [markdown]
# Running a Neural Network from scikit-learn that uses scikit-learn independent
-# parallelism using libraries such as `MKL, OpenBLAS or BLIS
-# <https://scikit-learn.org/stable/computing/parallelism.html#parallel-numpy-and-scipy-routines-from-numerical-libraries>`_.
+# parallelism using libraries such as
+# [MKL, OpenBLAS or BLIS](https://scikit-learn.org/stable/computing/parallelism.html#parallel-numpy-and-scipy-routines-from-numerical-libraries>).
+# %%
mlp = MLPClassifier(max_iter=10)
run8 = openml.runs.run_model_on_task(
@@ -437,16 +424,15 @@ def get_incumbent_trace(trace):
measures = run8.fold_evaluations
print_compare_runtimes(measures)
-################################################################################
# %% [markdown]
-# Case 5: Running Scikit-learn models that don't release GIL
-# **********************************************************
-# Certain Scikit-learn models do not release the `Python GIL
-# <https://docs.python.org/dev/glossary.html#term-global-interpreter-lock>`_ and
+# ## Case 5: Running Scikit-learn models that don't release GIL
+# Certain Scikit-learn models do not release the
+# [Python GIL](https://docs.python.org/dev/glossary.html#term-global-interpreter-lock) and
# are also not executed in parallel via a BLAS library. In such cases, the
# CPU times and wallclock times are most likely trustworthy. Note however
# that only very few models such as naive Bayes models are of this kind.
+# %%
clf = GaussianNB()
with parallel_backend("multiprocessing", n_jobs=-1):
@@ -456,10 +442,8 @@ def get_incumbent_trace(trace):
measures = run9.fold_evaluations
print_compare_runtimes(measures)
-################################################################################
# %% [markdown]
-# Summmary
-# *********
+# ## Summmary
# The scikit-learn extension for OpenML-Python records model runtimes for the
# CPU-clock and the wall-clock times. The above examples illustrated how these
# recorded runtimes can be extracted when using a scikit-learn model and under
@@ -504,3 +488,4 @@ def get_incumbent_trace(trace):
#
# Because of all the cases mentioned above it is crucial to understand which case is triggered
# when reporting runtimes for scikit-learn models measured with OpenML-Python!
+# License: BSD 3-Clause
diff --git a/examples/30_extended/flow_id_tutorial.py b/examples/30_extended/flow_id_tutorial.py
index 5abab7704..e0874d893 100644
--- a/examples/30_extended/flow_id_tutorial.py
+++ b/examples/30_extended/flow_id_tutorial.py
@@ -1,44 +1,34 @@
-"""
# %% [markdown]
-==================
-Obtaining Flow IDs
-==================
+# # Obtaining Flow IDs
+# This tutorial discusses different ways to obtain the ID of a flow in order to perform further
+# analysis.
-This tutorial discusses different ways to obtain the ID of a flow in order to perform further
-analysis.
-"""
-
-####################################################################################################
-
-# License: BSD 3-Clause
+# %%
import sklearn.tree
import openml
-############################################################################
# %% [markdown]
# .. warning::
# .. include:: ../../test_server_usage_warning.txt
-openml.config.start_using_configuration_for_example()
+# %%
+openml.config.start_using_configuration_for_example()
-############################################################################
+# %%
# Defining a classifier
clf = sklearn.tree.DecisionTreeClassifier()
-####################################################################################################
# %% [markdown]
-# 1. Obtaining a flow given a classifier
-# ======================================
-#
+# ## 1. Obtaining a flow given a classifier
+# %%
flow = openml.extensions.get_extension_by_model(clf).model_to_flow(clf).publish()
flow_id = flow.flow_id
print(flow_id)
-####################################################################################################
# %% [markdown]
# This piece of code is rather involved. First, it retrieves a
# :class:`~openml.extensions.Extension` which is registered and can handle the given model,
@@ -50,42 +40,46 @@
#
# To simplify the usage we have created a helper function which automates all these steps:
+# %%
flow_id = openml.flows.get_flow_id(model=clf)
print(flow_id)
-####################################################################################################
# %% [markdown]
-# 2. Obtaining a flow given its name
-# ==================================
-# The schema of a flow is given in XSD (`here
-# <https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/openml.implementation.upload.xsd>`_). # noqa E501
+# ## 2. Obtaining a flow given its name
+# The schema of a flow is given in XSD (
+# [here](https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/openml.implementation.upload.xsd)). # noqa E501
# Only two fields are required, a unique name, and an external version. While it should be pretty
# obvious why we need a name, the need for the additional external version information might not
# be immediately clear. However, this information is very important as it allows to have multiple
# flows with the same name for different versions of a software. This might be necessary if an
# algorithm or implementation introduces, renames or drop hyperparameters over time.
+# %%
print(flow.name, flow.external_version)
-####################################################################################################
# %% [markdown]
# The name and external version are automatically added to a flow when constructing it from a
# model. We can then use them to retrieve the flow id as follows:
+# %%
flow_id = openml.flows.flow_exists(name=flow.name, external_version=flow.external_version)
print(flow_id)
-####################################################################################################
# %% [markdown]
# We can also retrieve all flows for a given name:
+
+# %%
flow_ids = openml.flows.get_flow_id(name=flow.name)
print(flow_ids)
-####################################################################################################
# %% [markdown]
# This also works with the actual model (generalizing the first part of this example):
+
+# %%
flow_ids = openml.flows.get_flow_id(model=clf, exact_version=False)
print(flow_ids)
-# Deactivating test server
+# %%
+# Deactivating test configuration
openml.config.stop_using_configuration_for_example()
+# License: BSD 3-Clause
diff --git a/examples/30_extended/flows_and_runs_tutorial.py b/examples/30_extended/flows_and_runs_tutorial.py
index 2b23196ae..0b60c9b14 100644
--- a/examples/30_extended/flows_and_runs_tutorial.py
+++ b/examples/30_extended/flows_and_runs_tutorial.py
@@ -1,32 +1,27 @@
-"""
# %% [markdown]
-Flows and Runs
-==============
-
-How to train/run a model and how to upload the results.
-"""
-
-# License: BSD 3-Clause
+# #Flows and Runs
+# This tutorial covers how to train/run a model and how to upload the results.
+# %%
import openml
from sklearn import compose, ensemble, impute, neighbors, preprocessing, pipeline, tree
-############################################################################
# %% [markdown]
# We'll use the test server for the rest of this tutorial.
#
# .. warning::
# .. include:: ../../test_server_usage_warning.txt
+
+# %%
openml.config.start_using_configuration_for_example()
-############################################################################
# %% [markdown]
-# Train machine learning models
-# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+# ## Train machine learning models
#
# Train a scikit-learn model on the data manually.
+# %%
# NOTE: We are using dataset 68 from the test server: https://test.openml.org/d/68
dataset = openml.datasets.get_dataset(68)
X, y, categorical_indicator, attribute_names = dataset.get_data(
@@ -35,11 +30,12 @@
clf = neighbors.KNeighborsClassifier(n_neighbors=1)
clf.fit(X, y)
-############################################################################
# %% [markdown]
# You can also ask for meta-data to automatically preprocess the data.
#
# * e.g. categorical features -> do feature encoding
+
+# %%
dataset = openml.datasets.get_dataset(17)
X, y, categorical_indicator, attribute_names = dataset.get_data(
target=dataset.default_target_attribute
@@ -51,12 +47,11 @@
X = transformer.fit_transform(X)
clf.fit(X, y)
-############################################################################
# %% [markdown]
-# Runs: Easily explore models
-# ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+# ## Runs: Easily explore models
# We can run (many) scikit-learn algorithms on (many) OpenML tasks.
+# %%
# Get a task
task = openml.tasks.get_task(403)
@@ -68,34 +63,34 @@
print(run)
-############################################################################
# %% [markdown]
# Share the run on the OpenML server
#
# So far the run is only available locally. By calling the publish function,
# the run is sent to the OpenML server:
+# %%
myrun = run.publish()
# For this tutorial, our configuration publishes to the test server
# as to not pollute the main server.
print(f"Uploaded to {myrun.openml_url}")
-############################################################################
# %% [markdown]
# We can now also inspect the flow object which was automatically created:
+# %%
flow = openml.flows.get_flow(run.flow_id)
print(flow)
-############################################################################
# %% [markdown]
-# It also works with pipelines
-# ############################
+# ## It also works with pipelines
#
# When you need to handle 'dirty' data, build pipelines to model then automatically.
# To demonstrate this using the dataset `credit-a <https://test.openml.org/d/16>`_ via
# `task <https://test.openml.org/t/96>`_ as it contains both numerical and categorical
# variables and missing values in both.
+
+# %%
task = openml.tasks.get_task(96)
# OpenML helper functions for sklearn can be plugged in directly for complicated pipelines
@@ -134,6 +129,7 @@
# In the case, pandas is not available, or a NumPy based data processing is the requirement, the
# above pipeline is presented below to work with NumPy.
+# %%
# Extracting the indices of the categorical columns
features = task.get_dataset().features
categorical_feature_indices = []
@@ -173,15 +169,15 @@
myrun = run.publish()
print(f"Uploaded to {myrun.openml_url}")
-###############################################################################
# %% [markdown]
-# Running flows on tasks offline for later upload
-# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+# ## Running flows on tasks offline for later upload
# For those scenarios where there is no access to internet, it is possible to run
# a model on a task without uploading results or flows to the server immediately.
# To perform the following line offline, it is required to have been called before
# such that the task is cached on the local openml cache directory:
+
+# %%
task = openml.tasks.get_task(96)
# The following lines can then be executed offline:
@@ -202,10 +198,10 @@
# Publishing the run will automatically upload the related flow if
# it does not yet exist on the server.
-############################################################################
# %% [markdown]
# Alternatively, one can also directly run flows.
+# %%
# Get a task
task = openml.tasks.get_task(403)
@@ -219,10 +215,8 @@
run = openml.runs.run_flow_on_task(flow, task)
-############################################################################
# %% [markdown]
-# Challenge
-# ^^^^^^^^^
+# ## Challenge
#
# Try to build the best possible models on several OpenML tasks,
# compare your results with the rest of the class and learn from
@@ -239,6 +233,7 @@
# * Higgs (Physics): data_id:`23512 <https://www.openml.org/d/23512>`_,
# task_id:`52950 <https://www.openml.org/t/52950>`_, 100k instances, missing values.
+# %%
# Easy benchmarking:
for task_id in [115]: # Add further tasks. Disclaimer: they might take some time
task = openml.tasks.get_task(task_id)
@@ -250,5 +245,6 @@
print(f"kNN on {data.name}: {myrun.openml_url}")
-############################################################################
+# %%
openml.config.stop_using_configuration_for_example()
+# License: BSD 3-Clause
diff --git a/examples/30_extended/plot_svm_hyperparameters_tutorial.py b/examples/30_extended/plot_svm_hyperparameters_tutorial.py
index 96e13ed70..bd44adadf 100644
--- a/examples/30_extended/plot_svm_hyperparameters_tutorial.py
+++ b/examples/30_extended/plot_svm_hyperparameters_tutorial.py
@@ -1,25 +1,20 @@
-"""
# %% [markdown]
-================================
-Plotting hyperparameter surfaces
-================================
-"""
-
-# License: BSD 3-Clause
+# # Plotting hyperparameter surfaces
+# %%
import openml
import numpy as np
-####################################################################################################
# %% [markdown]
-# First step - obtaining the data
-# ===============================
+# # First step - obtaining the data
# First, we need to choose an SVM flow, for example 8353, and a task. Finding the IDs of them are
# not part of this tutorial, this could for example be done via the website.
#
# For this we use the function ``list_evaluations_setup`` which can automatically join
# evaluations conducted by the server with the hyperparameter settings extracted from the
# uploaded runs (called *setup*).
+
+# %%
df = openml.evaluations.list_evaluations_setups(
function="predictive_accuracy",
flows=[8353],
@@ -31,24 +26,25 @@
)
print(df.head(n=10))
-####################################################################################################
# %% [markdown]
# We can see all the hyperparameter names in the columns of the dataframe:
+
+# %%
for name in df.columns:
print(name)
-####################################################################################################
# %% [markdown]
# Next, we cast and transform the hyperparameters of interest (``C`` and ``gamma``) so that we
# can nicely plot them.
+
+# %%
hyperparameters = ["sklearn.svm.classes.SVC(16)_C", "sklearn.svm.classes.SVC(16)_gamma"]
df[hyperparameters] = df[hyperparameters].astype(float).apply(np.log10)
-####################################################################################################
# %% [markdown]
-# Option 1 - plotting via the pandas helper functions
-# ===================================================
-#
+# ## Option 1 - plotting via the pandas helper functions
+
+# %%
df.plot.hexbin(
x="sklearn.svm.classes.SVC(16)_C",
y="sklearn.svm.classes.SVC(16)_gamma",
@@ -58,11 +54,10 @@
title="SVM performance landscape",
)
-####################################################################################################
# %% [markdown]
-# Option 2 - plotting via matplotlib
-# ==================================
-#
+# ## Option 2 - plotting via matplotlib
+
+# %%
import matplotlib.pyplot as plt
fig, ax = plt.subplots()
@@ -85,3 +80,4 @@
ylabel="gamma (log10)",
)
ax.set_title("SVM performance landscape")
+# License: BSD 3-Clause
diff --git a/examples/30_extended/run_setup_tutorial.py b/examples/30_extended/run_setup_tutorial.py
index 087be849f..e0aad78f4 100644
--- a/examples/30_extended/run_setup_tutorial.py
+++ b/examples/30_extended/run_setup_tutorial.py
@@ -1,33 +1,26 @@
-"""
# %% [markdown]
-=========
-Run Setup
-=========
-
-By: Jan N. van Rijn
-
-One of the key features of the openml-python library is that is allows to
-reinstantiate flows with hyperparameter settings that were uploaded before.
-This tutorial uses the concept of setups. Although setups are not extensively
-described in the OpenML documentation (because most users will not directly
-use them), they form a important concept within OpenML distinguishing between
-hyperparameter configurations.
-A setup is the combination of a flow with all its hyperparameters set.
-
-A key requirement for reinstantiating a flow is to have the same scikit-learn
-version as the flow that was uploaded. However, this tutorial will upload the
-flow (that will later be reinstantiated) itself, so it can be ran with any
-scikit-learn version that is supported by this library. In this case, the
-requirement of the corresponding scikit-learn versions is automatically met.
-
-In this tutorial we will
- 1) Create a flow and use it to solve a task;
- 2) Download the flow, reinstantiate the model with same hyperparameters,
- and solve the same task again;
- 3) We will verify that the obtained results are exactly the same.
-"""
-
-# License: BSD 3-Clause
+# # Run Setup
+# One of the key features of the openml-python library is that is allows to
+# reinstantiate flows with hyperparameter settings that were uploaded before.
+# This tutorial uses the concept of setups. Although setups are not extensively
+# described in the OpenML documentation (because most users will not directly
+# use them), they form a important concept within OpenML distinguishing between
+# hyperparameter configurations.
+# A setup is the combination of a flow with all its hyperparameters set.
+#
+# A key requirement for reinstantiating a flow is to have the same scikit-learn
+# version as the flow that was uploaded. However, this tutorial will upload the
+# flow (that will later be reinstantiated) itself, so it can be ran with any
+# scikit-learn version that is supported by this library. In this case, the
+# requirement of the corresponding scikit-learn versions is automatically met.
+#
+# In this tutorial we will
+# 1) Create a flow and use it to solve a task;
+# 2) Download the flow, reinstantiate the model with same hyperparameters,
+# and solve the same task again;
+# 3) We will verify that the obtained results are exactly the same.
+
+# %%
import numpy as np
import openml
@@ -40,26 +33,28 @@
from sklearn.ensemble import RandomForestClassifier
from sklearn.decomposition import TruncatedSVD
-############################################################################
# %% [markdown]
# .. warning::
# .. include:: ../../test_server_usage_warning.txt
+
+# %%
openml.config.start_using_configuration_for_example()
-###############################################################################
# %% [markdown]
# 1) Create a flow and use it to solve a task
-###############################################################################
-# first, let's download the task that we are interested in
-task = openml.tasks.get_task(6)
+# First, let's download the task that we are interested in
+# %%
+task = openml.tasks.get_task(6)
+# %% [markdown]
# we will create a fairly complex model, with many preprocessing components and
# many potential hyperparameters. Of course, the model can be as complex and as
# easy as you want it to be
+# %%
cat_imp = make_pipeline(
OneHotEncoder(handle_unknown="ignore", sparse=False),
TruncatedSVD(),
@@ -78,6 +73,8 @@
# would tune them using, e.g., Random Search or Bayesian Optimization, but for
# the purpose of this tutorial we set them to some specific values that might
# or might not be optimal
+
+# %%
hyperparameters_original = {
"estimator__criterion": "gini",
"estimator__n_estimators": 50,
@@ -90,11 +87,10 @@
run = openml.runs.run_model_on_task(model_original, task, avoid_duplicate_runs=False)
run_original = run.publish() # this implicitly uploads the flow
-###############################################################################
# %% [markdown]
-# 2) Download the flow and solve the same task again.
-###############################################################################
+# ## 2) Download the flow and solve the same task again.
+# %%
# obtain setup id (note that the setup id is assigned by the OpenML server -
# therefore it was not yet available in our local copy of the run)
run_downloaded = openml.runs.get_run(run_original.run_id)
@@ -108,14 +104,16 @@
run_duplicate = openml.runs.run_model_on_task(model_duplicate, task, avoid_duplicate_runs=False)
-###############################################################################
# %% [markdown]
-# 3) We will verify that the obtained results are exactly the same.
-###############################################################################
+# ## 3) We will verify that the obtained results are exactly the same.
+# %%
# the run has stored all predictions in the field data content
np.testing.assert_array_equal(run_original.data_content, run_duplicate.data_content)
-###############################################################################
+# %%
openml.config.stop_using_configuration_for_example()
+
+# By: Jan N. van Rijn
+# License: BSD 3-Clause
diff --git a/examples/30_extended/study_tutorial.py b/examples/30_extended/study_tutorial.py
index 392eb9abc..363b5b305 100644
--- a/examples/30_extended/study_tutorial.py
+++ b/examples/30_extended/study_tutorial.py
@@ -1,17 +1,11 @@
-"""
# %% [markdown]
-=================
-Benchmark studies
-=================
-How to list, download and upload benchmark studies.
-In contrast to `benchmark suites <https://docs.openml.org/benchmark/#benchmarking-suites>`_ which
-hold a list of tasks, studies hold a list of runs. As runs contain all information on flows and
-tasks, all required information about a study can be retrieved.
-"""
-############################################################################
-
-# License: BSD 3-Clause
+# # Benchmark studies
+# How to list, download and upload benchmark studies.
+# In contrast to `benchmark suites <https://docs.openml.org/benchmark/#benchmarking-suites>`_ which
+# hold a list of tasks, studies hold a list of runs. As runs contain all information on flows and
+# tasks, all required information about a study can be retrieved.
+# %%
import uuid
from sklearn.ensemble import RandomForestClassifier
@@ -19,44 +13,45 @@
import openml
-############################################################################
# %% [markdown]
-# Listing studies
-# ***************
+# ## Listing studies
#
# * Use the output_format parameter to select output type
# * Default gives ``dict``, but we'll use ``dataframe`` to obtain an
# easier-to-work-with data structure
+# %%
studies = openml.study.list_studies(output_format="dataframe", status="all")
print(studies.head(n=10))
-############################################################################
# %% [markdown]
-# Downloading studies
-# ===================
+# ## Downloading studies
-############################################################################
# %% [markdown]
# This is done based on the study ID.
+
+# %%
study = openml.study.get_study(123)
print(study)
-############################################################################
# %% [markdown]
# Studies also features a description:
+
+# %%
print(study.description)
-############################################################################
# %% [markdown]
# Studies are a container for runs:
+
+# %%
print(study.runs)
-############################################################################
# %% [markdown]
# And we can use the evaluation listing functionality to learn more about
# the evaluations available for the conducted runs:
+
+# %%
evaluations = openml.evaluations.list_evaluations(
function="predictive_accuracy",
output_format="dataframe",
@@ -64,23 +59,23 @@
)
print(evaluations.head())
-############################################################################
# %% [markdown]
# We'll use the test server for the rest of this tutorial.
#
# .. warning::
# .. include:: ../../test_server_usage_warning.txt
+
+# %%
openml.config.start_using_configuration_for_example()
-############################################################################
# %% [markdown]
-# Uploading studies
-# =================
+# ## Uploading studies
#
# Creating a study is as simple as creating any kind of other OpenML entity.
# In this examples we'll create a few runs for the OpenML-100 benchmark
# suite which is available on the OpenML test server.
+# %%
# Model to be used
clf = RandomForestClassifier()
@@ -113,5 +108,6 @@
print(new_study)
-############################################################################
+# %%
openml.config.stop_using_configuration_for_example()
+# License: BSD 3-Clause
diff --git a/examples/30_extended/suites_tutorial.py b/examples/30_extended/suites_tutorial.py
index 1e1786cbb..47d7b30b9 100644
--- a/examples/30_extended/suites_tutorial.py
+++ b/examples/30_extended/suites_tutorial.py
@@ -1,19 +1,13 @@
-"""
# %% [markdown]
-================
-Benchmark suites
-================
-
-How to list, download and upload benchmark suites.
-
-If you want to learn more about benchmark suites, check out our
-brief introductory tutorial :ref:`sphx_glr_examples_20_basic_simple_suites_tutorial.py` or the
-`OpenML benchmark docs <https://docs.openml.org/benchmark/#benchmarking-suites>`_.
-"""
-############################################################################
-
-# License: BSD 3-Clause
+# # Benchmark suites
+#
+# How to list, download and upload benchmark suites.
+#
+# If you want to learn more about benchmark suites, check out our
+# brief introductory tutorial :ref:`sphx_glr_examples_20_basic_simple_suites_tutorial.py` or the
+# [OpenML benchmark docs](https://docs.openml.org/benchmark/#benchmarking-suites).
+# %%
import uuid
import numpy as np
@@ -21,62 +15,65 @@
import openml
-############################################################################
# %% [markdown]
-# Listing suites
-# **************
+# ## Listing suites
#
# * Use the output_format parameter to select output type
# * Default gives ``dict``, but we'll use ``dataframe`` to obtain an
# easier-to-work-with data structure
+# %%
suites = openml.study.list_suites(output_format="dataframe", status="all")
print(suites.head(n=10))
-############################################################################
# %% [markdown]
-# Downloading suites
-# ==================
+# ## Downloading suites
-############################################################################
# %% [markdown]
# This is done based on the dataset ID.
+
+# %%
suite = openml.study.get_suite(99)
print(suite)
-#############################################################################
# %% [markdown]
# Suites also feature a description:
+
+# %%
print(suite.description)
-############################################################################
# %% [markdown]
# Suites are a container for tasks:
+
+# %%
print(suite.tasks)
-############################################################################
# %% [markdown]
# And we can use the task listing functionality to learn more about them:
+
+# %%
tasks = openml.tasks.list_tasks(output_format="dataframe")
-# Using ``@`` in `pd.DataFrame.query <
-# https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.query.html>`_
+# %% [markdown]
+# Using ``@`` in
+# [pd.DataFrame.query](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.query.html)
# accesses variables outside of the current dataframe.
+
+# %%
tasks = tasks.query("tid in @suite.tasks")
print(tasks.describe().transpose())
-############################################################################
# %% [markdown]
# We'll use the test server for the rest of this tutorial.
#
# .. warning::
# .. include:: ../../test_server_usage_warning.txt
+
+# %%
openml.config.start_using_configuration_for_example()
-############################################################################
# %% [markdown]
-# Uploading suites
-# ================
+# ## Uploading suites
#
# Uploading suites is as simple as uploading any kind of other OpenML
# entity - the only reason why we need so much code in this example is
@@ -84,6 +81,8 @@
# We'll take a random subset of at least ten tasks of all available tasks on
# the test server:
+
+# %%
all_tasks = list(openml.tasks.list_tasks(output_format="dataframe")["tid"])
task_ids_for_suite = sorted(np.random.choice(all_tasks, replace=False, size=20))
@@ -101,6 +100,6 @@
new_suite.publish()
print(new_suite)
-
-############################################################################
+# %%
openml.config.stop_using_configuration_for_example()
+# License: BSD 3-Clause
diff --git a/examples/30_extended/task_manual_iteration_tutorial.py b/examples/30_extended/task_manual_iteration_tutorial.py
index ae7f9bf01..69bb97902 100644
--- a/examples/30_extended/task_manual_iteration_tutorial.py
+++ b/examples/30_extended/task_manual_iteration_tutorial.py
@@ -1,21 +1,17 @@
-"""
# %% [markdown]
-Tasks: retrieving splits
-========================
+# # Tasks: retrieving splits
-Tasks define a target and a train/test split. Normally, they are the input to the function
-``openml.runs.run_model_on_task`` which automatically runs the model on all splits of the task.
-However, sometimes it is necessary to manually split a dataset to perform experiments outside of
-the functions provided by OpenML. One such example is in the benchmark library
-`HPOBench <https://github.com/automl/HPOBench>`_ which extensively uses data from OpenML,
-but not OpenML's functionality to conduct runs.
-"""
+# Tasks define a target and a train/test split. Normally, they are the input to the function
+# ``openml.runs.run_model_on_task`` which automatically runs the model on all splits of the task.
+# However, sometimes it is necessary to manually split a dataset to perform experiments outside of
+# the functions provided by OpenML. One such example is in the benchmark library
+# [HPOBench](https://github.com/automl/HPOBench) which extensively uses data from OpenML,
+# but not OpenML's functionality to conduct runs.
-# License: BSD 3-Clause
+# %%
import openml
-####################################################################################################
# %% [markdown]
# For this tutorial we will use the famous King+Rook versus King+Pawn on A7 dataset, which has
# the dataset ID 3 (`dataset on OpenML <https://www.openml.org/d/3>`_), and for which there exist
@@ -24,17 +20,18 @@
#
# We will first start with (`task 233 <https://www.openml.org/t/233>`_), which is a task with a
# holdout estimation procedure.
+
+# %%
task_id = 233
task = openml.tasks.get_task(task_id)
-####################################################################################################
# %% [markdown]
# Now that we have a task object we can obtain the number of repetitions, folds and samples as
# defined by the task:
+# %%
n_repeats, n_folds, n_samples = task.get_split_dimensions()
-####################################################################################################
# %% [markdown]
# * ``n_repeats``: Number of times the model quality estimation is performed
# * ``n_folds``: Number of folds per repeat
@@ -46,6 +43,7 @@
# Task ``233`` is a simple task using the holdout estimation procedure and therefore has only a
# single repeat, a single fold and a single sample size:
+# %%
print(
"Task {}: number of repeats: {}, number of folds: {}, number of samples {}.".format(
task_id,
@@ -55,12 +53,12 @@
)
)
-####################################################################################################
# %% [markdown]
# We can now retrieve the train/test split for this combination of repeats, folds and number of
# samples (indexing is zero-based). Usually, one would loop over all repeats, folds and sample
# sizes, but we can neglect this here as there is only a single repetition.
+# %%
train_indices, test_indices = task.get_train_test_split_indices(
repeat=0,
fold=0,
@@ -74,6 +72,7 @@
# %% [markdown]
# And then split the data based on this:
+# %%
X, y = task.get_X_and_y(dataset_format="dataframe")
X_train = X.iloc[train_indices]
y_train = y.iloc[train_indices]
@@ -89,10 +88,10 @@
)
)
-####################################################################################################
# %% [markdown]
# Obviously, we can also retrieve cross-validation versions of the dataset used in task ``233``:
+# %%
task_id = 3
task = openml.tasks.get_task(task_id)
X, y = task.get_X_and_y(dataset_format="dataframe")
@@ -106,9 +105,10 @@
)
)
-####################################################################################################
# %% [markdown]
# And then perform the aforementioned iteration over all splits:
+
+# %%
for repeat_idx in range(n_repeats):
for fold_idx in range(n_folds):
for sample_idx in range(n_samples):
@@ -135,10 +135,10 @@
)
)
-####################################################################################################
# %% [markdown]
# And also versions with multiple repeats:
+# %%
task_id = 1767
task = openml.tasks.get_task(task_id)
X, y = task.get_X_and_y(dataset_format="dataframe")
@@ -152,9 +152,10 @@
)
)
-####################################################################################################
# %% [markdown]
# And then again perform the aforementioned iteration over all splits:
+
+# %%
for repeat_idx in range(n_repeats):
for fold_idx in range(n_folds):
for sample_idx in range(n_samples):
@@ -181,10 +182,10 @@
)
)
-####################################################################################################
# %% [markdown]
# And finally a task based on learning curves:
+# %%
task_id = 1702
task = openml.tasks.get_task(task_id)
X, y = task.get_X_and_y(dataset_format="dataframe")
@@ -198,9 +199,10 @@
)
)
-####################################################################################################
# %% [markdown]
# And then again perform the aforementioned iteration over all splits:
+
+# %%
for repeat_idx in range(n_repeats):
for fold_idx in range(n_folds):
for sample_idx in range(n_samples):
@@ -226,3 +228,4 @@
y_test.shape,
)
)
+# License: BSD 3-Clause
diff --git a/examples/30_extended/tasks_tutorial.py b/examples/30_extended/tasks_tutorial.py
index 42ed313e1..3ba2fd3f9 100644
--- a/examples/30_extended/tasks_tutorial.py
+++ b/examples/30_extended/tasks_tutorial.py
@@ -1,18 +1,12 @@
-"""
# %% [markdown]
-Tasks
-=====
-
-A tutorial on how to list and download tasks.
-"""
-
-# License: BSD 3-Clause
+# # Tasks
+# A tutorial on how to list and download tasks.
+# %%
import openml
from openml.tasks import TaskType
import pandas as pd
-############################################################################
# %% [markdown]
#
# Tasks are identified by IDs and can be accessed in two different ways:
@@ -27,10 +21,8 @@
# metric, the splits and an iterator which can be used to access the
# splits in a useful manner.
-############################################################################
# %% [markdown]
-# Listing tasks
-# ^^^^^^^^^^^^^
+# ## Listing tasks
#
# We will start by simply listing only *supervised classification* tasks.
# **openml.tasks.list_tasks()** returns a dictionary of dictionaries by default, but we
@@ -38,6 +30,7 @@
# `pandas dataframe <https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html>`_
# instead to have better visualization capabilities and easier access:
+# %%
tasks = openml.tasks.list_tasks(
task_type=TaskType.SUPERVISED_CLASSIFICATION, output_format="dataframe"
)
@@ -45,57 +38,56 @@
print(f"First 5 of {len(tasks)} tasks:")
print(tasks.head())
-############################################################################
# %% [markdown]
# We can filter the list of tasks to only contain datasets with more than
# 500 samples, but less than 1000 samples:
+# %%
filtered_tasks = tasks.query("NumberOfInstances > 500 and NumberOfInstances < 1000")
print(list(filtered_tasks.index))
-############################################################################
+# %%
# Number of tasks
print(len(filtered_tasks))
-############################################################################
-# %% [markdown]
# %% [markdown]
# Then, we can further restrict the tasks to all have the same resampling strategy:
+# %%
filtered_tasks = filtered_tasks.query('estimation_procedure == "10-fold Crossvalidation"')
print(list(filtered_tasks.index))
-############################################################################
-
+# %%
# Number of tasks
print(len(filtered_tasks))
-############################################################################
# %% [markdown]
# Resampling strategies can be found on the
-# `OpenML Website <https://www.openml.org/search?type=measure&q=estimation%20procedure>`_.
+# [OpenML Website](https://www.openml.org/search?type=measure&q=estimation%20procedure).
#
# Similar to listing tasks by task type, we can list tasks by tags:
+# %%
tasks = openml.tasks.list_tasks(tag="OpenML100", output_format="dataframe")
print(f"First 5 of {len(tasks)} tasks:")
print(tasks.head())
-############################################################################
+# %% [markdown]
# Furthermore, we can list tasks based on the dataset id:
+# %%
tasks = openml.tasks.list_tasks(data_id=1471, output_format="dataframe")
print(f"First 5 of {len(tasks)} tasks:")
print(tasks.head())
-############################################################################
+# %% [markdown]
# In addition, a size limit and an offset can be applied both separately and simultaneously:
+# %%
tasks = openml.tasks.list_tasks(size=10, offset=50, output_format="dataframe")
print(tasks)
-############################################################################
# %% [markdown]
#
# **OpenML 100**
@@ -104,51 +96,46 @@
# instances per task. To make things easier, the tasks do not contain highly
# unbalanced data and sparse data. However, the tasks include missing values and
# categorical features. You can find out more about the *OpenML 100* on
-# `the OpenML benchmarking page <https://docs.openml.org/benchmark/>`_.
+# [the OpenML benchmarking page](https://docs.openml.org/benchmark/).
#
# Finally, it is also possible to list all tasks on OpenML with:
-############################################################################
+# %%
tasks = openml.tasks.list_tasks(output_format="dataframe")
print(len(tasks))
-############################################################################
# %% [markdown]
-# Exercise
-# ########
+# ## Exercise
#
# Search for the tasks on the 'eeg-eye-state' dataset.
+# %%
tasks.query('name=="eeg-eye-state"')
-############################################################################
# %% [markdown]
-# Downloading tasks
-# ^^^^^^^^^^^^^^^^^
+# ## Downloading tasks
#
# We provide two functions to download tasks, one which downloads only a
# single task by its ID, and one which takes a list of IDs and downloads
# all of these tasks:
+# %%
task_id = 31
task = openml.tasks.get_task(task_id)
-############################################################################
+# %%
# Properties of the task are stored as member variables:
-
print(task)
-############################################################################
+# %%
# And:
ids = [2, 1891, 31, 9983]
tasks = openml.tasks.get_tasks(ids)
print(tasks[0])
-############################################################################
# %% [markdown]
-# Creating tasks
-# ^^^^^^^^^^^^^^
+# ## Creating tasks
#
# You can also create new tasks. Take the following into account:
#
@@ -174,18 +161,16 @@
# necessary (e.g. when other measure make no sense), since it will create a new task, which
# scatters results across tasks.
-############################################################################
# %% [markdown]
# We'll use the test server for the rest of this tutorial.
#
# .. warning::
# .. include:: ../../test_server_usage_warning.txt
+# %%
openml.config.start_using_configuration_for_example()
-############################################################################
# %% [markdown]
-# Example
-# #######
+# ## Example
#
# Let's create a classification task on a dataset. In this example we will do this on the
# Iris dataset (ID=128 (on test server)). We'll use 10-fold cross-validation (ID=1),
@@ -194,7 +179,7 @@
# If such a task doesn't exist, a task will be created and the corresponding task_id
# will be returned.
-
+# %%
try:
my_task = openml.tasks.create_task(
task_type=TaskType.SUPERVISED_CLASSIFICATION,
@@ -217,13 +202,14 @@
task_id = tasks.loc[:, "tid"].values[0]
print("Task already exists. Task ID is", task_id)
+# %%
# reverting to prod server
openml.config.stop_using_configuration_for_example()
-############################################################################
# %% [markdown]
-# * `Complete list of task types <https://www.openml.org/search?type=task_type>`_.
-# * `Complete list of model estimation procedures <https://www.openml.org/search?q=%2520measure_type%3Aestimation_procedure&type=measure>`_.
-# * `Complete list of evaluation measures <https://www.openml.org/search?q=measure_type%3Aevaluation_measure&type=measure>`_.
+# * [Complete list of task types](https://www.openml.org/search?type=task_type).
+# * [Complete list of model estimation procedures](https://www.openml.org/search?q=%2520measure_type%3Aestimation_procedure&type=measure).
+# * [Complete list of evaluation measures](https://www.openml.org/search?q=measure_type%3Aevaluation_measure&type=measure).
#
+# License: BSD 3-Clause
diff --git a/examples/40_paper/2015_neurips_feurer_example.py b/examples/40_paper/2015_neurips_feurer_example.py
index 24ca3832c..fdfeb8acd 100644
--- a/examples/40_paper/2015_neurips_feurer_example.py
+++ b/examples/40_paper/2015_neurips_feurer_example.py
@@ -1,32 +1,29 @@
-"""
# %% [markdown]
-Feurer et al. (2015)
-====================
+# # Feurer et al. (2015)
-A tutorial on how to get the datasets used in the paper introducing *Auto-sklearn* by Feurer et al..
-
-Auto-sklearn website: https://automl.github.io/auto-sklearn/
-
-Publication
-~~~~~~~~~~~
-
-| Efficient and Robust Automated Machine Learning
-| Matthias Feurer, Aaron Klein, Katharina Eggensperger, Jost Springenberg, Manuel Blum and Frank Hutter
-| In *Advances in Neural Information Processing Systems 28*, 2015
-| Available at https://papers.nips.cc/paper/5872-efficient-and-robust-automated-machine-learning.pdf
+# A tutorial on how to get the datasets used in the paper introducing *Auto-sklearn* by Feurer et al..
+#
+# Auto-sklearn website: https://automl.github.io/auto-sklearn/
+#
+# ## Publication
+#
+# | Efficient and Robust Automated Machine Learning
+# | Matthias Feurer, Aaron Klein, Katharina Eggensperger, Jost Springenberg, Manuel Blum and Frank Hutter
+# | In *Advances in Neural Information Processing Systems 28*, 2015
+# | Available at https://papers.nips.cc/paper/5872-efficient-and-robust-automated-machine-learning.pdf
""" # noqa F401
-# License: BSD 3-Clause
+# %%
import pandas as pd
import openml
-####################################################################################################
# %% [markdown]
# List of dataset IDs given in the supplementary material of Feurer et al.:
# https://papers.nips.cc/paper/5872-efficient-and-robust-automated-machine-learning-supplemental.zip
-# fmt: off
+
+# %%
dataset_ids = [
3, 6, 12, 14, 16, 18, 21, 22, 23, 24, 26, 28, 30, 31, 32, 36, 38, 44, 46,
57, 60, 179, 180, 181, 182, 184, 185, 273, 293, 300, 351, 354, 357, 389,
@@ -39,9 +36,7 @@
1056, 1067, 1068, 1069, 1111, 1112, 1114, 1116, 1119, 1120, 1128, 1130,
1134, 1138, 1139, 1142, 1146, 1161, 1166,
]
-# fmt: on
-####################################################################################################
# %% [markdown]
# The dataset IDs could be used directly to load the dataset and split the data into a training set
# and a test set. However, to be reproducible, we will first obtain the respective tasks from
@@ -55,12 +50,13 @@
# Please check the `OpenML documentation of tasks <https://docs.openml.org/#tasks>`_ if you
# want to learn more about them.
-####################################################################################################
# %% [markdown]
# This lists both active and inactive tasks (because of ``status='all'``). Unfortunately,
# this is necessary as some of the datasets contain issues found after the publication and became
# deactivated, which also deactivated the tasks on them. More information on active or inactive
-# datasets can be found in the `online docs <https://docs.openml.org/#dataset-status>`_.
+# datasets can be found in the [online docs](https://docs.openml.org/#dataset-status).
+
+# %%
tasks = openml.tasks.list_tasks(
task_type=openml.tasks.TaskType.SUPERVISED_CLASSIFICATION,
status="all",
@@ -95,3 +91,5 @@
# These are the tasks to work with:
print(task_ids)
+
+# License: BSD 3-Clause
diff --git a/examples/40_paper/2018_ida_strang_example.py b/examples/40_paper/2018_ida_strang_example.py
index 255f461d5..e55008dd9 100644
--- a/examples/40_paper/2018_ida_strang_example.py
+++ b/examples/40_paper/2018_ida_strang_example.py
@@ -1,27 +1,21 @@
-"""
# %% [markdown]
-Strang et al. (2018)
-====================
-
-A tutorial on how to reproduce the analysis conducted for *Don't Rule Out Simple Models
-Prematurely: A Large Scale Benchmark Comparing Linear and Non-linear Classifiers in OpenML*.
-
-Publication
-~~~~~~~~~~~
-
-| Don't Rule Out Simple Models Prematurely: A Large Scale Benchmark Comparing Linear and Non-linear Classifiers in OpenML
-| Benjamin Strang, Peter van der Putten, Jan N. van Rijn and Frank Hutter
-| In *Advances in Intelligent Data Analysis XVII 17th International Symposium*, 2018
-| Available at https://link.springer.com/chapter/10.1007%2F978-3-030-01768-2_25
-"""
-
-# License: BSD 3-Clause
+# # Strang et al. (2018)
+#
+# A tutorial on how to reproduce the analysis conducted for *Don't Rule Out Simple Models
+# Prematurely: A Large Scale Benchmark Comparing Linear and Non-linear Classifiers in OpenML*.
+#
+# ## Publication
+#
+# | Don't Rule Out Simple Models Prematurely: A Large Scale Benchmark Comparing Linear and Non-linear Classifiers in OpenML
+# | Benjamin Strang, Peter van der Putten, Jan N. van Rijn and Frank Hutter
+# | In *Advances in Intelligent Data Analysis XVII 17th International Symposium*, 2018
+# | Available at https://link.springer.com/chapter/10.1007%2F978-3-030-01768-2_25
+# %%
import matplotlib.pyplot as plt
import openml
import pandas as pd
-##############################################################################
# %% [markdown]
# A basic step for each data-mining or machine learning task is to determine
# which model to choose based on the problem and the data at hand. In this
@@ -37,6 +31,7 @@
# more effort to distinguish the same flow with different hyperparameter
# values.
+# %%
study_id = 123
# for comparing svms: flow_ids = [7754, 7756]
# for comparing nns: flow_ids = [7722, 7729]
@@ -67,11 +62,10 @@
# adds column that indicates the difference between the two classifiers
evaluations["diff"] = evaluations[flow_ids[0]] - evaluations[flow_ids[1]]
-
-##############################################################################
# %% [markdown]
# makes the s-plot
+# %%
fig_splot, ax_splot = plt.subplots()
ax_splot.plot(range(len(evaluations)), sorted(evaluations["diff"]))
ax_splot.set_title(classifier_family)
@@ -81,12 +75,12 @@
plt.show()
-##############################################################################
# %% [markdown]
# adds column that indicates the difference between the two classifiers,
# needed for the scatter plot
+# %%
def determine_class(val_lin, val_nonlin):
if val_lin < val_nonlin:
return class_values[0]
@@ -113,11 +107,11 @@ def determine_class(val_lin, val_nonlin):
ax_scatter.set_yscale("log")
plt.show()
-##############################################################################
# %% [markdown]
# makes a scatter plot where each data point represents the performance of the
# two algorithms on various axis (not in the paper)
+# %%
fig_diagplot, ax_diagplot = plt.subplots()
ax_diagplot.grid(linestyle="--")
ax_diagplot.plot([0, 1], ls="-", color="black")
@@ -127,3 +121,4 @@ def determine_class(val_lin, val_nonlin):
ax_diagplot.set_xlabel(measure)
ax_diagplot.set_ylabel(measure)
plt.show()
+# License: BSD 3-Clause
diff --git a/examples/40_paper/2018_kdd_rijn_example.py b/examples/40_paper/2018_kdd_rijn_example.py
index 7733be6a0..2929f6429 100644
--- a/examples/40_paper/2018_kdd_rijn_example.py
+++ b/examples/40_paper/2018_kdd_rijn_example.py
@@ -1,23 +1,17 @@
-"""
# %% [markdown]
-van Rijn and Hutter (2018)
-==========================
-
-A tutorial on how to reproduce the paper *Hyperparameter Importance Across Datasets*.
-
-This is a Unix-only tutorial, as the requirements can not be satisfied on a Windows machine (Untested on other
-systems).
-
-Publication
-~~~~~~~~~~~
-
-| Hyperparameter importance across datasets
-| Jan N. van Rijn and Frank Hutter
-| In *Proceedings of the 24th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining*, 2018
-| Available at https://dl.acm.org/doi/10.1145/3219819.3220058
-"""
-
-# License: BSD 3-Clause
+# # van Rijn and Hutter (2018)
+#
+# A tutorial on how to reproduce the paper *Hyperparameter Importance Across Datasets*.
+#
+# This is a Unix-only tutorial, as the requirements can not be satisfied on a Windows machine (Untested on other
+# systems).
+#
+# ## Publication
+#
+# | Hyperparameter importance across datasets
+# | Jan N. van Rijn and Frank Hutter
+# | In *Proceedings of the 24th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining*, 2018
+# | Available at https://dl.acm.org/doi/10.1145/3219819.3220058
import sys
@@ -36,7 +30,6 @@
import openml
-##############################################################################
# %% [markdown]
# With the advent of automated machine learning, automated hyperparameter
# optimization methods are by now routinely used in data mining. However, this
@@ -69,6 +62,7 @@
# this, please see:
# https://github.com/janvanrijn/openml-pimp/blob/d0a14f3eb480f2a90008889f00041bdccc7b9265/examples/plot/plot_fanova_aggregates.py # noqa F401
+# %%
suite = openml.study.get_suite("OpenML100")
flow_id = 7707
parameter_filters = {"sklearn.svm.classes.SVC(17)_kernel": "sigmoid"}
@@ -161,12 +155,13 @@
# transform ``fanova_results`` from a list of dicts into a DataFrame
fanova_results = pd.DataFrame(fanova_results)
-##############################################################################
# %% [markdown]
# make the boxplot of the variance contribution. Obviously, we can also use
# this data to make the Nemenyi plot, but this relies on the rather complex
# ``Orange`` dependency (``pip install Orange3``). For the complete example,
# the reader is referred to the more elaborate script (referred to earlier)
+
+# %%
fig, ax = plt.subplots()
sns.boxplot(x="hyperparameter", y="fanova", data=fanova_results, ax=ax)
ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha="right")
@@ -174,3 +169,4 @@
ax.set_xlabel(None)
plt.tight_layout()
plt.show()
+# License: BSD 3-Clause
diff --git a/examples/40_paper/2018_neurips_perrone_example.py b/examples/40_paper/2018_neurips_perrone_example.py
index 4e7dd136a..2bfc84938 100644
--- a/examples/40_paper/2018_neurips_perrone_example.py
+++ b/examples/40_paper/2018_neurips_perrone_example.py
@@ -1,33 +1,27 @@
-"""
# %% [markdown]
-Perrone et al. (2018)
-=====================
-
-A tutorial on how to build a surrogate model based on OpenML data as done for *Scalable
-Hyperparameter Transfer Learning* by Perrone et al..
-
-Publication
-~~~~~~~~~~~
-
-| Scalable Hyperparameter Transfer Learning
-| Valerio Perrone and Rodolphe Jenatton and Matthias Seeger and Cedric Archambeau
-| In *Advances in Neural Information Processing Systems 31*, 2018
-| Available at https://papers.nips.cc/paper/7917-scalable-hyperparameter-transfer-learning.pdf
-
-This example demonstrates how OpenML runs can be used to construct a surrogate model.
-
-In the following section, we shall do the following:
-
-* Retrieve tasks and flows as used in the experiments by Perrone et al. (2018).
-* Build a tabular data by fetching the evaluations uploaded to OpenML.
-* Impute missing values and handle categorical data before building a Random Forest model that
- maps hyperparameter values to the area under curve score.
-"""
-
-############################################################################
+# # Perrone et al. (2018)
+#
+# A tutorial on how to build a surrogate model based on OpenML data as done for *Scalable
+# Hyperparameter Transfer Learning* by Perrone et al..
+#
+# ## Publication
+#
+# | Scalable Hyperparameter Transfer Learning
+# | Valerio Perrone and Rodolphe Jenatton and Matthias Seeger and Cedric Archambeau
+# | In *Advances in Neural Information Processing Systems 31*, 2018
+# | Available at https://papers.nips.cc/paper/7917-scalable-hyperparameter-transfer-learning.pdf
+#
+# This example demonstrates how OpenML runs can be used to construct a surrogate model.
+#
+# In the following section, we shall do the following:
+#
+# * Retrieve tasks and flows as used in the experiments by Perrone et al. (2018).
+# * Build a tabular data by fetching the evaluations uploaded to OpenML.
+# * Impute missing values and handle categorical data before building a Random Forest model that
+# maps hyperparameter values to the area under curve score.
-# License: BSD 3-Clause
+# %%
import openml
import numpy as np
import pandas as pd
@@ -40,12 +34,13 @@
from sklearn.ensemble import RandomForestRegressor
flow_type = "svm" # this example will use the smaller svm flow evaluations
-############################################################################
+
# %% [markdown]
# The subsequent functions are defined to fetch tasks, flows, evaluations and preprocess them into
# a tabular format that can be used to build models.
+# %%
def fetch_evaluations(run_full=False, flow_type="svm", metric="area_under_roc_curve"):
"""
Fetch a list of evaluations based on the flows and tasks used in the experiments.
@@ -156,7 +151,6 @@ def list_categorical_attributes(flow_type="svm"):
return ["booster"]
-#############################################################################
# %% [markdown]
# Fetching the data from OpenML
# *****************************
@@ -164,19 +158,19 @@ def list_categorical_attributes(flow_type="svm"):
# Here, we are reading all the tasks and evaluations for the SVM flow and
# pre-processing all retrieved evaluations.
+# %%
eval_df, task_ids, flow_id = fetch_evaluations(run_full=False, flow_type=flow_type)
X, y = create_table_from_evaluations(eval_df, flow_type=flow_type)
print(X.head())
print("Y : ", y[:5])
-#############################################################################
# %% [markdown]
-# Creating pre-processing and modelling pipelines
-# ***********************************************
+# ## Creating pre-processing and modelling pipelines
# The two primary tasks are to impute the missing values, that is, account for the hyperparameters
# that are not available with the runs from OpenML. And secondly, to handle categorical variables
# using One-hot encoding prior to modelling.
+# %%
# Separating data into categorical and non-categorical (numeric for this example) columns
cat_cols = list_categorical_attributes(flow_type=flow_type)
num_cols = list(set(X.columns) - set(cat_cols))
@@ -195,14 +189,13 @@ def list_categorical_attributes(flow_type="svm"):
model = Pipeline(steps=[("preprocess", ct), ("surrogate", clf)])
-#############################################################################
# %% [markdown]
-# Building a surrogate model on a task's evaluation
-# *************************************************
+# ## Building a surrogate model on a task's evaluation
# The same set of functions can be used for a single task to retrieve a singular table which can
# be used for the surrogate model construction. We shall use the SVM flow here to keep execution
# time simple and quick.
+# %%
# Selecting a task for the surrogate
task_id = task_ids[-1]
print("Task ID : ", task_id)
@@ -213,10 +206,8 @@ def list_categorical_attributes(flow_type="svm"):
print("Training RMSE : {:.5}".format(mean_squared_error(y, y_pred)))
-
-#############################################################################
-# Evaluating the surrogate model
-# ******************************
+# %% [markdown]
+# ## Evaluating the surrogate model
# The surrogate model built from a task's evaluations fetched from OpenML will be put into
# trivial action here, where we shall randomly sample configurations and observe the trajectory
# of the area under curve (auc) we can obtain from the surrogate we've built.
@@ -224,6 +215,7 @@ def list_categorical_attributes(flow_type="svm"):
# NOTE: This section is written exclusively for the SVM flow
+# %%
# Sampling random configurations
def random_sample_configurations(num_samples=100):
colnames = ["cost", "degree", "gamma", "kernel"]
@@ -246,7 +238,7 @@ def random_sample_configurations(num_samples=100):
configs = random_sample_configurations(num_samples=1000)
print(configs)
-#############################################################################
+# %%
preds = model.predict(configs)
# tracking the maximum AUC obtained over the functions evaluations
@@ -259,3 +251,4 @@ def random_sample_configurations(num_samples=100):
plt.title("AUC regret for Random Search on surrogate")
plt.xlabel("Numbe of function evaluations")
plt.ylabel("Regret")
+# License: BSD 3-Clause
From 792d759ba350f38f87e472bc374b88c0abf13149 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 21 Oct 2024 13:00:26 +0200
Subject: [PATCH 22/25] Add titles
---
examples/30_extended/custom_flow_.py | 2 +-
examples/30_extended/fetch_evaluations_tutorial.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/examples/30_extended/custom_flow_.py b/examples/30_extended/custom_flow_.py
index 7e28a92ca..2d93de2b9 100644
--- a/examples/30_extended/custom_flow_.py
+++ b/examples/30_extended/custom_flow_.py
@@ -1,5 +1,5 @@
# %% [markdown]
-# Creating and Using a Custom Flow
+# # Creating and Using a Custom Flow
# The most convenient way to create a flow for your machine learning workflow is to generate it
# automatically as described in the :ref:`sphx_glr_examples_30_extended_flow_id_tutorial.py` tutorial.
diff --git a/examples/30_extended/fetch_evaluations_tutorial.py b/examples/30_extended/fetch_evaluations_tutorial.py
index 0c735c235..7df63847c 100644
--- a/examples/30_extended/fetch_evaluations_tutorial.py
+++ b/examples/30_extended/fetch_evaluations_tutorial.py
@@ -1,5 +1,5 @@
# %% [markdown]
-# Fetching Evaluations
+# # Fetching Evaluations
# Evaluations contain a concise summary of the results of all runs made. Each evaluation
# provides information on the dataset used, the flow applied, the setup used, the metric
From d148f5d4ff32cbbd0d303929c5e95c6e6a8df8bc Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 21 Oct 2024 20:56:04 +0300
Subject: [PATCH 23/25] Make codeblock have light background
---
mkdocs.yml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/mkdocs.yml b/mkdocs.yml
index a8ab9735e..4ec95544f 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -3,6 +3,8 @@ theme:
name: material
features:
- content.code.copy
+ palette:
+ - scheme: default
nav:
- index.md
From df5099aef59f89572ccc303da4962da67b50b4b9 Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 21 Oct 2024 21:20:35 +0300
Subject: [PATCH 24/25] Hide the "In [#]" area
I also tried `no_input: true` but that removed the whole cell.
---
docs/stylesheets/extra.css | 3 +++
mkdocs.yml | 3 +++
2 files changed, 6 insertions(+)
create mode 100644 docs/stylesheets/extra.css
diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css
new file mode 100644
index 000000000..d0c4f79d8
--- /dev/null
+++ b/docs/stylesheets/extra.css
@@ -0,0 +1,3 @@
+.jp-InputArea-prompt, .jp-InputPrompt {
+ display: none !important;
+}
diff --git a/mkdocs.yml b/mkdocs.yml
index 4ec95544f..7cdfa7df9 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -6,6 +6,9 @@ theme:
palette:
- scheme: default
+extra_css:
+ - stylesheets/extra.css
+
nav:
- index.md
- Code Reference: reference/
From cb49496d3d2857f118cd8cc80f2e450f1e39e20f Mon Sep 17 00:00:00 2001
From: PGijsbers <p.gijsbers@tue.nl>
Date: Mon, 21 Oct 2024 21:35:44 +0300
Subject: [PATCH 25/25] Fix (link) formatting errors
---
examples/20_basic/simple_datasets_tutorial.py | 3 +++
examples/20_basic/simple_flows_and_runs_tutorial.py | 6 +++++-
examples/20_basic/simple_suites_tutorial.py | 8 +++-----
examples/30_extended/configure_logging.py | 2 +-
examples/30_extended/custom_flow_.py | 6 ++++--
examples/30_extended/datasets_tutorial.py | 1 +
examples/30_extended/study_tutorial.py | 3 ++-
examples/30_extended/suites_tutorial.py | 2 +-
examples/30_extended/task_manual_iteration_tutorial.py | 7 +++----
examples/30_extended/tasks_tutorial.py | 3 ++-
examples/40_paper/2015_neurips_feurer_example.py | 2 --
11 files changed, 25 insertions(+), 18 deletions(-)
diff --git a/examples/20_basic/simple_datasets_tutorial.py b/examples/20_basic/simple_datasets_tutorial.py
index b0381533c..5bdbca14b 100644
--- a/examples/20_basic/simple_datasets_tutorial.py
+++ b/examples/20_basic/simple_datasets_tutorial.py
@@ -37,8 +37,11 @@
# ## Load a dataset
# X - An array/dataframe where each row represents one example with
# the corresponding feature values.
+#
# y - the classes for each example
+#
# categorical_indicator - an array that indicates which feature is categorical
+#
# attribute_names - the names of the features for the examples (X) and
# target feature (y)
diff --git a/examples/20_basic/simple_flows_and_runs_tutorial.py b/examples/20_basic/simple_flows_and_runs_tutorial.py
index 79c69245c..0de16d74c 100644
--- a/examples/20_basic/simple_flows_and_runs_tutorial.py
+++ b/examples/20_basic/simple_flows_and_runs_tutorial.py
@@ -18,6 +18,8 @@
# ## Train a machine learning model
# NOTE: We are using dataset 20 from the test server: https://test.openml.org/d/20
+
+# %%
dataset = openml.datasets.get_dataset(20)
X, y, categorical_indicator, attribute_names = dataset.get_data(
target=dataset.default_target_attribute
@@ -28,6 +30,7 @@
# %% [markdown]
# ## Running a model on a task
+# %%
task = openml.tasks.get_task(119)
clf = ensemble.RandomForestClassifier()
run = openml.runs.run_model_on_task(clf, task)
@@ -36,10 +39,11 @@
# %% [markdown]
# ## Publishing the run
+# %%
myrun = run.publish()
print(f"Run was uploaded to {myrun.openml_url}")
print(f"The flow can be found at {myrun.flow.openml_url}")
-# %% [markdown]
+# %%
openml.config.stop_using_configuration_for_example()
# License: BSD 3-Clause
diff --git a/examples/20_basic/simple_suites_tutorial.py b/examples/20_basic/simple_suites_tutorial.py
index 34aefc913..5a1b429b1 100644
--- a/examples/20_basic/simple_suites_tutorial.py
+++ b/examples/20_basic/simple_suites_tutorial.py
@@ -8,7 +8,6 @@
# %%
import openml
-####################################################################################################
# %% [markdown]
# OpenML-CC18
# ===========
@@ -61,9 +60,8 @@
# Further examples
# ================
#
-# * [ex](../../30_extended/suites_tutorial)
-# * :ref:`sphx_glr_examples_30_extended_suites_tutorial.py`
-# * :ref:`sphx_glr_examples_30_extended_study_tutorial.py`
-# * :ref:`sphx_glr_examples_40_paper_2018_ida_strang_example.py`
+# * [Suites Tutorial](../../30_extended/suites_tutorial)
+# * [Study Tutoral](../../30_extended/study_tutorial)
+# * [Paper example: Strang et al.](../../40_paper/2018_ida_strang_example.py)
# License: BSD 3-Clause
diff --git a/examples/30_extended/configure_logging.py b/examples/30_extended/configure_logging.py
index bb7fc22a3..bc7bbc722 100644
--- a/examples/30_extended/configure_logging.py
+++ b/examples/30_extended/configure_logging.py
@@ -1,7 +1,7 @@
# %% [markdown]
# # Logging
# This tutorial explains openml-python logging, and shows how to configure it.
-# Openml-python uses the [Python logging module[(https://docs.python.org/3/library/logging.html)
+# Openml-python uses the [Python logging module](https://docs.python.org/3/library/logging.html)
# to provide users with log messages. Each log message is assigned a level of importance, see
# the table in Python's logging tutorial
# [here](https://docs.python.org/3/howto/logging.html#when-to-use-logging).
diff --git a/examples/30_extended/custom_flow_.py b/examples/30_extended/custom_flow_.py
index 2d93de2b9..15ec0e1fb 100644
--- a/examples/30_extended/custom_flow_.py
+++ b/examples/30_extended/custom_flow_.py
@@ -2,12 +2,14 @@
# # Creating and Using a Custom Flow
# The most convenient way to create a flow for your machine learning workflow is to generate it
-# automatically as described in the :ref:`sphx_glr_examples_30_extended_flow_id_tutorial.py` tutorial.
+# automatically as described in the
+# ["Obtaining Flow IDs"](../../30_extended/flow_id_tutorial) tutorial.
# However, there are scenarios where this is not possible, such
# as when the flow uses a framework without an extension or when the flow is described by a script.
# In those cases you can still create a custom flow by following the steps of this tutorial.
-# As an example we will use the flows generated for the `AutoML Benchmark <https://openml.github.io/automlbenchmark/>`_,
+# As an example we will use the flows generated for the
+# [AutoML Benchmark](https://openml.github.io/automlbenchmark/),
# and also show how to link runs to the custom flow.
# %%
diff --git a/examples/30_extended/datasets_tutorial.py b/examples/30_extended/datasets_tutorial.py
index 4cd04de0c..8c1b8a675 100644
--- a/examples/30_extended/datasets_tutorial.py
+++ b/examples/30_extended/datasets_tutorial.py
@@ -42,6 +42,7 @@
# %% [markdown]
# ## Download datasets
+# %%
# This is done based on the dataset ID.
dataset = openml.datasets.get_dataset(1471)
diff --git a/examples/30_extended/study_tutorial.py b/examples/30_extended/study_tutorial.py
index 363b5b305..f8f0b0b76 100644
--- a/examples/30_extended/study_tutorial.py
+++ b/examples/30_extended/study_tutorial.py
@@ -1,7 +1,8 @@
# %% [markdown]
# # Benchmark studies
# How to list, download and upload benchmark studies.
-# In contrast to `benchmark suites <https://docs.openml.org/benchmark/#benchmarking-suites>`_ which
+# In contrast to
+# [benchmark suites](https://docs.openml.org/benchmark/#benchmarking-suites) which
# hold a list of tasks, studies hold a list of runs. As runs contain all information on flows and
# tasks, all required information about a study can be retrieved.
diff --git a/examples/30_extended/suites_tutorial.py b/examples/30_extended/suites_tutorial.py
index 47d7b30b9..a92c1cdb5 100644
--- a/examples/30_extended/suites_tutorial.py
+++ b/examples/30_extended/suites_tutorial.py
@@ -4,7 +4,7 @@
# How to list, download and upload benchmark suites.
#
# If you want to learn more about benchmark suites, check out our
-# brief introductory tutorial :ref:`sphx_glr_examples_20_basic_simple_suites_tutorial.py` or the
+# brief introductory tutorial ["Simple suites tutorial"](../20_basic/simple_suites_tutorial) or the
# [OpenML benchmark docs](https://docs.openml.org/benchmark/#benchmarking-suites).
# %%
diff --git a/examples/30_extended/task_manual_iteration_tutorial.py b/examples/30_extended/task_manual_iteration_tutorial.py
index 69bb97902..00e81d823 100644
--- a/examples/30_extended/task_manual_iteration_tutorial.py
+++ b/examples/30_extended/task_manual_iteration_tutorial.py
@@ -14,11 +14,11 @@
# %% [markdown]
# For this tutorial we will use the famous King+Rook versus King+Pawn on A7 dataset, which has
-# the dataset ID 3 (`dataset on OpenML <https://www.openml.org/d/3>`_), and for which there exist
+# the dataset ID 3 ([dataset on OpenML](https://www.openml.org/d/3)), and for which there exist
# tasks with all important estimation procedures. It is small enough (less than 5000 samples) to
# efficiently use it in an example.
#
-# We will first start with (`task 233 <https://www.openml.org/t/233>`_), which is a task with a
+# We will first start with ([task 233](https://www.openml.org/t/233)), which is a task with a
# holdout estimation procedure.
# %%
@@ -38,7 +38,7 @@
# * ``n_samples``: How many data points to use. This is only relevant for learning curve tasks
#
# A list of all available estimation procedures is available
-# `here <https://www.openml.org/search?q=%2520measure_type%3Aestimation_procedure&type=measure>`_.
+# [here](https://www.openml.org/search?q=%2520measure_type%3Aestimation_procedure&type=measure).
#
# Task ``233`` is a simple task using the holdout estimation procedure and therefore has only a
# single repeat, a single fold and a single sample size:
@@ -68,7 +68,6 @@
print(train_indices.shape, train_indices.dtype)
print(test_indices.shape, test_indices.dtype)
-####################################################################################################
# %% [markdown]
# And then split the data based on this:
diff --git a/examples/30_extended/tasks_tutorial.py b/examples/30_extended/tasks_tutorial.py
index 3ba2fd3f9..e6c79d79d 100644
--- a/examples/30_extended/tasks_tutorial.py
+++ b/examples/30_extended/tasks_tutorial.py
@@ -25,9 +25,10 @@
# ## Listing tasks
#
# We will start by simply listing only *supervised classification* tasks.
+#
# **openml.tasks.list_tasks()** returns a dictionary of dictionaries by default, but we
# request a
-# `pandas dataframe <https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html>`_
+# [pandas dataframe](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html)
# instead to have better visualization capabilities and easier access:
# %%
diff --git a/examples/40_paper/2015_neurips_feurer_example.py b/examples/40_paper/2015_neurips_feurer_example.py
index fdfeb8acd..1900c7204 100644
--- a/examples/40_paper/2015_neurips_feurer_example.py
+++ b/examples/40_paper/2015_neurips_feurer_example.py
@@ -11,8 +11,6 @@
# | Matthias Feurer, Aaron Klein, Katharina Eggensperger, Jost Springenberg, Manuel Blum and Frank Hutter
# | In *Advances in Neural Information Processing Systems 28*, 2015
# | Available at https://papers.nips.cc/paper/5872-efficient-and-robust-automated-machine-learning.pdf
-""" # noqa F401
-
# %%
import pandas as pd