Import branch

databricks_notebooks/bulk/Export_All.py

@@ -62,15 +62,29 @@
 
 # COMMAND ----------
 
-from mlflow_export_import.bulk.export_all import export_all
-
-export_all(
-    output_dir = output_dir, 
-    stages = stages,
-    export_latest_versions = export_latest_versions,
-    run_start_time = run_start_date,
-    export_permissions = export_permissions,
-    export_deleted_runs = export_deleted_runs,
-    notebook_formats = notebook_formats, 
-    use_threads = use_threads
-)
+# MAGIC %%capture captured
+# MAGIC 
+# MAGIC from mlflow_export_import.bulk.export_all import export_all
+# MAGIC 
+# MAGIC export_all(
+# MAGIC     output_dir = output_dir, 
+# MAGIC     stages = stages,
+# MAGIC     export_latest_versions = export_latest_versions,
+# MAGIC     run_start_time = run_start_date,
+# MAGIC     export_permissions = export_permissions,
+# MAGIC     export_deleted_runs = export_deleted_runs,
+# MAGIC     notebook_formats = notebook_formats, 
+# MAGIC     use_threads = use_threads
+# MAGIC )
+
+# COMMAND ----------
+
+# DBTITLE 1,write log file
+filepath = "/mnt/public-blobs/dcoles/mlflow_export_log.txt"
+
+dbutils.fs.rm(filepath)
+dbutils.fs.put(filepath, captured.stdout)
+
+# COMMAND ----------
+
+

databricks_notebooks/bulk/Export_Experiments.py

@@ -1,8 +1,8 @@
 # Databricks notebook source
 # MAGIC %md ## Export Experiments
-# MAGIC 
+# MAGIC
 # MAGIC Export multiple experiments and all their runs.
-# MAGIC 
+# MAGIC
 # MAGIC Widgets
 # MAGIC * `1. Experiments` - comma delimited list of either experiment IDs or experiment names. `all` will export all experiments.
 # MAGIC * `2. Output directory` - shared directory between source and destination workspaces.
@@ -53,6 +53,22 @@
 
 # COMMAND ----------
 
+# DBTITLE 1,set up log file
+import os 
+from datetime import datetime
+import pytz
+
+cst = pytz.timezone('US/Central')
+now = datetime.now(tz=cst)
+date = now.strftime("%Y-%m-%d-%H:%M:%S")
+
+logfile = f"export_experiments.{date}.log"
+os.environ["MLFLOW_EXPORT_IMPORT_LOG_OUTPUT_FILE"] = logfile 
+
+print("Logging to", logfile)
+
+# COMMAND ----------
+
 assert_widget(experiments, "1. Experiments")
 assert_widget(output_dir, "2. Output directory")
 

databricks_notebooks/bulk/Export_Models.py

@@ -1,8 +1,8 @@
 # Databricks notebook source
 # MAGIC %md ## Export Models
-# MAGIC 
+# MAGIC
 # MAGIC Export specified models, their version runs and the experiments that the runs belong to.
-# MAGIC 
+# MAGIC
 # MAGIC Widgets
 # MAGIC * `1. Models` - comma seperated registered model names to be exported. `all` will export all models.
 # MAGIC * `2. Output directory` - shared directory between source and destination workspaces.
@@ -13,7 +13,7 @@
 # MAGIC * `7. Export deleted runs`
 # MAGIC * `8. Notebook formats`
 # MAGIC * `9. Use threads`
-# MAGIC 
+# MAGIC
 # MAGIC See: https://github.com/mlflow/mlflow-export-import/blob/master/README_bulk.md#registered-models.
 
 # COMMAND ----------
@@ -69,6 +69,22 @@
 
 # COMMAND ----------
 
+# DBTITLE 1,set up log file
+import os 
+from datetime import datetime
+import pytz
+
+cst = pytz.timezone('US/Central')
+now = datetime.now(tz=cst)
+date = now.strftime("%Y-%m-%d-%H:%M:%S")
+
+logfile = f"export_models.{date}.log"
+os.environ["MLFLOW_EXPORT_IMPORT_LOG_OUTPUT_FILE"] = logfile 
+
+print("Logging to", logfile)
+
+# COMMAND ----------
+
 assert_widget(models, "1. Models")
 assert_widget(output_dir, "2. Output directory")
 
@@ -91,33 +107,3 @@
     notebook_formats = notebook_formats,
     use_threads = use_threads
 )
-
-# COMMAND ----------
-
-# MAGIC %md ### Display exported files
-
-# COMMAND ----------
-
-# MAGIC %sh 
-# MAGIC echo $OUTPUT_DIR
-# MAGIC ls -l $OUTPUT_DIR
-
-# COMMAND ----------
-
-# MAGIC %sh cat $OUTPUT_DIR/manifest.json
-
-# COMMAND ----------
-
-# MAGIC %sh ls -l $OUTPUT_DIR/models
-
-# COMMAND ----------
-
-# MAGIC %sh cat $OUTPUT_DIR/models/models.json
-
-# COMMAND ----------
-
-# MAGIC %sh ls -l $OUTPUT_DIR/experiments
-
-# COMMAND ----------
-
-# MAGIC %sh cat $OUTPUT_DIR/experiments/experiments.json

databricks_notebooks/bulk/Import_Experiments.py

@@ -1,12 +1,12 @@
 # Databricks notebook source
 # MAGIC %md ## Import Experiments
-# MAGIC 
+# MAGIC
 # MAGIC Widgets
 # MAGIC * `1. Input directory` - directory of exported experiments.
 # MAGIC * `2. Import source tags`
 # MAGIC * `3. Experiment rename file` - Experiment rename file.
 # MAGIC * `4. Use threads` - use multi-threaded import.
-# MAGIC 
+# MAGIC
 # MAGIC See https://github.com/mlflow/mlflow-export-import/blob/master/README_bulk.md#Import-experiments.
 
 # COMMAND ----------
@@ -15,6 +15,22 @@
 
 # COMMAND ----------
 
+# DBTITLE 1,set up log file
+import os 
+from datetime import datetime
+import pytz
+
+cst = pytz.timezone('US/Central')
+now = datetime.now(tz=cst)
+date = now.strftime("%Y-%m-%d-%H:%M:%S")
+
+logfile = f"import_experiments.{date}.log"
+os.environ["MLFLOW_EXPORT_IMPORT_LOG_OUTPUT_FILE"] = logfile 
+
+print("Logging to", logfile)
+
+# COMMAND ----------
+
 dbutils.widgets.text("1. Input directory", "") 
 input_dir = dbutils.widgets.get("1. Input directory")
 input_dir = input_dir.replace("dbfs:","/dbfs")
@@ -40,6 +56,8 @@
 
 # COMMAND ----------
 
+#%%capture captured
+
 from mlflow_export_import.bulk.import_experiments import import_experiments
 
 import_experiments(
@@ -48,3 +66,7 @@
     experiment_renames = experiment_rename_file,
     use_threads = use_threads
 )
+
+# COMMAND ----------
+
+

databricks_notebooks/bulk/Import_Models.py

@@ -1,14 +1,14 @@
 # Databricks notebook source
 # MAGIC %md ## Import Models
-# MAGIC 
+# MAGIC
 # MAGIC Widgets 
 # MAGIC * `1. Input directory` - directory of exported models. 
 # MAGIC * `2. Delete model` - delete the current contents of model
 # MAGIC * `3. Model rename file` - Model rename file.
 # MAGIC * `4. Experiment rename file` - Experiment rename file.
 # MAGIC * `5. Import source tags`
 # MAGIC * `6. Use threads` - use multi-threaded import
-# MAGIC 
+# MAGIC
 # MAGIC See https://github.com/mlflow/mlflow-export-import/blob/master/README_bulk.md#Import-registered-models
 
 # COMMAND ----------
@@ -47,17 +47,35 @@
 
 # COMMAND ----------
 
+# DBTITLE 1,set up log file
+import os 
+from datetime import datetime
+import pytz
+
+cst = pytz.timezone('US/Central')
+now = datetime.now(tz=cst)
+date = now.strftime("%Y-%m-%d-%H:%M:%S")
+
+logfile = f"import_models.{date}.log"
+os.environ["MLFLOW_EXPORT_IMPORT_LOG_OUTPUT_FILE"] = logfile 
+
+print("Logging to", logfile)
+
+# COMMAND ----------
+
 assert_widget(input_dir, "1. Input directory")
 
 # COMMAND ----------
 
-from mlflow_export_import.bulk.import_models import import_all
+from mlflow_export_import.bulk.import_models import import_models
 
-import_all(
-    input_dir = input_dir,
-    delete_model = delete_model,
-    import_source_tags = import_source_tags,
-    experiment_renames = experiment_rename_file,
-    model_renames = model_rename_file,
-    use_threads = use_threads
+import_models(
+  input_dir = input_dir,
+  delete_model = delete_model,
+  use_src_user_id = True,
+  verbose=True,
+  import_source_tags = import_source_tags,
+  experiment_renames = experiment_rename_file,
+  model_renames = model_rename_file,
+  use_threads = use_threads
 )

databricks_notebooks/bulk/export_common.py

@@ -0,0 +1,2 @@
+# Databricks notebook source
+# MAGIC %run ./Common

databricks_notebooks/bulk/import_common.py

@@ -0,0 +1,14 @@
+# Databricks notebook source
+# DBTITLE 1,install mlflow-export-import from local
+# MAGIC %pip install ../../../mlflow-export-import --use-feature=in-tree-build
+
+# COMMAND ----------
+
+def assert_widget(value, name):
+    if len(value.rstrip())==0: 
+        raise Exception(f"ERROR: '{name}' widget is required")
+
+# COMMAND ----------
+
+import mlflow
+mlflow_client = mlflow.client.MlflowClient()

mlflow_export_import/bulk/export_experiments.py

@@ -99,7 +99,7 @@ def export_experiments(
     export_results = []
     futures = []
     notebook_formats = utils.string_to_list(notebook_formats)
-    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+    with ThreadPoolExecutor() as executor:
         for exp_id_or_name in experiments:
             run_ids = experiments_dct.get(exp_id_or_name, None)
             future = executor.submit(_export_experiment,

mlflow_export_import/bulk/export_models.py

@@ -104,7 +104,7 @@ def _export_models(
 
     notebook_formats = utils.string_to_list(notebook_formats),
     futures = []
-    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+    with ThreadPoolExecutor() as executor:
         for model_name in model_names:
             dir = os.path.join(output_dir, model_name)
             future = executor.submit(export_model,

mycode/.gitignore

@@ -0,0 +1,5 @@
+export_*
+exported_*
+import_*
+imported_*
+scratch*