Support for Argo backend

.dockerignore

@@ -0,0 +1,5 @@
+*
+!example/yaml
+!example/*py
+!bin
+!pirlib

example/example.py

@@ -1,13 +1,14 @@
-import pandas
 import tempfile
-import yaml
 from dataclasses import asdict
 from typing import Tuple, TypedDict
 
+import pandas
+import yaml
+
 from pirlib.frameworks.adaptdl import AdaptDL
 from pirlib.iotypes import DirectoryPath, FilePath
-from pirlib.task import task
 from pirlib.pipeline import pipeline
+from pirlib.task import task
 
 
 @task
@@ -47,15 +48,14 @@ def evaluate(kwargs: EvaluateInput) -> pandas.DataFrame:
 
 
 @task
-def translate(args: Tuple[FilePath, DirectoryPath]) -> DirectoryPath:
-    model, sentences = args
+def translate(args: Tuple[DirectoryPath, DirectoryPath]) -> DirectoryPath:
+    model_dir, sentences = args
     task_ctx = task.context()
-    with open(model) as f, open(sentences / "file.txt") as g:
+    with open(model_dir / "translate_model.txt") as f, open(sentences / "file.txt") as g:
         print(
             "translate({}, {}, config={})".format(
-                f.read().strip(),
-                g.read().strip(),
-                task_ctx.config)
+                f.read().strip(), g.read().strip(), task_ctx.config
+            )
         )
     outdir = task_ctx.output
     with open(outdir / "file.txt", "w") as f:
@@ -74,19 +74,18 @@ def sentiment(model: FilePath, sentences: DirectoryPath) -> DirectoryPath:
 
 
 @pipeline
-def infer_pipeline(translate_model: FilePath,
-                   sentiment_model: FilePath,
-                   sentences: DirectoryPath) -> DirectoryPath:
+def infer_pipeline(
+    translate_model: DirectoryPath, sentiment_model: FilePath, sentences: DirectoryPath
+) -> DirectoryPath:
     translate_1 = translate.instance("translate_1")
     translate_1.config["key"] = "value"
     return sentiment(sentiment_model, translate_1((translate_model, sentences)))
 
 
 @pipeline
 def train_pipeline(
-        train_dataset: DirectoryPath,
-        translate_model: FilePath,
-        sentences: DirectoryPath) -> Tuple[FilePath, pandas.DataFrame]:
+    train_dataset: DirectoryPath, translate_model: FilePath, sentences: DirectoryPath
+) -> Tuple[FilePath, pandas.DataFrame]:
     sentiment_model = train(clean(train_dataset))
     sentiment = infer_pipeline(translate_model, sentiment_model, sentences)
     eval_input = {"test_dataset": sentences, "predictions": sentiment}
@@ -107,9 +106,9 @@ def train_pipeline(
     with open(f"{dir_3.name}/file.txt", "w") as f:
         f.write("sentences")
     # Test calling end-to-end pipeline.
-    model_path, metrics = train_pipeline(DirectoryPath(dir_1.name),
-                                         FilePath(file_2.name),
-                                         DirectoryPath(dir_3.name))
+    model_path, metrics = train_pipeline(
+        DirectoryPath(dir_1.name), FilePath(file_2.name), DirectoryPath(dir_3.name)
+    )
     with open(model_path) as f:
         print("pipeline model: {}".format(f.read().strip()))
     print("pipeline metrics: {}".format(metrics.to_records()))

example/run_argo.sh

@@ -0,0 +1,25 @@
+EXAMPLEDIR=$(dirname $0)
+ROOTDIR=$EXAMPLEDIR/..
+
+### Module 1: Docker_Packaging
+python $ROOTDIR/bin/pircli dockerize \
+    --auto $ROOTDIR \
+	--pipeline example.example:train_pipeline \
+	--output $EXAMPLEDIR/package_argo.yml \
+	--flatten
+
+# Convert EXAMPLEDIR to absolute path since docker can't bind-mount relative paths.
+EXAMPLEDIR=$([[ $EXAMPLEDIR = /* ]] && echo "$EXAMPLEDIR" || echo "$PWD/${EXAMPLEDIR#./}")
+
+### Module 2: Argoize_Module
+INPUT_train_dataset=$EXAMPLEDIR/inputs/train_dataset \
+INPUT_translate_model=$EXAMPLEDIR/inputs/ \
+INPUT_sentences=$EXAMPLEDIR/inputs/sentences \
+OUTPUT=$EXAMPLEDIR/outputs \
+NFS_SERVER=k8s-master.cm.cluster \
+python  $ROOTDIR/bin/pircli generate $EXAMPLEDIR/package_argo.yml \
+	--target pirlib.backends.argo_batch:ArgoBatchBackend \
+	--output $EXAMPLEDIR/argo-train.yml
+
+# Run the Argo workflow
+argo submit -n argo --watch $EXAMPLEDIR/argo-train.yml

example/run_docker.sh

@@ -1,20 +1,20 @@
 EXAMPLEDIR=$(dirname $0)
 ROOTDIR=$EXAMPLEDIR/..
 
-PYTHONPATH=$ROOTDIR $ROOTDIR/bin/pircli dockerize --auto $ROOTDIR \
+python $ROOTDIR/bin/pircli dockerize --auto $ROOTDIR \
 	--pipeline example.example:train_pipeline \
 	--output $EXAMPLEDIR/package_docker.yml \
     --flatten
 
-PYTHONPATH=$ROOTDIR $ROOTDIR/bin/pircli generate $EXAMPLEDIR/package_docker.yml \
+python $ROOTDIR/bin/pircli generate $EXAMPLEDIR/package_docker.yml \
 	--target pirlib.backends.docker_batch:DockerBatchBackend \
 	--output $EXAMPLEDIR/docker-compose.yml
 
 # Convert EXAMPLEDIR to absolute path since docker can't bind-mount relative paths.
 EXAMPLEDIR=$([[ $EXAMPLEDIR = /* ]] && echo "$EXAMPLEDIR" || echo "$PWD/${EXAMPLEDIR#./}")
 
 INPUT_train_dataset=$EXAMPLEDIR/inputs/train_dataset \
-INPUT_translate_model=$EXAMPLEDIR/inputs/translate_model.txt \
+INPUT_translate_model=$EXAMPLEDIR/inputs \
 INPUT_sentences=$EXAMPLEDIR/inputs/sentences \
 OUTPUT=$EXAMPLEDIR/outputs \
 docker-compose -f $EXAMPLEDIR/docker-compose.yml up --force-recreate -V