Support for Argo backend

.dockerignore

@@ -0,0 +1,5 @@
+*
+!example/yaml
+!example/*py
+!bin
+!pirlib

docs/README.rst

@@ -80,6 +80,34 @@ workflow.
 Open up ``example/run_docker.sh``, ``example/package_docker.yml``, and
 ``example/docker-compose.yml`` and see what's inside.
 
+Running as an Argo Workflow
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The following steps require and existing installations of Docker, Kubernetes and Argo.
+
+
+In order for Argo to have access to the docker images, a docker registry needs to be configured. Currently the `dockerize` module uses Docker Hub as the docker registry and only supports public repositories. Follow the following steps to configure Docker Hub:
+
+::
+
+   $ docker login
+   $ export DOCKER_USER=<username>
+   $ export PIRLIB_REPO=<reponame>
+
+Please ensure that the repository already exists under the user name in Docker Hub
+
+
+Follow the instructions `here <https://argoproj.github.io/argo-workflows/quick-start/>`_ and navigate your browser to ``https://127.0.0.1:2746``.
+
+Finally, execute the example.
+::
+
+   bash example/run_argo.sh
+
+
+You should be able to see the live execution of the different steps of the pipeline in the browser.
+Open up ``example/package_argo.yml``, ``argo-train.yml`` and see what's inside.
+
+
 .. include-end-before
 
 TODOs

example/argo-train.yml

@@ -0,0 +1,206 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: argo-train-
+spec:
+  entrypoint: DAG-train-pipeline
+  templates:
+  - name: clean-template
+    container:
+      image: nilabhra/pirlib
+      command:
+      - python
+      - -m
+      - pirlib.backends.argo_batch
+      - node
+      - gASVOQIAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojAJpZJSMBWNsZWFulIwLZW50cnlwb2ludHOUfZSMBG1haW6UaACMCkVudHJ5cG9pbnSUk5QpgZR9lCiMB3ZlcnNpb26UjAJ2MZSMB2hhbmRsZXKUjBVleGFtcGxlLmV4YW1wbGU6Y2xlYW6UjAdydW50aW1llIwKcHl0aG9uOjMuOJSMB2NvZGV1cmyUTowFaW1hZ2WUjA9uaWxhYmhyYS9waXJsaWKUdWJzjAlmcmFtZXdvcmuUTowGY29uZmlnlH2UjAZpbnB1dHOUXZRoAIwFSW5wdXSUk5QpgZR9lChoBYwHZGF0YXNldJSMBmlvdHlwZZSMCURJUkVDVE9SWZSMBnNvdXJjZZRoAIwKRGF0YVNvdXJjZZSTlCmBlH2UKIwHbm9kZV9pZJROjAtzdWJncmFwaF9pZJROjAlvdXRwdXRfaWSUTowOZ3JhcGhfaW5wdXRfaWSUjA10cmFpbl9kYXRhc2V0lHVijARtZXRhlGgAjAhNZXRhZGF0YZSTlCmBlH2UKIwEbmFtZZSMB2RhdGFzZXSUjAthbm5vdGF0aW9uc5ROdWJ1YmGMB291dHB1dHOUXZRoAIwGT3V0cHV0lJOUKYGUfZQoaAWMBnJldHVybpRoIYwJRElSRUNUT1JZlGgtaC8pgZR9lChoMowGcmV0dXJulGg0TnVidWJhaC1oLymBlH2UKGgyjAVjbGVhbpRoNE51YnViLg==
+      - gASVJAEAAAAAAABdlCiMCnBpcmxpYi5waXKUjApHcmFwaElucHV0lJOUKYGUfZQojAJpZJSMDXRyYWluX2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjARtZXRhlGgBjAhNZXRhZGF0YZSTlCmBlH2UKIwEbmFtZZSMDXRyYWluX2RhdGFzZXSUjAthbm5vdGF0aW9uc5ROdWJ1YmgDKYGUfZQoaAaMD3RyYW5zbGF0ZV9tb2RlbJRoCIwERklMRZRoCmgMKYGUfZQoaA+MD3RyYW5zbGF0ZV9tb2RlbJRoEU51YnViaAMpgZR9lChoBowJc2VudGVuY2VzlGgIjAlESVJFQ1RPUlmUaApoDCmBlH2UKGgPjAlzZW50ZW5jZXOUaBFOdWJ1YmUu
+      volumeMounts:
+      - name: node-outputs
+        mountPath: /mnt/node_outputs
+      - name: train-dataset
+        mountPath: /mnt/graph_inputs/train_dataset
+    volumes:
+    - name: node-outputs
+      nfs:
+        server: k8s-master.cm.cluster
+        path: /home/xxxx/pirlib/example/outputs
+        readOnly: no
+    - name: train-dataset
+      nfs:
+        server: k8s-master.cm.cluster
+        path: /home/xxxx/pirlib/example/inputs/train_dataset
+        readOnly: yes
+  - name: train-template
+    container:
+      image: nilabhra/pirlib
+      command:
+      - python
+      - -m
+      - pirlib.backends.argo_batch
+      - node
+      - gASVkQIAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojAJpZJSMBXRyYWlulIwLZW50cnlwb2ludHOUfZSMBG1haW6UaACMCkVudHJ5cG9pbnSUk5QpgZR9lCiMB3ZlcnNpb26UjAJ2MZSMB2hhbmRsZXKUjBVleGFtcGxlLmV4YW1wbGU6dHJhaW6UjAdydW50aW1llIwKcHl0aG9uOjMuOJSMB2NvZGV1cmyUTowFaW1hZ2WUjA9uaWxhYmhyYS9waXJsaWKUdWJzjAlmcmFtZXdvcmuUaACMCUZyYW1ld29ya5STlCmBlH2UKIwEbmFtZZSMB2FkYXB0ZGyUaA5OjAZjb25maWeUTnViaB59lCiMFGFkYXB0ZGwvbWluX3JlcGxpY2FzlEsBjBRhZGFwdGRsL21heF9yZXBsaWNhc5RLBHWMBmlucHV0c5RdlGgAjAVJbnB1dJSTlCmBlH2UKGgFjAdkYXRhc2V0lIwGaW90eXBllIwJRElSRUNUT1JZlIwGc291cmNllGgAjApEYXRhU291cmNllJOUKYGUfZQojAdub2RlX2lklIwFY2xlYW6UjAtzdWJncmFwaF9pZJROjAlvdXRwdXRfaWSUjAZyZXR1cm6UjA5ncmFwaF9pbnB1dF9pZJROdWKMBG1ldGGUaACMCE1ldGFkYXRhlJOUKYGUfZQoaByMB2RhdGFzZXSUjAthbm5vdGF0aW9uc5ROdWJ1YmGMB291dHB1dHOUXZRoAIwGT3V0cHV0lJOUKYGUfZQoaAWMBnJldHVybpRoKYwERklMRZRoNmg4KYGUfZQoaByMBnJldHVybpRoPE51YnViYWg2aDgpgZR9lChoHIwFdHJhaW6UaDxOdWJ1Yi4=
+      - gASVJAEAAAAAAABdlCiMCnBpcmxpYi5waXKUjApHcmFwaElucHV0lJOUKYGUfZQojAJpZJSMDXRyYWluX2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjARtZXRhlGgBjAhNZXRhZGF0YZSTlCmBlH2UKIwEbmFtZZSMDXRyYWluX2RhdGFzZXSUjAthbm5vdGF0aW9uc5ROdWJ1YmgDKYGUfZQoaAaMD3RyYW5zbGF0ZV9tb2RlbJRoCIwERklMRZRoCmgMKYGUfZQoaA+MD3RyYW5zbGF0ZV9tb2RlbJRoEU51YnViaAMpgZR9lChoBowJc2VudGVuY2VzlGgIjAlESVJFQ1RPUlmUaApoDCmBlH2UKGgPjAlzZW50ZW5jZXOUaBFOdWJ1YmUu
+      volumeMounts:
+      - name: node-outputs
+        mountPath: /mnt/node_outputs
+    volumes:
+    - name: node-outputs
+      nfs:
+        server: k8s-master.cm.cluster
+        path: /home/xxxx/pirlib/example/outputs
+        readOnly: no
+  - name: evaluate-template
+    container:
+      image: nilabhra/pirlib
+      command:
+      - python
+      - -m
+      - pirlib.backends.argo_batch
+      - node
+      - gASV5gIAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojAJpZJSMCGV2YWx1YXRllIwLZW50cnlwb2ludHOUfZSMBG1haW6UaACMCkVudHJ5cG9pbnSUk5QpgZR9lCiMB3ZlcnNpb26UjAJ2MZSMB2hhbmRsZXKUjBhleGFtcGxlLmV4YW1wbGU6ZXZhbHVhdGWUjAdydW50aW1llIwKcHl0aG9uOjMuOJSMB2NvZGV1cmyUTowFaW1hZ2WUjA9uaWxhYmhyYS9waXJsaWKUdWJzjAlmcmFtZXdvcmuUTowGY29uZmlnlH2UjAZpbnB1dHOUXZQoaACMBUlucHV0lJOUKYGUfZQoaAWME2t3YXJncy50ZXN0X2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjAZzb3VyY2WUaACMCkRhdGFTb3VyY2WUk5QpgZR9lCiMB25vZGVfaWSUTowLc3ViZ3JhcGhfaWSUTowJb3V0cHV0X2lklE6MDmdyYXBoX2lucHV0X2lklIwJc2VudGVuY2VzlHVijARtZXRhlGgAjAhNZXRhZGF0YZSTlCmBlH2UKIwEbmFtZZSME2t3YXJncy50ZXN0X2RhdGFzZXSUjAthbm5vdGF0aW9uc5ROdWJ1YmgdKYGUfZQoaAWMEmt3YXJncy5wcmVkaWN0aW9uc5RoIYwJRElSRUNUT1JZlGgjaCUpgZR9lChoKIwYaW5mZXJfcGlwZWxpbmUuc2VudGltZW50lGgpTmgqjAZyZXR1cm6UaCtOdWJoLWgvKYGUfZQoaDKMEmt3YXJncy5wcmVkaWN0aW9uc5RoNE51YnViZYwHb3V0cHV0c5RdlGgAjAZPdXRwdXSUk5QpgZR9lChoBYwGcmV0dXJulGghjAlEQVRBRlJBTUWUaC1oLymBlH2UKGgyjAZyZXR1cm6UaDROdWJ1YmFoLWgvKYGUfZQoaDKMCGV2YWx1YXRllGg0TnVidWIu
+      - gASVJAEAAAAAAABdlCiMCnBpcmxpYi5waXKUjApHcmFwaElucHV0lJOUKYGUfZQojAJpZJSMDXRyYWluX2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjARtZXRhlGgBjAhNZXRhZGF0YZSTlCmBlH2UKIwEbmFtZZSMDXRyYWluX2RhdGFzZXSUjAthbm5vdGF0aW9uc5ROdWJ1YmgDKYGUfZQoaAaMD3RyYW5zbGF0ZV9tb2RlbJRoCIwERklMRZRoCmgMKYGUfZQoaA+MD3RyYW5zbGF0ZV9tb2RlbJRoEU51YnViaAMpgZR9lChoBowJc2VudGVuY2VzlGgIjAlESVJFQ1RPUlmUaApoDCmBlH2UKGgPjAlzZW50ZW5jZXOUaBFOdWJ1YmUu
+      volumeMounts:
+      - name: node-outputs
+        mountPath: /mnt/node_outputs
+      - name: sentences
+        mountPath: /mnt/graph_inputs/sentences
+    volumes:
+    - name: node-outputs
+      nfs:
+        server: k8s-master.cm.cluster
+        path: /home/xxxx/pirlib/example/outputs
+        readOnly: no
+    - name: sentences
+      nfs:
+        server: k8s-master.cm.cluster
+        path: /home/xxxx/pirlib/example/inputs/sentences
+        readOnly: yes
+  - name: infer-pipeline-translate-1-template
+    container:
+      image: nilabhra/pirlib
+      command:
+      - python
+      - -m
+      - pirlib.backends.argo_batch
+      - node
+      - gASVwwIAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojAJpZJSMGmluZmVyX3BpcGVsaW5lLnRyYW5zbGF0ZV8xlIwLZW50cnlwb2ludHOUfZSMBG1haW6UaACMCkVudHJ5cG9pbnSUk5QpgZR9lCiMB3ZlcnNpb26UjAJ2MZSMB2hhbmRsZXKUjBlleGFtcGxlLmV4YW1wbGU6dHJhbnNsYXRllIwHcnVudGltZZSMCnB5dGhvbjozLjiUjAdjb2RldXJslE6MBWltYWdllIwPbmlsYWJocmEvcGlybGlilHVic4wJZnJhbWV3b3JrlE6MBmNvbmZpZ5R9lIwDa2V5lIwFdmFsdWWUc4wGaW5wdXRzlF2UKGgAjAVJbnB1dJSTlCmBlH2UKGgFjAZhcmdzLjCUjAZpb3R5cGWUjARGSUxFlIwGc291cmNllGgAjApEYXRhU291cmNllJOUKYGUfZQojAdub2RlX2lklE6MC3N1YmdyYXBoX2lklE6MCW91dHB1dF9pZJROjA5ncmFwaF9pbnB1dF9pZJSMD3RyYW5zbGF0ZV9tb2RlbJR1YowEbWV0YZRoAIwITWV0YWRhdGGUk5QpgZR9lCiMBG5hbWWUjAZhcmdzLjCUjAthbm5vdGF0aW9uc5ROdWJ1YmgfKYGUfZQoaAWMBmFyZ3MuMZRoI4wJRElSRUNUT1JZlGglaCcpgZR9lChoKk5oK05oLE5oLYwJc2VudGVuY2VzlHViaC9oMSmBlH2UKGg0jAZhcmdzLjGUaDZOdWJ1YmWMB291dHB1dHOUXZRoAIwGT3V0cHV0lJOUKYGUfZQoaAWMBnJldHVybpRoI4wJRElSRUNUT1JZlGgvaDEpgZR9lChoNIwGcmV0dXJulGg2TnVidWJhaC9oMSmBlH2UKGg0jAt0cmFuc2xhdGVfMZRoNk51YnViLg==
+      - gASVJAEAAAAAAABdlCiMCnBpcmxpYi5waXKUjApHcmFwaElucHV0lJOUKYGUfZQojAJpZJSMDXRyYWluX2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjARtZXRhlGgBjAhNZXRhZGF0YZSTlCmBlH2UKIwEbmFtZZSMDXRyYWluX2RhdGFzZXSUjAthbm5vdGF0aW9uc5ROdWJ1YmgDKYGUfZQoaAaMD3RyYW5zbGF0ZV9tb2RlbJRoCIwERklMRZRoCmgMKYGUfZQoaA+MD3RyYW5zbGF0ZV9tb2RlbJRoEU51YnViaAMpgZR9lChoBowJc2VudGVuY2VzlGgIjAlESVJFQ1RPUlmUaApoDCmBlH2UKGgPjAlzZW50ZW5jZXOUaBFOdWJ1YmUu
+      volumeMounts:
+      - name: node-outputs
+        mountPath: /mnt/node_outputs
+      - name: translate-model
+        mountPath: /mnt/graph_inputs/translate_model
+      - name: sentences
+        mountPath: /mnt/graph_inputs/sentences
+    volumes:
+    - name: node-outputs
+      nfs:
+        server: k8s-master.cm.cluster
+        path: /home/xxxx/pirlib/example/outputs
+        readOnly: no
+    - name: translate-model
+      nfs:
+        server: k8s-master.cm.cluster
+        path: /home/xxxx/pirlib/example/inputs/
+        readOnly: yes
+    - name: sentences
+      nfs:
+        server: k8s-master.cm.cluster
+        path: /home/xxxx/pirlib/example/inputs/sentences
+        readOnly: yes
+  - name: infer-pipeline-sentiment-template
+    container:
+      image: nilabhra/pirlib
+      command:
+      - python
+      - -m
+      - pirlib.backends.argo_batch
+      - node
+      - gASVywIAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojAJpZJSMGGluZmVyX3BpcGVsaW5lLnNlbnRpbWVudJSMC2VudHJ5cG9pbnRzlH2UjARtYWlulGgAjApFbnRyeXBvaW50lJOUKYGUfZQojAd2ZXJzaW9ulIwCdjGUjAdoYW5kbGVylIwZZXhhbXBsZS5leGFtcGxlOnNlbnRpbWVudJSMB3J1bnRpbWWUjApweXRob246My44lIwHY29kZXVybJROjAVpbWFnZZSMD25pbGFiaHJhL3BpcmxpYpR1YnOMCWZyYW1ld29ya5ROjAZjb25maWeUfZSMBmlucHV0c5RdlChoAIwFSW5wdXSUk5QpgZR9lChoBYwFbW9kZWyUjAZpb3R5cGWUjARGSUxFlIwGc291cmNllGgAjApEYXRhU291cmNllJOUKYGUfZQojAdub2RlX2lklIwFdHJhaW6UjAtzdWJncmFwaF9pZJROjAlvdXRwdXRfaWSUjAZyZXR1cm6UjA5ncmFwaF9pbnB1dF9pZJROdWKMBG1ldGGUaACMCE1ldGFkYXRhlJOUKYGUfZQojARuYW1llIwFbW9kZWyUjAthbm5vdGF0aW9uc5ROdWJ1YmgdKYGUfZQoaAWMCXNlbnRlbmNlc5RoIYwJRElSRUNUT1JZlGgjaCUpgZR9lChoKIwaaW5mZXJfcGlwZWxpbmUudHJhbnNsYXRlXzGUaCpOaCuMBnJldHVybpRoLU51YmguaDApgZR9lChoM4wJc2VudGVuY2VzlGg1TnVidWJljAdvdXRwdXRzlF2UaACMBk91dHB1dJSTlCmBlH2UKGgFjAZyZXR1cm6UaCGMCURJUkVDVE9SWZRoLmgwKYGUfZQoaDOMBnJldHVybpRoNU51YnViYWguaDApgZR9lChoM4wJc2VudGltZW50lGg1TnVidWIu
+      - gASVJAEAAAAAAABdlCiMCnBpcmxpYi5waXKUjApHcmFwaElucHV0lJOUKYGUfZQojAJpZJSMDXRyYWluX2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjARtZXRhlGgBjAhNZXRhZGF0YZSTlCmBlH2UKIwEbmFtZZSMDXRyYWluX2RhdGFzZXSUjAthbm5vdGF0aW9uc5ROdWJ1YmgDKYGUfZQoaAaMD3RyYW5zbGF0ZV9tb2RlbJRoCIwERklMRZRoCmgMKYGUfZQoaA+MD3RyYW5zbGF0ZV9tb2RlbJRoEU51YnViaAMpgZR9lChoBowJc2VudGVuY2VzlGgIjAlESVJFQ1RPUlmUaApoDCmBlH2UKGgPjAlzZW50ZW5jZXOUaBFOdWJ1YmUu
+      volumeMounts:
+      - name: node-outputs
+        mountPath: /mnt/node_outputs
+    volumes:
+    - name: node-outputs
+      nfs:
+        server: k8s-master.cm.cluster
+        path: /home/xxxx/pirlib/example/outputs
+        readOnly: no
+  - name: train-pipeline-template
+    container:
+      image: nilabhra/pirlib
+      command:
+      - python
+      - -m
+      - pirlib.backends.argo_batch
+      - graph
+      - gASVXgEAAAAAAABdlCiMCnBpcmxpYi5waXKUjAtHcmFwaE91dHB1dJSTlCmBlH2UKIwCaWSUjAhyZXR1cm4uMJSMBmlvdHlwZZSMBEZJTEWUjAZzb3VyY2WUaAGMCkRhdGFTb3VyY2WUk5QpgZR9lCiMB25vZGVfaWSUjAV0cmFpbpSMC3N1YmdyYXBoX2lklE6MCW91dHB1dF9pZJSMBnJldHVybpSMDmdyYXBoX2lucHV0X2lklE51YowEbWV0YZRoAYwITWV0YWRhdGGUk5QpgZR9lCiMBG5hbWWUjAhyZXR1cm4uMJSMC2Fubm90YXRpb25zlE51YnViaAMpgZR9lChoBowIcmV0dXJuLjGUaAiMCURBVEFGUkFNRZRoCmgMKYGUfZQoaA+MCGV2YWx1YXRllGgRTmgSjAZyZXR1cm6UaBROdWJoFWgXKYGUfZQoaBqMCHJldHVybi4xlGgcTnVidWJlLg==
+      volumeMounts:
+      - name: node-outputs
+        mountPath: /mnt/node_outputs
+      - name: train-dataset
+        mountPath: /mnt/graph_inputs/train_dataset
+      - name: translate-model
+        mountPath: /mnt/graph_inputs/translate_model
+      - name: sentences
+        mountPath: /mnt/graph_inputs/sentences
+      - name: graph-outputs
+        mountPath: /mnt/graph_outputs
+    volumes:
+    - name: node-outputs
+      nfs:
+        server: k8s-master.cm.cluster
+        path: /home/xxxx/pirlib/example/outputs
+        readOnly: no
+    - name: train-dataset
+      nfs:
+        server: k8s-master.cm.cluster
+        path: /home/xxxx/pirlib/example/inputs/train_dataset
+        readOnly: yes
+    - name: translate-model
+      nfs:
+        server: k8s-master.cm.cluster
+        path: /home/xxxx/pirlib/example/inputs/
+        readOnly: yes
+    - name: sentences
+      nfs:
+        server: k8s-master.cm.cluster
+        path: /home/xxxx/pirlib/example/inputs/sentences
+        readOnly: yes
+    - name: graph-outputs
+      nfs:
+        server: k8s-master.cm.cluster
+        path: /home/xxxx/pirlib/example/outputs
+        readOnly: no
+  - name: DAG-train-pipeline
+    dag:
+      tasks:
+      - name: clean
+        template: clean-template
+        dependencies: []
+      - name: train
+        template: train-template
+        dependencies:
+        - clean
+      - name: evaluate
+        template: evaluate-template
+        dependencies:
+        - infer-pipeline-sentiment
+      - name: infer-pipeline-translate-1
+        template: infer-pipeline-translate-1-template
+        dependencies: []
+      - name: infer-pipeline-sentiment
+        template: infer-pipeline-sentiment-template
+        dependencies:
+        - train
+        - infer-pipeline-translate-1
+      - name: train-pipeline
+        template: train-pipeline-template
+        dependencies:
+        - clean
+        - train
+        - evaluate
+        - infer-pipeline-translate-1
+        - infer-pipeline-sentiment

example/example.py

@@ -1,13 +1,14 @@
-import pandas
 import tempfile
-import yaml
 from dataclasses import asdict
 from typing import Tuple, TypedDict
 
+import pandas
+import yaml
+
 from pirlib.frameworks.adaptdl import AdaptDL
 from pirlib.iotypes import DirectoryPath, FilePath
-from pirlib.task import task
 from pirlib.pipeline import pipeline
+from pirlib.task import task
 
 
 @task
@@ -48,14 +49,13 @@ def evaluate(kwargs: EvaluateInput) -> pandas.DataFrame:
 
 @task
 def translate(args: Tuple[FilePath, DirectoryPath]) -> DirectoryPath:
-    model, sentences = args
+    model_dir, sentences = args
     task_ctx = task.context()
-    with open(model) as f, open(sentences / "file.txt") as g:
+    with open(model_dir / "translate_model.txt") as f, open(sentences / "file.txt") as g:
         print(
             "translate({}, {}, config={})".format(
-                f.read().strip(),
-                g.read().strip(),
-                task_ctx.config)
+                f.read().strip(), g.read().strip(), task_ctx.config
+            )
         )
     outdir = task_ctx.output
     with open(outdir / "file.txt", "w") as f:
@@ -74,19 +74,18 @@ def sentiment(model: FilePath, sentences: DirectoryPath) -> DirectoryPath:
 
 
 @pipeline
-def infer_pipeline(translate_model: FilePath,
-                   sentiment_model: FilePath,
-                   sentences: DirectoryPath) -> DirectoryPath:
+def infer_pipeline(
+    translate_model: FilePath, sentiment_model: FilePath, sentences: DirectoryPath
+) -> DirectoryPath:
     translate_1 = translate.instance("translate_1")
     translate_1.config["key"] = "value"
     return sentiment(sentiment_model, translate_1((translate_model, sentences)))
 
 
 @pipeline
 def train_pipeline(
-        train_dataset: DirectoryPath,
-        translate_model: FilePath,
-        sentences: DirectoryPath) -> Tuple[FilePath, pandas.DataFrame]:
+    train_dataset: DirectoryPath, translate_model: FilePath, sentences: DirectoryPath
+) -> Tuple[FilePath, pandas.DataFrame]:
     sentiment_model = train(clean(train_dataset))
     sentiment = infer_pipeline(translate_model, sentiment_model, sentences)
     eval_input = {"test_dataset": sentences, "predictions": sentiment}
@@ -107,9 +106,9 @@ def train_pipeline(
     with open(f"{dir_3.name}/file.txt", "w") as f:
         f.write("sentences")
     # Test calling end-to-end pipeline.
-    model_path, metrics = train_pipeline(DirectoryPath(dir_1.name),
-                                         FilePath(file_2.name),
-                                         DirectoryPath(dir_3.name))
+    model_path, metrics = train_pipeline(
+        DirectoryPath(dir_1.name), FilePath(file_2.name), DirectoryPath(dir_3.name)
+    )
     with open(model_path) as f:
         print("pipeline model: {}".format(f.read().strip()))
     print("pipeline metrics: {}".format(metrics.to_records()))