Merge pull request #5 from jonheng/polyaxon

Add polyaxon files

.gitignore

@@ -7,6 +7,8 @@ data/
 build/
 dist/
 output/
+.polyaxon
+.polyaxonignore
 
 # Docs
 docs/build

polyaxon/README.md

@@ -0,0 +1,20 @@
+### Training on Polyaxon
+
+```
+polyaxon config set --host=polyaxon.okdapp.tekong.aisingapore.net --port=80 --use_https=False
+
+polyaxon login -u <username>
+polyaxon project create --name=<project_name> --description='Some description.'
+polyaxon init <project_name>
+
+polyaxon upload
+
+polyaxon run -f polyaxon/train.yml
+```
+
+### Using Polyaxon notebook
+
+```
+polyaxon notebook start -f polyaxon/notebook.yml
+polyaxon notebook stop
+```

polyaxon/emotion_entailment/conda.yml

@@ -0,0 +1,16 @@
+name: emotion_entailment
+channels:
+  - defaults
+dependencies:
+  - python=3.8
+  - pip=21.0.1
+  - cudatoolkit=10.2
+  - pip:
+      - numpy==1.18.2
+      - pandas==1.0.1
+      - scikit-learn==0.23.1
+      - torch==1.6.0
+      - transformers==4.4.2
+      - tokenizers==0.10.1
+      - tqdm==4.48.0
+      - six==1.15.0

polyaxon/emotion_entailment/experiment.df

@@ -0,0 +1,20 @@
+FROM registry.aisingapore.net/polyaxon/cuda10:latest
+
+ARG USER="polyaxon"
+ARG WORK_DIR="/home/$USER"
+
+RUN rm /bin/sh && ln -s /bin/bash /bin/sh && \
+    apt update && apt install -y jq ca-certificates
+
+WORKDIR $WORK_DIR
+USER $USER
+
+COPY build/conda.yml .
+RUN conda env update -f conda.yml -n base && \
+    rm conda.yml
+
+WORKDIR /code
+
+COPY --chown=$USER:$USER build .
+
+ENV LD_LIBRARY_PATH /usr/local/cuda/lib64:$LD_LIBRARY_PATH

polyaxon/emotion_entailment/notebook.df

@@ -0,0 +1,29 @@
+FROM registry.aisingapore.net/aiap/polyaxon/pytorch-tf2-cpu:latest
+
+ARG WORK_DIR="/code"
+
+RUN pip install jupyterlab==0.33.12
+
+WORKDIR $WORK_DIR
+
+RUN mkdir -p $WORK_DIR && chown -R 2222:2222 $WORK_DIR
+
+ARG ORG_JUPYTER="/opt/conda/bin/jupyter"
+ARG MOD_JUPYTER="/opt/conda/bin/jupyter.real"
+
+RUN mv $ORG_JUPYTER $MOD_JUPYTER && \
+    echo "#!/bin/bash" >  $ORG_JUPYTER && \
+    echo "/code/link_workspace.sh &" >> $ORG_JUPYTER && \
+    echo "export SHELL=/bin/bash" >> $ORG_JUPYTER && \
+    echo "$MOD_JUPYTER \"\$@\"" >>  $ORG_JUPYTER && \
+    chmod +x $ORG_JUPYTER
+
+COPY build/conda.yml /code
+COPY build/scripts/link_workspace.sh /code
+
+RUN apt-get update && apt-get -y install vim jq
+
+RUN conda env update -n polyaxon --file conda.yml
+RUN rm /code/conda.yml
+
+ENV LANG "C.UTF-8"

polyaxon/emotion_entailment/notebook.yml

@@ -0,0 +1,15 @@
+---
+version: 1
+
+kind: notebook
+
+build:
+  dockerfile: polyaxon/docker/notebook.df
+  context: .
+
+environment:
+  persistence:
+    data: ["data"]
+
+logging:
+  level: DEBUG

polyaxon/emotion_entailment/train.yml

@@ -0,0 +1,18 @@
+version: 1
+kind: experiment
+logging:
+  level: DEBUG
+build:
+  dockerfile: polyaxon/docker/experiment.df
+  context: .
+environment:
+  resources:
+    gpu:
+      requests: 1
+      limits: 1
+  persistence:
+    data:
+      - data
+run:
+  cmd: >-
+    python train.py --config config/polyaxon_emotion_entailment_config.json

polyaxon/lif_3way_ap/conda.yml

@@ -0,0 +1,8 @@
+channels:
+  - defaults
+dependencies:
+  - python=3.8
+  - pip=21.0.1
+  - cudatoolkit=10.2 # comment out for local install
+  - pip:
+      - -r file:requirements.txt

polyaxon/lif_3way_ap/experiment.Dockerfile

@@ -0,0 +1,21 @@
+FROM registry.aisingapore.net/polyaxon/cuda10:latest
+
+ARG USER="polyaxon"
+ARG WORK_DIR="/home/$USER"
+
+RUN rm /bin/sh && ln -s /bin/bash /bin/sh && \
+    apt update && apt install -y jq ca-certificates
+
+WORKDIR $WORK_DIR
+USER $USER
+
+COPY build/polyaxon/lif_3way_ap/conda.yml .
+COPY build/sgnlp_models/models/lif_3way_ap/requirements.txt .
+RUN conda env update -f conda.yml -n base
+RUN python -m spacy download en_core_web_sm
+
+WORKDIR /code
+
+COPY --chown=$USER:$USER build .
+
+ENV LD_LIBRARY_PATH /usr/local/cuda/lib64:$LD_LIBRARY_PATH

polyaxon/lif_3way_ap/notebook.Dockerfile

@@ -0,0 +1,3 @@
+FROM registry.aisingapore.net/polyaxon/ml-workspace:latest
+
+WORKDIR /workspace

polyaxon/lif_3way_ap/notebook.yml

@@ -0,0 +1,16 @@
+---
+version: 1
+
+kind: notebook
+
+build:
+  dockerfile: polyaxon/l2af/notebook.Dockerfile
+  context: .
+
+environment:
+  persistence:
+    data: ["100e-nlp-lab-data"]
+    outputs: exp-output
+
+logging:
+  level: DEBUG

polyaxon/lif_3way_ap/train.yml

@@ -0,0 +1,31 @@
+---
+version: 1
+
+kind: experiment
+
+build:
+  dockerfile: polyaxon/lif_3way_ap/experiment.Dockerfile
+
+environment:
+  persistence:
+    data: ["100e-nlp-lab-data"]
+  resources:
+    gpu:
+      requests: 1
+      limits: 1
+
+framework: pytorch
+
+logging:
+  level: INFO
+
+run:
+  cmd: python -m sgnlp_models.models.lif_3way_ap.train \
+    --train_file="/polyaxon-data/workspace/jonheng/l2af/dataset/train.json" \
+    --validation_file="/polyaxon-data/workspace/jonheng/l2af/dataset/dev.json" \
+    --batch_size=32 \
+    --num_epoch=25 \
+    --seed=10 \
+    --model_config_path="/polyaxon-data/workspace/jonheng/l2af/model_config.json" \
+    --pretrained_embeddings_path="/polyaxon-data/workspace/jonheng/l2af/glove.840B.300d.txt" \
+    --output_dir="/polyaxon-data/workspace/jonheng/l2af/experiment-output/"

polyaxon/lsr/conda.yml

@@ -0,0 +1,18 @@
+channels:
+  - defaults
+dependencies:
+  - python=3.8
+  - pip=21.0.1
+  - cudatoolkit=10.2 # comment out for local install
+  - pip:
+    - nltk
+    - networkx
+    - numpy
+    - pandas
+    - scikit-learn
+    - sentencepiece
+    - spacy>=3
+    - tokenizers
+    - torch>=1.8,<2
+    - torchtext
+    - transformers

polyaxon/lsr/experiment.Dockerfile

@@ -0,0 +1,21 @@
+FROM registry.aisingapore.net/polyaxon/cuda10:latest
+
+ARG USER="polyaxon"
+ARG WORK_DIR="/home/$USER"
+
+RUN rm /bin/sh && ln -s /bin/bash /bin/sh && \
+    apt update && apt install -y jq ca-certificates
+
+WORKDIR $WORK_DIR
+USER $USER
+
+COPY build/polyaxon/lsr/conda.yml .
+RUN conda env update -f conda.yml -n base
+RUN python -m spacy download en_core_web_sm
+
+WORKDIR /code
+
+COPY --chown=$USER:$USER build .
+
+ENV LD_LIBRARY_PATH /usr/local/cuda/lib64:$LD_LIBRARY_PATH
+ENV TRANSFORMERS_CACHE="/polyaxon-data/workspace/$USER/.cache"

polyaxon/lsr/notebook.Dockerfile

@@ -0,0 +1,3 @@
+FROM registry.aisingapore.net/polyaxon/ml-workspace:latest
+
+WORKDIR /workspace

polyaxon/lsr/notebook.yml

@@ -0,0 +1,16 @@
+---
+version: 1
+
+kind: notebook
+
+build:
+  dockerfile: polyaxon/lsr/notebook.Dockerfile
+  context: .
+
+environment:
+  persistence:
+    data: ["100e-nlp-lab-data"]
+    outputs: exp-output
+
+logging:
+  level: DEBUG

polyaxon/lsr/train.yml

@@ -0,0 +1,31 @@
+---
+version: 1
+
+kind: experiment
+
+build:
+  dockerfile: polyaxon/lsr/experiment.Dockerfile
+
+environment:
+  persistence:
+    data: ["100e-nlp-lab-data"]
+  resources:
+    gpu:
+      requests: 1
+      limits: 1
+
+framework: pytorch
+
+logging:
+  level: INFO
+
+run:
+  cmd: python -m sgnlp_models.models.lsr.train \
+    --train_file="/polyaxon-data/workspace/jonheng/lsr/docred/train_annotated.json" \
+    --validation_file="/polyaxon-data/workspace/jonheng/lsr/docred/dev.json" \
+    --output_dir="/polyaxon-data/workspace/jonheng/lsr/experiment-output/bert" \
+    --metadata_dir="/polyaxon-data/workspace/jonheng/lsr/metadata" \
+    --batch_size=4 \
+    --num_epoch=100 \
+    --seed=64 \
+    --model_config_path="/polyaxon-data/workspace/jonheng/lsr/config/config_bert.json"

polyaxon/nea/conda.yml

@@ -0,0 +1,14 @@
+name: emotion_entailment
+channels:
+  - defaults
+dependencies:
+  - python=3.8
+  - pip=21.0.1
+  - cudatoolkit=10.2
+  - pip:
+      - transformers==4.4.2
+      - tokenizers==0.10.1
+      - torch==1.8.1
+      - nltk==3.5
+      - numpy
+      - wandb

polyaxon/nea/experiment.df

@@ -0,0 +1,20 @@
+FROM registry.aisingapore.net/polyaxon/cuda10:latest
+
+ARG USER="polyaxon"
+ARG WORK_DIR="/home/$USER"
+
+RUN rm /bin/sh && ln -s /bin/bash /bin/sh && \
+    apt update && apt install -y jq ca-certificates
+
+WORKDIR $WORK_DIR
+USER $USER
+
+COPY build/conda.yml .
+RUN conda env update -f conda.yml -n base && \
+    rm conda.yml
+
+WORKDIR /code
+
+COPY --chown=$USER:$USER build .
+
+ENV LD_LIBRARY_PATH /usr/local/cuda/lib64:$LD_LIBRARY_PATH

polyaxon/nea/notebook.df

@@ -0,0 +1,29 @@
+FROM registry.aisingapore.net/aiap/polyaxon/pytorch-tf2-cpu:latest
+
+ARG WORK_DIR="/code"
+
+RUN pip install jupyterlab==0.33.12
+
+WORKDIR $WORK_DIR
+
+RUN mkdir -p $WORK_DIR && chown -R 2222:2222 $WORK_DIR
+
+ARG ORG_JUPYTER="/opt/conda/bin/jupyter"
+ARG MOD_JUPYTER="/opt/conda/bin/jupyter.real"
+
+RUN mv $ORG_JUPYTER $MOD_JUPYTER && \
+    echo "#!/bin/bash" >  $ORG_JUPYTER && \
+    echo "/code/link_workspace.sh &" >> $ORG_JUPYTER && \
+    echo "export SHELL=/bin/bash" >> $ORG_JUPYTER && \
+    echo "$MOD_JUPYTER \"\$@\"" >>  $ORG_JUPYTER && \
+    chmod +x $ORG_JUPYTER
+
+COPY build/conda.yml /code
+COPY build/scripts/link_workspace.sh /code
+
+RUN apt-get update && apt-get -y install vim jq
+
+RUN conda env update -n polyaxon --file conda.yml
+RUN rm /code/conda.yml
+
+ENV LANG "C.UTF-8"