Skip to content

Commit 4ee75fa

Browse files
deep-diverdeep-diver
authored
Merge pull request #5 from codingpot/feat/wandb
Feat/wandb
2 parents 64540a7 + 0ec7e02 commit 4ee75fa

File tree

7 files changed

+399
-117
lines changed

7 files changed

+399
-117
lines changed

‎.github/actions/exp/jl-dvc/action.yml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
name: "Run experiments with JarvisLabs and DVCLive"
2+
description: "Run experiments with JarvisLabs and DVCLive"
3+
inputs:
4+
CUR_BRANCH:
5+
required: true
6+
CUR_PR_ID:
7+
required: true
8+
GDRIVE_CREDENTIAL:
9+
required: true
10+
GH_ACCESS_TOKEN:
11+
required: true
12+
JARVISLABS_ID:
13+
required: true
14+
JARVISLABS_ACCESS_TOKEN:
15+
required: true
16+
runs:
17+
using: "composite"
18+
steps:
19+
- name: prepare script
20+
if: steps.check.outputs.triggered == 'true'
21+
env:
22+
CUR_BRANCH: ${{ steps.pr_data.outputs.branch }}
23+
CUR_PR_ID: ${{ steps.pr_data.outputs.number }}
24+
GDRIVE_CREDENTIAL: ${{ secrets.GDRIVE_CREDENTIAL }}
25+
GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
26+
CLOUD_ID: ${{ env.JARVISLABS_ID }}
27+
CLOUD_AT: ${{ env.JARVISLABS_ACCESS_TOKEN }}
28+
run: |
29+
envsubst \
30+
'$CUR_BRANCH, \
31+
$CUR_PR_ID, \
32+
$GDRIVE_CREDENTIAL, \
33+
$GH_ACCESS_TOKEN, \
34+
$CLOUD_ID, \
35+
$CLOUD_AT' \
36+
< scripts/experiments.sh \
37+
> scripts/experiments_tmp.sh
38+
39+
- name: install jarvislabs client
40+
if: steps.check.outputs.triggered == 'true'
41+
run: |
42+
pip install typer
43+
pip install git+https://github.com/jarvislabsai/jlclient.git
44+
45+
- name: add script to jarvislabs
46+
id: add_script
47+
if: steps.check.outputs.triggered == 'true'
48+
run: |
49+
python clouds/jarvislabs.py \
50+
script add \
51+
${{ env.CLOUD_AT }} ${{ env.CLOUD_ID }} scripts/experiments_tmp.sh \
52+
> outputs.txt
53+
echo "::set-output name=script_id::$(cat outputs.txt)"
54+
55+
- name: create vm on jarvislabs
56+
if: steps.check.outputs.triggered == 'true'
57+
run: |
58+
python clouds/jarvislabs.py \
59+
vm create \
60+
${{ env.CLOUD_AT }} ${{ env.CLOUD_ID }} ${{ steps.add_script.outputs.script_id }}
61+
62+
- name: remove script from jarvislabs
63+
if: steps.check.outputs.triggered == 'true'
64+
run: |
65+
python clouds/jarvislabs.py \
66+
script remove \
67+
${{ env.CLOUD_AT }} ${{ env.CLOUD_ID }} ${{ steps.add_script.outputs.script_id }}

‎.github/actions/exp/jl-wandb/action.yml‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
name: "Run experiments with JarvisLabs and W&B"
2+
description: "Run experiments with JarvisLabs and W&B"
3+
inputs:
4+
CUR_BRANCH:
5+
required: true
6+
CUR_PR_ID:
7+
required: true
8+
GDRIVE_CREDENTIAL:
9+
required: true
10+
GH_ACCESS_TOKEN:
11+
required: true
12+
JARVISLABS_ID:
13+
required: true
14+
JARVISLABS_ACCESS_TOKEN:
15+
required: true
16+
WANDB_PROJECT:
17+
required: true
18+
WANDB_API_KEY:
19+
required: true
20+
runs:
21+
using: "composite"
22+
steps:
23+
- name: prepare script
24+
env:
25+
CUR_BRANCH: $CUR_BRANCH
26+
CUR_PR_ID: $CUR_PR_ID
27+
GDRIVE_CREDENTIAL: $GDRIVE_CREDENTIAL
28+
GH_ACCESS_TOKEN: $GH_ACCESS_TOKEN
29+
CLOUD_ID: $JARVISLABS_ID
30+
CLOUD_AT: $JARVISLABS_ACCESS_TOKEN
31+
WANDB_PROJECT: $WANDB_PROJECT
32+
WANDB_API_KEY: $WANDB_API_KEY
33+
run: |
34+
envsubst \
35+
'$CUR_BRANCH, \
36+
$CUR_PR_ID, \
37+
$GDRIVE_CREDENTIAL, \
38+
$GH_ACCESS_TOKEN, \
39+
$CLOUD_ID, \
40+
$CLOUD_AT' \
41+
< scripts/js_exp_wandb.sh \
42+
> scripts/js_exp_wandb_tmp.sh
43+
44+
- name: install jarvislabs client
45+
run: |
46+
pip install typer
47+
pip install git+https://github.com/jarvislabsai/jlclient.git
48+
49+
- name: add script to jarvislabs
50+
id: add_script
51+
run: |
52+
python clouds/jarvislabs.py \
53+
script add \
54+
${{ env.CLOUD_AT }} ${{ env.CLOUD_ID }} scripts/js_exp_wandb_tmp.sh \
55+
> outputs.txt
56+
echo "::set-output name=script_id::$(cat outputs.txt)"
57+
58+
- name: create vm on jarvislabs
59+
run: |
60+
python clouds/jarvislabs.py \
61+
vm create \
62+
${{ env.CLOUD_AT }} ${{ env.CLOUD_ID }} ${{ steps.add_script.outputs.script_id }}
63+
64+
- name: remove script from jarvislabs
65+
run: |
66+
python clouds/jarvislabs.py \
67+
script remove \
68+
${{ env.CLOUD_AT }} ${{ env.CLOUD_ID }} ${{ steps.add_script.outputs.script_id }}

‎.github/workflows/experiments.yml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
name: Train Trigger
2+
3+
on:
4+
pull_request:
5+
types: [opened]
6+
7+
issue_comment:
8+
types: [created]
9+
10+
jobs:
11+
experiments-on-jarvislabs:
12+
runs-on: ubuntu-latest
13+
steps:
14+
- uses: octokit/[email protected]
15+
name: GitHub API Request
16+
id: request
17+
with:
18+
route: ${{ github.event.issue.pull_request.url }}
19+
env:
20+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
21+
22+
- name: Get PR informations
23+
id: pr_data
24+
run: |
25+
echo "::set-output name=number::${{ fromJson(steps.request.outputs.data).number }}"
26+
echo "::set-output name=branch::${{ fromJson(steps.request.outputs.data).head.ref }}"
27+
echo "::set-output name=repo_owner::${{ github.event.repository.owner.login }}"
28+
echo "::set-output name=comment_owner::${{ github.event.sender.login }}"
29+
echo "::set-output name=comment::${{ github.event.comment.body }}"
30+
31+
- name: Extract comment
32+
if: ${{ steps.pr_data.outputs.repo_owner == steps.pr_data.outputs.comment_owner }}
33+
run: |
34+
echo "Eligible!!"
35+
36+
- uses: khan/[email protected]
37+
name: Listen to comment on PR (training)
38+
id: check_dvclive
39+
if: ${{ steps.pr_data.outputs.repo_owner == steps.pr_data.outputs.comment_owner }}
40+
with:
41+
trigger: '#train --with dvclive'
42+
env:
43+
GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
44+
45+
- uses: khan/[email protected]
46+
name: Listen to comment on PR (training)
47+
id: check_wandb
48+
if: ${{ steps.pr_data.outputs.repo_owner == steps.pr_data.outputs.comment_owner }}
49+
with:
50+
trigger: '#train --with wandb'
51+
env:
52+
GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
53+
54+
- uses: actions/checkout@v3
55+
name: clone branch of PR
56+
with:
57+
token: ${{ secrets.GITHUB_TOKEN }}
58+
ref: ${{ steps.pr_data.outputs.branch }}
59+
60+
- name: clone branch of PR
61+
if: steps.check_dvclive.outputs.triggered == 'true'
62+
uses: ./.github/actions/exp/jl-dvc
63+
with:
64+
CUR_BRANCH: ${{ steps.pr_data.outputs.branch }}
65+
CUR_PR_ID: ${{ steps.pr_data.outputs.number }}
66+
GDRIVE_CREDENTIAL: ${{ secrets.GDRIVE_CREDENTIAL }}
67+
GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
68+
JARVISLABS_ID: ${{ secrets.JARVISLABS_USER_EMAIL }}
69+
JARVISLABS_ACCESS_TOKEN: ${{ secrets.JARVISLABS_ACCESS_TOKEN }}
70+
71+
- name: clone branch of PR
72+
if: steps.check_wandb.outputs.triggered == 'true'
73+
uses: ./.github/actions/exp/jl-wandb
74+
with:
75+
CUR_BRANCH: ${{ steps.pr_data.outputs.branch }}
76+
CUR_PR_ID: ${{ steps.pr_data.outputs.number }}
77+
GDRIVE_CREDENTIAL: ${{ secrets.GDRIVE_CREDENTIAL }}
78+
GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
79+
JARVISLABS_ID: ${{ secrets.JARVISLABS_USER_EMAIL }}
80+
JARVISLABS_ACCESS_TOKEN: ${{ secrets.JARVISLABS_ACCESS_TOKEN }}
81+
WANDB_PROJECT: ${{ secrets.WANDB_PROJECT }}
82+
WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}

‎.github/workflows/jarvislabs-experiments.yml‎

Lines changed: 0 additions & 105 deletions
This file was deleted.

‎README.md‎

Lines changed: 32 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,12 @@ This project shows how to realize MLOps in Git/GitHub. In order to achieve this
2121
4. Run `dvc add [ADDED FILE OR DIRECTORY]` to track your data with DVC
2222
5. Run `dvc remote add -d gdrive_storage gdrive://[ID of specific folder in gdrive]` to add Google Drive as the remote data storage
2323
6. Run `dvc push`, then URL to auth is provided. Copy and paste it to the browser, and autheticate
24-
7. Copy the content of `.dvc/tmp/gdrive-user-credentials.json` and put it as in [GitHub Secret](https://docs.github.com/en/actions/security-guides/encrypted-secrets#creating-encrypted-secrets-for-a-repository) with the name of `GDRIVE_CREDENTIALS`
24+
7. Copy the content of `.dvc/tmp/gdrive-user-credentials.json` and put it as in [GitHub Secret](https://docs.github.com/en/actions/security-guides/encrypted-secrets#creating-encrypted-secrets-for-a-repository) with the name of `GDRIVE_CREDENTIAL`
2525
8. Run `git add . && git commit -m "initial commit" && git push origin main` to keep the initial setup
2626
9. Write your own pipeline under `pipeline` directory. Codes for basic image classification in TensorFlow are provided initially.
2727
10. Run the following `dvc stage add` for training stage
2828
```bash
29+
# if you want to use Iterative Studio / DVCLive for tracking training progress
2930
$ dvc stage add -n train \
3031
-p train.train_size,train.batch_size,train.epoch,train.lr \
3132
-d pipeline/modeling.py -d pipeline/train.py -d data \
@@ -35,25 +36,44 @@ $ dvc stage add -n train \
3536
--plots-no-cache dvclive/scalars/eval/sparse_categorical_accuracy.tsv \
3637
-o outputs/model \
3738
python pipeline/train.py outputs/model
39+
40+
# if you want to use W&B for tracking training progress
41+
$ dvc stage add -n train \
42+
-p train.train_size,train.batch_size,train.epoch,train.lr \
43+
-d pipeline/modeling.py -d pipeline/train.py -d data \
44+
-o outputs/model \
45+
python pipeline/train.py outputs/model
3846
```
39-
10. Run the following `dvc stage add` for evaluate stage
47+
11. Run the following `dvc stage add` for evaluate stage
4048
```bash
49+
# if you want to use Iterative Studio / DVCLive for tracking training progress
4150
$ dvc stage add -n evaluate \
4251
-p evaluate.test,evaluate.batch_size \
4352
-d pipeline/evaluate.py -d data/test -d outputs/model \
4453
-M outputs/metrics.json \
4554
python pipeline/evaluate.py outputs/model
55+
56+
# if you want to use W&B for tracking training progress
57+
$ dvc stage add -n evaluate \
58+
-p evaluate.test,evaluate.batch_size \
59+
-d pipeline/evaluate.py -d data/test -d outputs/model \
60+
python pipeline/evaluate.py outputs/model
4661
```
47-
11. Update `params.yaml` as you need.
48-
12. Run `git add . && git commit -m "add initial pipeline setup" && git push origin main`
49-
13. Run `dvc repro` to run the pipeline initially
50-
14. Run `dvc add outputs/model.tar.gz` to add compressed version of model
51-
15. Run `dvc push outputs/model.tar.gz`
52-
16. Run `echo "/pipeline/__pycache__" >> .gitignore` to ignore unnecessary directory
53-
17. Run `git add . && git commit -m "add initial pipeline run" && git push origin main`
54-
18. Add access token and user email of [JarvisLabs.ai](https://jarvislabs.ai/) to GitHub Secret as `JARVISLABS_ACCESS_TOKEN` and `JARVISLABS_USER_EMAIL`
55-
19. Add GitHub access token to GitHub Secret as `GH_ACCESS_TOKEN`
56-
20. Create a PR and write `#train` as in comment (you have to be the onwer of the repo)
62+
12. Update `params.yaml` as you need.
63+
13. Run `git add . && git commit -m "add initial pipeline setup" && git push origin main`
64+
14. Run `dvc repro` to run the pipeline initially
65+
15. Run `dvc add outputs/model.tar.gz` to add compressed version of model
66+
16. Run `dvc push outputs/model.tar.gz`
67+
17. Run `echo "/pipeline/__pycache__" >> .gitignore` to ignore unnecessary directory
68+
18. Run `git add . && git commit -m "add initial pipeline run" && git push origin main`
69+
19. Add access token and user email of [JarvisLabs.ai](https://jarvislabs.ai/) to GitHub Secret as `JARVISLABS_ACCESS_TOKEN` and `JARVISLABS_USER_EMAIL`
70+
20. Add GitHub access token to GitHub Secret as `GH_ACCESS_TOKEN`
71+
21. Create a PR and write `#train` as in comment (you have to be the onwer of the repo)
72+
73+
### W&B Integration Setup
74+
75+
1. Add W&B's project name to GitHub Secret as `WANDB_PROJECT`
76+
2. Add W&B's API KEY to GitHub Secret as `WANDB_API_KEY`
5777

5878
### HuggingFace Integration Setup
5979

0 commit comments

Comments
 (0)