diff --git a/.github/workflows/nested_eagle.yml b/.github/workflows/nested_eagle.yml index 6421ca1..0188825 100644 --- a/.github/workflows/nested_eagle.yml +++ b/.github/workflows/nested_eagle.yml @@ -1,12 +1,12 @@ -# This is a basic workflow that is manually triggered -name: Nested Eagle Pipeline +name: Nested Eagle CI Pipeline # Controls when the action will run. Workflow runs when manually triggered using the UI # or API. on: - workflow_dispatch: -# A workflow run is made up of one or more jobs that can run sequentially or in parallel + pull_request: + branches: + - main jobs: # This workflow contains a job called submit_grids. Submit_Grids: @@ -16,10 +16,10 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: - - name: Checkout a specific branch + - name: Checkout pull request branch uses: actions/checkout@v4 with: - ref: main + ref: ${{ github.event.pull_request.head.ref }} - name: Submit Initial Grids run: | cd nested_eagle/scientific_workflow/data @@ -126,10 +126,3 @@ jobs: done echo "Slurm job $JOB_ID to perform validation has been completed." - - - - - - - diff --git a/.github/workflows/nested_eagle_manual.yml b/.github/workflows/nested_eagle_manual.yml new file mode 100644 index 0000000..6421ca1 --- /dev/null +++ b/.github/workflows/nested_eagle_manual.yml @@ -0,0 +1,135 @@ +# This is a basic workflow that is manually triggered + +name: Nested Eagle Pipeline + +# Controls when the action will run. Workflow runs when manually triggered using the UI +# or API. +on: + workflow_dispatch: +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a job called submit_grids. + Submit_Grids: + # The type of runner that the job will run on + runs-on: ursa + timeout-minutes: 1440 + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - name: Checkout a specific branch + uses: actions/checkout@v4 + with: + ref: main + - name: Submit Initial Grids + run: | + cd nested_eagle/scientific_workflow/data + + + + JOB_ID=$(sbatch submit_grids.sh | awk '{print $4}') + echo "Submitted Slurm job with ID: $JOB_ID" + + while squeue -j $JOB_ID &>/dev/null; do + echo "Slurm job $JOB_ID is still running or pending. Waiting..." + sleep 30 # Wait for 30 seconds before checking again + done + + echo "Slurm job $JOB_ID to generate the grids has completed." + + Generate_Forecasts: + # The type of runner that the job will run on + runs-on: ursa + timeout-minutes: 1440 + needs: Submit_Grids + steps: + - name: Generate GFS & HRRR Forecasts + run: | + cd nested_eagle/scientific_workflow/data + echo "Slurm jobs to generate the GFS & HRRR forecasts have completed." + + JOB_ID1=$(sbatch submit_gfs.sh | awk '{print $4}') + JOB_ID2=$(sbatch submit_hrrr.sh | awk '{print $4}') + + echo "Submitted Slurm job with ID: $JOB_ID1" + echo "Submitted Slurm job with ID: $JOB_ID2" + while squeue -h -j ${JOB_ID1},${JOB_ID2} | grep -q "${JOB_ID1}\|${JOB_ID2}"; do + squeue -u $USER + sleep 30 # Wait for 30 seconds before checking again. + done + + echo "Slurm jobs $JOB_ID1 and $JOB_ID2 to generate the GFS & HRRR forecasts have completed." + Run_Training: + # The type of runner that the job will run on + runs-on: ursa + timeout-minutes: 1440 + needs: Generate_Forecasts + steps: + - name: Run Training Against the Generated Forecasts + run: | + cd nested_eagle/scientific_workflow/training + + + JOB_ID=$(sbatch submit_training.sh | awk '{print $4}') + CHECKPOINT="$PWD/outputs/checkpoint" + + echo "Submitted Slurm job with ID: $JOB_ID" + while [ ! -d "$CHECKPOINT" ]; do + echo "Directory '$CHECKPOINT' doesn't exist yet." + squeue -u $USER + sleep 30 # Wait for 30 seconds before checking again. + done + scancel $JOB_ID + + echo "Slurm job to train the data has been completed." + + + Run_Inference: + # The type of runner that the job will run on + runs-on: ursa + timeout-minutes: 1440 + needs: Run_Training + steps: + - name: Run Inference against the training data + run: | + cd nested_eagle/scientific_workflow/inference + + JOB_ID=$(sbatch submit_inference.sh | awk '{print $4}') + + + echo "Submitted Inference Slurm job with ID: $JOB_ID" + + while squeue -h -j ${JOB_ID} | grep -q "${JOB_ID}"; do + squeue -u $USER + sleep 30 # Wait for 30 seconds before checking again. + done + + echo "Slurm job $JOB_ID to run inference on the data has been completed." + + Perform_Validation: + # The type of runner that the job will run on + runs-on: ursa + timeout-minutes: 1440 + needs: Run_Inference + steps: + - name: Perform validation against the results + run: | + cd nested_eagle/scientific_workflow/validation + + JOB_ID=$(sbatch submit_validation.sh | awk '{print $4}') + + + echo "Submitted Validation Slurm job with ID: $JOB_ID" + + while squeue -h -j ${JOB_ID} | grep -q "${JOB_ID}"; do + squeue -u $USER + sleep 30 # Wait for 30 seconds before checking again. + done + + echo "Slurm job $JOB_ID to perform validation has been completed." + + + + + + +