local agent cicd WIP #25
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Dagster Cloud Hybrid Deployment | |
| on: | |
| push: # For full deployments | |
| branches: | |
| - "main" | |
| - "master" | |
| pull_request: # For branch deployments | |
| types: [opened, synchronize, reopened, closed] | |
| concurrency: | |
| # Cancel in-progress deploys to the same branch | |
| group: ${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| # The organization name in Dagster Cloud | |
| DAGSTER_CLOUD_ORGANIZATION: "izzy-hybrid-sandbox" | |
| # The API token from https://dagster.cloud/ should be stored in Secrets | |
| DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} | |
| # Path to the root folder containing the dagster project | |
| DAGSTER_PROJECT_DIR: "." | |
| # Path to dagster_cloud.yaml relative to DAGSTER_PROJECT_DIR | |
| DAGSTER_CLOUD_YAML_PATH: "dagster_cloud.yaml" | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # SSH variables for authentication with the machine hosting the local agent | |
| SSH_USER: ${{ secrets.SSH_USER }} | |
| SSH_HOST: ${{ secrets.SSH_HOST }} | |
| SERVER_IP: ${{ secrets.SERVER_IP }} | |
| # Path to the root folder of the dagster project on the machine running the local agent | |
| DAGSTER_PROJECT_PATH: ${{ secrets.DAGSTER_PROJECT_PATH }} | |
| # Append datetime to venv, so each venv is unique | |
| NEW_VENV: "venv_$(date +%Y%m%d_%H%M%S)" | |
| jobs: | |
| dagster-cloud-deploy: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| # Checkout the project | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| if: steps.prerun.outputs.result != 'skip' | |
| with: | |
| ref: ${{ github.head_ref }} | |
| # Validate dagster_cloud.yaml and the connection to dagster.cloud | |
| - name: Validate configuration | |
| id: ci-validate | |
| if: steps.prerun.outputs.result != 'skip' | |
| uses: dagster-io/dagster-cloud-action/actions/utils/[email protected] | |
| with: | |
| command: "ci check --project-dir ${{ env.DAGSTER_PROJECT_DIR }} --dagster-cloud-yaml-path ${{ env.DAGSTER_CLOUD_YAML_PATH }}" | |
| # Optional — Use SSH to authenticate with the machine hosting the local agent | |
| - name: Set up SSH | |
| uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| # Add known host to authenticate via SSH | |
| - name: Add known host | |
| run: | | |
| mkdir -p ~/.ssh | |
| ssh-keyscan -t ed25519 ${{ secrets.SSH_HOST }} >> ~/.ssh/known_hosts | |
| # Create and activate venv, checkout main branch, pull latest changes, install latest deps | |
| - name: Update Dagster project on the local server | |
| run: | | |
| ssh -vvv ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} ' | |
| cd ${{ secrets.DAGSTER_PROJECT_PATH }} && \ | |
| python3 -m venv "$NEW_VENV" && \ | |
| echo "Virtual environment created at: $(realpath "\$NEW_VENV")" | |
| echo "Activating virtual environment from \$NEW_VENV/bin/activate" | |
| source "$NEW_VENV/bin/activate" && \ | |
| current_branch=$(git rev-parse --abbrev-ref HEAD) && \ | |
| if [ "$current_branch" != "main" ]; then | |
| git checkout main | |
| fi && \ | |
| git pull && \ | |
| pip install -e . | |
| ' | |
| # run dbt deps and dbt parse to generate up to date manifest.json | |
| - name: Update dbt manifest | |
| run: | | |
| ssh -vvv ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} ' | |
| cd ${{ secrets.DAGSTER_PROJECT_PATH }} && \ | |
| source "$NEW_VENV/bin/activate" && \ | |
| cd my_dbt_project && \ | |
| dbt deps && \ | |
| dbt parse | |
| ' | |
| # Install Dagster Cloud CLI to push changes to Dagster+ | |
| - name: Install Dagster Cloud CLI | |
| run: pip install dagster-cloud | |
| # Push changes to Dagster+ | |
| - name: Add or Update Dagster Cloud Code Location | |
| run: | | |
| dagster-cloud deployment sync-locations -w dagster_cloud.yaml \ | |
| --api-token ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} \ | |
| --deployment "prod" |