local agent cicd WIP #15
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Dagster Cloud Hybrid Deployment | |
| on: | |
| push: # For full deployments | |
| branches: | |
| - "main" | |
| - "master" | |
| pull_request: # For branch deployments | |
| types: [opened, synchronize, reopened, closed] | |
| concurrency: | |
| # Cancel in-progress deploys to the same branch | |
| group: ${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| # The organization name in Dagster Cloud | |
| DAGSTER_CLOUD_ORGANIZATION: "izzy-hybrid-sandbox" | |
| # The API token from https://dagster.cloud/ should be stored in Secrets | |
| DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} | |
| # Path to the root folder containing the dagster project | |
| DAGSTER_PROJECT_DIR: "." | |
| # Path to dagster_cloud.yaml relative to DAGSTER_PROJECT_DIR | |
| DAGSTER_CLOUD_YAML_PATH: "dagster_cloud.yaml" | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| SSH_USER: ${{ secrets.SSH_USER }} | |
| SSH_HOST: ${{ secrets.SSH_HOST }} | |
| SERVER_IP: ${{ secrets.SERVER_IP }} | |
| DAGSTER_PROJECT_PATH: ${{ secrets.DAGSTER_PROJECT_PATH }} | |
| jobs: | |
| dagster-cloud-deploy: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| # Checkout the project | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| if: steps.prerun.outputs.result != 'skip' | |
| with: | |
| ref: ${{ github.head_ref }} | |
| # Validate dagster_cloud.yaml and the connection to dagster.cloud | |
| - name: Validate configuration | |
| id: ci-validate | |
| if: steps.prerun.outputs.result != 'skip' | |
| uses: dagster-io/dagster-cloud-action/actions/utils/[email protected] | |
| with: | |
| command: "ci check --project-dir ${{ env.DAGSTER_PROJECT_DIR }} --dagster-cloud-yaml-path ${{ env.DAGSTER_CLOUD_YAML_PATH }}" | |
| # | |
| - name: Install Dagster Cloud CLI | |
| run: pip install dagster-cloud | |
| - name: Set up SSH | |
| uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| - name: Add known host | |
| run: | | |
| mkdir -p ~/.ssh | |
| ssh-keyscan -t ed25519 ${{ secrets.SSH_HOST }} >> ~/.ssh/known_hosts | |
| - name: Update Dagster project on the server | |
| run: | | |
| ssh -vvv ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} 'echo "SSH connection successful"' | |
| ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} ' | |
| cd ${{ secrets.DAGSTER_PROJECT_PATH }} && \ | |
| current_branch=$(git rev-parse --abbrev-ref HEAD) && \ | |
| if [ "$current_branch" != "main" ]; then | |
| git checkout main | |
| fi && \ | |
| git fetch origin && \ | |
| git reset --hard origin/main | |
| ' | |
| - name: Update dbt manifest | |
| run: | | |
| ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} ' | |
| cd ${{ secrets.DAGSTER_PROJECT_PATH }} && \ | |
| dagster-dbt project prepare-and-package --file my_dagster_code_location/project.py | |
| ' | |
| - name: Add or Update Dagster Cloud Code Location | |
| run: | | |
| dagster-cloud workspace add-location my_dagster_code_location \ | |
| --module-name my_dagster_code_location \ | |
| --api-token ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} | |
| # Update a PR comment - this runs always() so the comment is updated on success and failure | |
| - name: Update PR comment for branch deployments | |
| id: ci-notify | |
| if: steps.prerun.outputs.result != 'skip' && always() | |
| uses: dagster-io/dagster-cloud-action/actions/utils/[email protected] | |
| with: | |
| command: "ci notify --project-dir=${{ env.DAGSTER_PROJECT_DIR }}" | |
| # Generate a summary that shows up on the Workflow Summary page | |
| - name: Generate a summary | |
| id: ci-summary | |
| if: steps.prerun.outputs.result != 'skip' && always() | |
| uses: dagster-io/dagster-cloud-action/actions/utils/[email protected] | |
| with: | |
| command: "ci status --output-format=markdown >> $GITHUB_STEP_SUMMARY" |