diff --git a/.github/workflows/CD.yml b/.github/workflows/CD.yml new file mode 100644 index 000000000..5a6432bbc --- /dev/null +++ b/.github/workflows/CD.yml @@ -0,0 +1,85 @@ +name: CD + +on: + push: + branches: [main] + pull_request: + branches: [main] + release: + types: [published] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v3 + + - name: Set up Python ${{ matrix.python-version }} + run: uv python install ${{ matrix.python-version }} + + - name: Install the project + run: uv sync --all-extras --dev + + - name: Run build + run: uv build + publish_pypi: + if: github.event_name == 'release' + needs: build + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - name: Install uv + uses: astral-sh/setup-uv@v3 + - name: Set up Python 3.12 + run: uv python install 3.12 + - name: Install the project + run: uv sync --all-extras --dev + - name: Run build + run: uv build + - name: Publish package + uses: pypa/gh-action-pypi-publish@v1.10.3 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + publish_docker: + runs-on: ubuntu-latest + continue-on-error: true + permissions: + packages: write + contents: read + attestations: write + id-token: write + steps: + - name: Check out the repo + uses: actions/checkout@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + + - name: Build and push Docker image + id: push + uses: docker/build-push-action@v5 + with: + context: . + file: ./containers/Containerfile.cuda + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/run_test.yml b/.github/workflows/CI.yml similarity index 50% rename from .github/workflows/run_test.yml rename to .github/workflows/CI.yml index aeacddc02..bdfa73447 100644 --- a/.github/workflows/run_test.yml +++ b/.github/workflows/CI.yml @@ -1,20 +1,17 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python +name: CI -name: Run Tests - -on: [push, pull_request] +on: + pull_request: + push: + branches: [main] jobs: build: - runs-on: ubuntu-latest strategy: fail-fast: false matrix: python-version: ["3.11", "3.12"] - - steps: - uses: actions/checkout@v4 @@ -35,4 +32,23 @@ jobs: COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} - name: Run build - run: uv build \ No newline at end of file + run: uv build + pre-commit: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + python -m pip install . + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml deleted file mode 100644 index 268173710..000000000 --- a/.github/workflows/container.yml +++ /dev/null @@ -1,56 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -# GitHub recommends pinning actions to a commit SHA. -# To get a newer version, you will need to update the SHA. -# You can also reference a tag or branch, but the action may change without warning. - -name: Publish Docker image - -on: - release: - types: [published] - -jobs: - push_to_registry: - name: Push Docker image to Docker Hub - runs-on: ubuntu-latest - permissions: - packages: write - contents: read - attestations: write - id-token: write - steps: - - name: Check out the repo - uses: actions/checkout@v4 - - - name: Log in to Docker Hub - uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: my-docker-hub-namespace/my-docker-hub-repository - - - name: Build and push Docker image - id: push - uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 - with: - context: . - file: ./containers/Containerfile.cuda - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - - name: Generate artifact attestation - uses: actions/attest-build-provenance@v2 - with: - subject-name: index.docker.io/my-docker-hub-namespace/my-docker-hub-repository - subject-digest: ${{ steps.push.outputs.digest }} - push-to-registry: true diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml deleted file mode 100644 index de71f8257..000000000 --- a/.github/workflows/pre-commit.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: pre-commit - -on: - pull_request: - push: - branches: [main] - -jobs: - - pre-commit: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.11", "3.12"] - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - python -m pip install . - - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml deleted file mode 100644 index 4a97e012d..000000000 --- a/.github/workflows/python-publish.yml +++ /dev/null @@ -1,43 +0,0 @@ -# This workflow will upload a Python Package using Twine when a release is created -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: Upload Python Package - -on: - release: - types: [published] - -permissions: - contents: read - -jobs: - deploy: - - runs-on: ubuntu-latest - - - steps: - - uses: actions/checkout@v4 - - - name: Install uv - uses: astral-sh/setup-uv@v3 - - - name: Set up Python ${{ matrix.python-version }} - run: uv python install ${{ matrix.python-version }} - - - name: Install the project - run: uv sync --all-extras --dev - - - name: Run build - run: uv build - - - name: Publish package - uses: pypa/gh-action-pypi-publish@v1.10.3 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/containers/Containerfile.cpu b/containers/Containerfile.cpu index d61eac810..ea5adfa0f 100644 --- a/containers/Containerfile.cpu +++ b/containers/Containerfile.cpu @@ -12,4 +12,4 @@ WORKDIR /home/jim RUN git checkout jim-dev # Run uv sync -RUN uv sync --extra pipeline \ No newline at end of file +RUN uv sync --extra dagster \ No newline at end of file diff --git a/containers/Containerfile.cuda b/containers/Containerfile.cuda index b297aa920..c394a5f5d 100644 --- a/containers/Containerfile.cuda +++ b/containers/Containerfile.cuda @@ -12,4 +12,4 @@ WORKDIR /home/jim RUN git checkout jim-dev # Run uv sync -RUN uv sync --extra cuda --extra pipeline \ No newline at end of file +RUN uv sync --extra cuda --extra dagster \ No newline at end of file diff --git a/jim_dagster/SingleRun/assets.py b/jim_dagster/SingleRun/assets.py deleted file mode 100644 index a4d069845..000000000 --- a/jim_dagster/SingleRun/assets.py +++ /dev/null @@ -1,182 +0,0 @@ -import dagster as dg - - -# Create asset group for run and configuration0 - - -@dg.asset( - key_prefix="SingleRun", - group_name="prerun", - description="Configuration file for the run.", -) -def config_file(): - pass - - -@dg.multi_asset( - specs=[ - dg.AssetSpec("SingleRun_strain", deps=[config_file]), - dg.AssetSpec("SingleRun_psd", deps=[config_file]), - ], - group_name="prerun", -) -def raw_data(): - """ - This is a placeholder function for the raw data. - It is used to demonstrate how to create a Dagster asset. - """ - pass - - -@dg.multi_asset( - specs=[ - dg.AssetSpec("SingleRun_training_chains", deps=[raw_data]), - dg.AssetSpec("SingleRun_training_log_prob", deps=[raw_data]), - dg.AssetSpec("SingleRun_training_local_acceptance", deps=[raw_data]), - dg.AssetSpec("SingleRun_training_global_acceptance", deps=[raw_data]), - dg.AssetSpec("SingleRun_training_loss", deps=[raw_data]), - dg.AssetSpec("SingleRun_production_chains", deps=[raw_data]), - dg.AssetSpec("SingleRun_production_log_prob", deps=[raw_data]), - dg.AssetSpec("SingleRun_production_local_acceptance", deps=[raw_data]), - dg.AssetSpec("SingleRun_production_global_acceptance", deps=[raw_data]), - dg.AssetSpec("SingleRun_auxiliary_nf_samples", deps=[raw_data]), - dg.AssetSpec("SingleRun_auxiliary_prior_samples", deps=[raw_data]), - ], - group_name="run", -) -def run(): - """ - This is a placeholder function for the IMRPhenomPv2StandardCBCRun class. - It is used to demonstrate how to create a Dagster asset. - """ - pass - - -# Create asset group for diagnostics - - -@dg.asset( - group_name="diagnostics", deps=["SingleRun_training_loss"], key_prefix="SingleRun" -) -def loss_plot(): - pass - - -@dg.asset( - group_name="diagnostics", deps=["SingleRun_training_chains"], key_prefix="SingleRun" -) -def training_chains_corner_plot(): - pass - - -@dg.asset( - group_name="diagnostics", deps=["SingleRun_training_chains"], key_prefix="SingleRun" -) -def training_chains_trace_plot(): - pass - - -@dg.asset( - group_name="diagnostics", deps=["SingleRun_training_chains"], key_prefix="SingleRun" -) -def training_chains_rhat_plot(): - pass - - -@dg.asset( - group_name="diagnostics", - deps=["SingleRun_training_log_prob"], - key_prefix="SingleRun", -) -def training_log_prob_distribution(): - pass - - -@dg.asset( - group_name="diagnostics", - deps=["SingleRun_training_log_prob"], - key_prefix="SingleRun", -) -def training_log_prob_evolution(): - pass - - -@dg.asset( - group_name="diagnostics", - deps=["SingleRun_training_local_acceptance"], - key_prefix="SingleRun", -) -def training_local_acceptance_plot(): - pass - - -@dg.asset( - group_name="diagnostics", - deps=["SingleRun_training_global_acceptance"], - key_prefix="SingleRun", -) -def training_global_acceptance_plot(): - pass - - -@dg.asset( - group_name="diagnostics", - deps=["SingleRun_production_chains"], - key_prefix="SingleRun", -) -def production_chains_corner_plot(): - pass - - -@dg.asset( - group_name="diagnostics", - deps=["SingleRun_production_chains"], - key_prefix="SingleRun", -) -def production_chains_trace_plot(): - pass - - -@dg.asset( - group_name="diagnostics", - deps=["SingleRun_production_chains"], - key_prefix="SingleRun", -) -def production_chains_rhat_plot(): - pass - - -@dg.asset( - group_name="diagnostics", - deps=["SingleRun_production_log_prob"], - key_prefix="SingleRun", -) -def production_log_prob_distribution(): - pass - - -@dg.asset( - group_name="diagnostics", - deps=["SingleRun_production_log_prob"], - key_prefix="SingleRun", -) -def production_log_prob_evolution(): - pass - - -@dg.asset( - group_name="diagnostics", - deps=["SingleRun_production_local_acceptance"], - key_prefix="SingleRun", -) -def production_local_acceptance_plot(): - pass - - -@dg.asset( - group_name="diagnostics", - deps=["SingleRun_production_global_acceptance"], - key_prefix="SingleRun", -) -def production_global_acceptance_plot(): - pass diff --git a/jim_dagster/SingleRun/definitions.py b/jim_dagster/SingleRun/definitions.py deleted file mode 100644 index a36556aa3..000000000 --- a/jim_dagster/SingleRun/definitions.py +++ /dev/null @@ -1,9 +0,0 @@ -from dagster import Definitions, load_assets_from_modules - -from . import assets - -all_assets = load_assets_from_modules([assets]) - -defs = Definitions( - assets=all_assets, -) diff --git a/jim_dagster/__init__.py b/jim_dagster/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/jim_dagster/InjectionRecovery/__init__.py b/pipeline/InjectionRecovery/__init__.py similarity index 100% rename from jim_dagster/InjectionRecovery/__init__.py rename to pipeline/InjectionRecovery/__init__.py diff --git a/jim_dagster/InjectionRecovery/assets.py b/pipeline/InjectionRecovery/assets.py similarity index 100% rename from jim_dagster/InjectionRecovery/assets.py rename to pipeline/InjectionRecovery/assets.py diff --git a/jim_dagster/InjectionRecovery/definitions.py b/pipeline/InjectionRecovery/definitions.py similarity index 100% rename from jim_dagster/InjectionRecovery/definitions.py rename to pipeline/InjectionRecovery/definitions.py diff --git a/jim_dagster/RealDataCatalog/__init__.py b/pipeline/RealDataCatalog/__init__.py similarity index 100% rename from jim_dagster/RealDataCatalog/__init__.py rename to pipeline/RealDataCatalog/__init__.py diff --git a/jim_dagster/RealDataCatalog/assets.py b/pipeline/RealDataCatalog/assets.py similarity index 100% rename from jim_dagster/RealDataCatalog/assets.py rename to pipeline/RealDataCatalog/assets.py diff --git a/jim_dagster/RealDataCatalog/definitions.py b/pipeline/RealDataCatalog/definitions.py similarity index 100% rename from jim_dagster/RealDataCatalog/definitions.py rename to pipeline/RealDataCatalog/definitions.py diff --git a/jim_dagster/SingleRun/__init__.py b/pipeline/__init__.py similarity index 100% rename from jim_dagster/SingleRun/__init__.py rename to pipeline/__init__.py