ci_sim.yml

# Copyright (c) 2026 Huawei Technologies Co., Ltd.
# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
# CANN Open Software License Agreement Version 2.0 (the "License").
# Please refer to the License for details. You may not use this file except in compliance with the License.
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
# See LICENSE in the root of the software repository for the full text of the License.

name: ci-sim

on:
  pull_request:
  # Nightly self-hosted simulator validation (GitHub cron is UTC).
  schedule:
    - cron: "0 14 * * *"
  workflow_dispatch:

permissions:
  contents: read
  pull-requests: read

jobs:
  pr-context:
    if: ${{ github.event_name == 'pull_request' && github.run_attempt == 1 }}
    runs-on: ubuntu-22.04
    steps:
      - name: Write PR context
        env:
          PR_NUMBER: ${{ github.event.pull_request.number }}
          PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        shell: bash
        run: |
          set -euo pipefail
          mkdir -p pr-context
          printf '%s\n' "${PR_NUMBER}" > pr-context/pr-number
          printf '%s\n' "${PR_HEAD_SHA}" > pr-context/pr-head-sha

      - name: Upload PR context
        uses: actions/upload-artifact@v4
        with:
          name: pr-context
          path: pr-context
          if-no-files-found: error

  detect-vpto-sim-changes:
    runs-on: ubuntu-22.04
    outputs:
      should_run: ${{ steps.decide.outputs.should_run }}
    steps:
      - name: Detect PR changes relevant to VPTO SIM validation
        if: ${{ github.event_name == 'pull_request' }}
        id: filter
        uses: dorny/paths-filter@v3
        with:
          token: ${{ github.token }}
          filters: |
            relevant:
              - '.github/workflows/ci_sim.yml'
              - 'scripts/sim_dsl.sh'
              - 'test/vpto/**'
              - 'test/tilelang_st/**'
              - 'test/dsl-st/**'
              - 'tilelang-dsl/**'
              - 'ptodsl/**'
              - 'tools/ptoas/**'
              - 'lib/TileOps/**'
              - 'include/PTO/IR/VPTO*.td'
              - 'lib/PTO/IR/VPTO.cpp'
              - 'include/PTO/Transforms/Passes.h'
              - 'include/PTO/Transforms/Passes.td'
              - 'include/PTO/Transforms/*VPTO*'
              - 'include/PTO/Transforms/**/*VPTO*'
              - 'lib/PTO/Transforms/CMakeLists.txt'
              - 'lib/PTO/Transforms/ExpandTileOp.cpp'
              - 'lib/PTO/Transforms/FoldTileBufIntrinsics.cpp'
              - 'lib/PTO/Transforms/*VPTO*'
              - 'lib/PTO/Transforms/**/*VPTO*'

      - name: Decide whether to run VPTO SIM validation
        id: decide
        shell: bash
        env:
          EVENT_NAME: ${{ github.event_name }}
          PR_RELEVANT: ${{ steps.filter.outputs.relevant || 'false' }}
        run: |
          set -euo pipefail
          if [[ "${EVENT_NAME}" != "pull_request" ]]; then
            echo "should_run=true" >> "${GITHUB_OUTPUT}"
            echo "Running vpto-sim-validation on ${EVENT_NAME}"
          elif [[ "${PR_RELEVANT}" == "true" ]]; then
            echo "should_run=true" >> "${GITHUB_OUTPUT}"
            echo "Running vpto-sim-validation because the PR touched VPTO-related paths"
          else
            echo "should_run=false" >> "${GITHUB_OUTPUT}"
            echo "Skipping vpto-sim-validation because the PR did not touch VPTO-related paths"
          fi

  vpto-sim-validation:
    needs: detect-vpto-sim-changes
    runs-on: [self-hosted, Linux, X64, label-1]
    timeout-minutes: 120
    concurrency:
      group: vpto-sim-${{ github.event.pull_request.number || github.ref }}
      cancel-in-progress: true
    if: >-
      ${{
        needs.detect-vpto-sim-changes.outputs.should_run == 'true'
      }}
    env:
      LLVM_REPO: https://github.com/vpto-dev/llvm-project.git
      LLVM_REF: feature-vpto
      PTO_INSTALL_DIR: ${{ github.workspace }}/install
      VPTO_SIM_WORKSPACE: ${{ github.workspace }}/.work/vpto-sim-ci
      TILELANG_DSL_WORKSPACE: ${{ github.workspace }}/.work/tilelang-dsl-ci
      TILELANG_DSL_UT_WORKSPACE: ${{ github.workspace }}/.work/tilelang-dsl-ut-ci
      PYPTO_REF: main
      PYPTO_WORKSPACE: ${{ github.workspace }}/.work/pypto-ci
      PYPTO_RUN_WORKSPACE: ${{ github.workspace }}/.work/pypto-run-ci
      PTO_ISA_COMMIT: 016396b57e2c17093f1194e6acd89bb112b0ab24
      PTO_ISA_ROOT: ${{ github.workspace }}/.work/pto-isa-ci
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          persist-credentials: false

      - name: Resolve LLVM directories
        shell: bash
        run: |
          set -euo pipefail
          echo "LLVM_ROOT=${RUNNER_TOOL_CACHE}/llvm-project" >> "${GITHUB_ENV}"
          echo "LLVM_DIR=${RUNNER_TOOL_CACHE}/llvm-project/llvm/build-assert" >> "${GITHUB_ENV}"
          echo "MLIR_PYTHONPATH=${RUNNER_TOOL_CACHE}/llvm-project/llvm/build-assert/tools/mlir/python_packages/mlir_core" >> "${GITHUB_ENV}"

      - name: Resolve LLVM cache key
        id: llvm-cache-key
        shell: bash
        run: |
          set -euo pipefail
          # Resolve to a Git object that ls-remote can handle: either a tag
          # (LLVM_TAG) or a branch head (LLVM_REF).  Only one is expected.
          ref="${LLVM_TAG:-${LLVM_REF}}"
          sha="$(git ls-remote "${LLVM_REPO}" "${ref}" | awk '{print $1}')"
          if [[ -z "${sha}" ]]; then
            echo "ERROR: failed to resolve ${LLVM_REPO} ${ref}" >&2
            exit 1
          fi
          echo "sha=${sha}" >> "${GITHUB_OUTPUT}"
          echo "key=llvm-build-${sha}-assert-v1" >> "${GITHUB_OUTPUT}"

      - name: Restore LLVM build cache
        id: llvm-cache
        continue-on-error: true
        uses: actions/cache/restore@v4
        with:
          path: ${{ env.LLVM_DIR }}
          key: ${{ steps.llvm-cache-key.outputs.key }}

      - name: Ensure runner dependencies
        shell: bash
        run: |
          set -euo pipefail
          missing_tools=()
          for tool in python3 git cmake ninja make; do
            if ! command -v "${tool}" >/dev/null 2>&1; then
              missing_tools+=("${tool}")
            fi
          done

          if [[ "${#missing_tools[@]}" -gt 0 ]]; then
            if command -v sudo >/dev/null 2>&1 && command -v apt-get >/dev/null 2>&1; then
              sudo apt-get update
              sudo apt-get install -y python3 python3-pip git cmake ninja-build make
            else
              echo "ERROR: missing required tools on self-hosted runner: ${missing_tools[*]}" >&2
              echo "ERROR: automatic installation requires sudo + apt-get" >&2
              exit 1
            fi
          fi

          python3 -m pip --version >/dev/null 2>&1 || {
            if command -v sudo >/dev/null 2>&1 && command -v apt-get >/dev/null 2>&1; then
              sudo apt-get update
              sudo apt-get install -y python3-pip
            else
              echo "ERROR: python3-pip is required on self-hosted runner" >&2
              exit 1
            fi
          }

          need_pip_install=0
          python3 -c "import numpy" >/dev/null 2>&1 || need_pip_install=1
          python3 -c "import setuptools, wheel" >/dev/null 2>&1 || need_pip_install=1
          python3 -m pybind11 --cmakedir >/dev/null 2>&1 || need_pip_install=1
          python3 -c "import ml_dtypes" >/dev/null 2>&1 || need_pip_install=1

          if [[ "${need_pip_install}" -eq 1 ]]; then
            python3 -m pip install --upgrade pip
            python3 -m pip install setuptools wheel 'pybind11<3' numpy ml-dtypes
          fi

      - name: Clean CI work dirs
        shell: bash
        run: |
          set -euo pipefail
          rm -rf "${GITHUB_WORKSPACE}/build"
          rm -rf "${PTO_INSTALL_DIR}"
          rm -rf "${VPTO_SIM_WORKSPACE}"
          rm -rf "${TILELANG_DSL_WORKSPACE}"
          rm -rf "${PYPTO_WORKSPACE}"
          rm -rf "${PYPTO_RUN_WORKSPACE}"
          rm -rf "${PTO_ISA_ROOT}"

      - name: Prepare LLVM source
        shell: bash
        run: |
          set -euo pipefail
          mkdir -p "${LLVM_ROOT}"
          cd "${LLVM_ROOT}"

          if [ ! -d .git ]; then
            git init
            git remote add origin "${LLVM_REPO}"
          fi

          git remote set-url origin "${LLVM_REPO}"
          ref="${LLVM_TAG:-${LLVM_REF}}"
          git fetch --depth 1 origin "${ref}"
          git checkout --force FETCH_HEAD

      - name: Build LLVM/MLIR
        if: steps.llvm-cache.outputs.cache-hit != 'true'
        shell: bash
        run: |
          set -euo pipefail
          cd "${LLVM_ROOT}"
          # Pin system compilers.  Self-hosted runners may have conda GCC on
          # PATH; letting CMake autodetect it produces build errors or ABI
          # mismatches with system libraries.
          export CC=gcc
          export CXX=g++
          # Clean the build directory so that stale generated files from a
          # previous run (e.g. _smt_ops_gen.py left behind when the ref
          # changed) do not leak into the fresh build.
          rm -rf llvm/build-assert
          cmake -G Ninja -S llvm -B llvm/build-assert \
            -DLLVM_ENABLE_PROJECTS="mlir;clang" \
            -DBUILD_SHARED_LIBS=ON \
            -DLLVM_ENABLE_ASSERTIONS=ON \
            -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
            -DPython3_EXECUTABLE=python3 \
            -DCMAKE_BUILD_TYPE=Release \
            -DLLVM_TARGETS_TO_BUILD="host"

          ninja -C llvm/build-assert

      - name: Save LLVM build cache
        if: steps.llvm-cache.outputs.cache-hit != 'true'
        continue-on-error: true
        uses: actions/cache/save@v4
        with:
          path: ${{ env.LLVM_DIR }}
          key: ${{ steps.llvm-cache-key.outputs.key }}

      - name: Build PTOAS
        shell: bash
        run: |
          set -euo pipefail
          rm -rf "${PTO_INSTALL_DIR}"
          # Pin system compilers for the same reason as LLVM above.
          export CC=gcc
          export CXX=g++
          # LLVM_BUILD_DIR is the env var read by the build backend (_ptoas_build_backend.py).
          LLVM_BUILD_DIR="${LLVM_DIR}" \
            PTO_INSTALL_DIR="${PTO_INSTALL_DIR}" \
            python3 -m pip install . --no-build-isolation --no-deps --ignore-installed --prefix "${PTO_INSTALL_DIR}"

      - name: Resolve simulator environment
        shell: bash
        run: |
          set -euo pipefail

          detect_ascend_home() {
            for d in \
              "${ASCEND_HOME_PATH:-}" \
              /usr/local/Ascend/cann \
              /usr/local/Ascend/cann-* \
              /usr/local/Ascend/ascend-toolkit/latest
            do
              [[ -n "${d}" && -d "${d}" ]] || continue
              printf '%s\n' "${d}"
              return 0
            done
            return 1
          }

          ASCEND_HOME_PATH_DETECTED="$(detect_ascend_home || true)"
          if [[ -z "${ASCEND_HOME_PATH_DETECTED}" ]]; then
            echo "ERROR: failed to detect ASCEND_HOME_PATH on self-hosted runner" >&2
            exit 1
          fi

          echo "ASCEND_HOME_PATH=${ASCEND_HOME_PATH_DETECTED}" >> "${GITHUB_ENV}"
          echo "PTOAS_BIN=${GITHUB_WORKSPACE}/build/tools/ptoas/ptoas" >> "${GITHUB_ENV}"

      - name: Checkout PyPTO
        uses: actions/checkout@v4
        with:
          repository: hw-native-sys/pypto
          ref: ${{ env.PYPTO_REF }}
          path: ${{ env.PYPTO_WORKSPACE }}
          fetch-depth: 1
          persist-credentials: false

      - name: Checkout PTO-ISA
        uses: actions/checkout@v4
        with:
          repository: hw-native-sys/pto-isa
          ref: ${{ env.PTO_ISA_COMMIT }}
          path: ${{ env.PTO_ISA_ROOT }}
          fetch-depth: 1
          persist-credentials: false

      - name: Ensure GCC 15 for PyPTO simulator
        shell: bash
        run: |
          set -euo pipefail

          compiler_major() {
            "$1" -dumpfullversion -dumpversion | cut -d. -f1
          }

          if command -v g++-15 >/dev/null 2>&1; then
            if [[ "$(compiler_major "$(command -v g++-15)")" != "15" ]]; then
              echo "ERROR: g++-15 exists but is not GCC 15: $(g++-15 --version | head -1)" >&2
              exit 1
            fi
            if ! command -v gcc-15 >/dev/null 2>&1; then
              echo "ERROR: g++-15 exists but gcc-15 is missing." >&2
              exit 1
            fi
            exit 0
          fi

          if ! command -v x86_64-conda-linux-gnu-g++ >/dev/null 2>&1 || \
             ! command -v x86_64-conda-linux-gnu-gcc >/dev/null 2>&1 || \
             [[ "$(compiler_major "$(command -v x86_64-conda-linux-gnu-g++)")" != "15" ]]; then
            if ! command -v conda >/dev/null 2>&1; then
              echo "ERROR: PyPTO simulator smoke requires GCC/G++ 15 and conda is unavailable to install it." >&2
              exit 1
            fi
            conda install -y -c conda-forge gcc_linux-64=15 gxx_linux-64=15
            hash -r
          fi

          mkdir -p "${PYPTO_WORKSPACE}/.work/bin"
          ln -sf "$(command -v x86_64-conda-linux-gnu-g++)" "${PYPTO_WORKSPACE}/.work/bin/g++-15"
          ln -sf "$(command -v x86_64-conda-linux-gnu-gcc)" "${PYPTO_WORKSPACE}/.work/bin/gcc-15"
          echo "${PYPTO_WORKSPACE}/.work/bin" >> "${GITHUB_PATH}"

      - name: Prepare PyPTO Python dependencies
        shell: bash
        run: |
          set -euo pipefail

          python3 -m pip install --upgrade pip
          python3 -m pip install --index-url https://download.pytorch.org/whl/cpu torch
          python3 -m pip install scikit-build-core nanobind ninja pytest cloudpickle

      - name: Install PyPTO frontend and runtime
        shell: bash
        env:
          PTO_ISA_ROOT: ${{ env.PTO_ISA_ROOT }}
        run: |
          set -euo pipefail
          rm -rf "${PYPTO_WORKSPACE}/build" "${PYPTO_WORKSPACE}/_skbuild" "${PYPTO_WORKSPACE}/runtime/build"
          python3 -m pip install --no-build-isolation --no-deps "${PYPTO_WORKSPACE}"
          # ci_sim only runs PyPTO simulator tests. Keep the runtime install from
          # auto-detecting onboard platforms from the runner's CANN environment.
          env -u ASCEND_HOME_PATH python3 -m pip install --no-build-isolation --no-deps "${PYPTO_WORKSPACE}/runtime"

      - name: Run PyPTO PTOAS end-to-end simulator smoke
        shell: bash
        env:
          PTOAS_ROOT: ${{ github.workspace }}/build/tools/ptoas
          PTO_ISA_ROOT: ${{ env.PTO_ISA_ROOT }}
        run: |
          set -euo pipefail
          mkdir -p "${PYPTO_RUN_WORKSPACE}"
          cd "${PYPTO_WORKSPACE}"

          run_pypto_pytest() {
            local platform="$1"
            local suite_name="$2"
            local kernels_dir="${PYPTO_RUN_WORKSPACE}/${suite_name}_${platform}"
            local log_file="${PYPTO_RUN_WORKSPACE}/${suite_name}_${platform}.log"
            shift 2

            python3 -m pytest "$@" \
              -v \
              --platform="${platform}" \
              --pto-isa-commit="${PTO_ISA_COMMIT}" \
              --save-kernels \
              --kernels-dir="${kernels_dir}" \
              2>&1 | tee "${log_file}"
          }

          # All suites below compile through PyPTO's PTO backend and run ptoas.
          # They are split into separate pytest processes to avoid backend global-state conflicts.
          run_pypto_core_smoke() {
            local platform="$1"
            local dyn_valid_shape="tests/st/runtime/control_flow/test_dyn_orch_shape.py::TestDynOrchShapeOperations::test_dyn_orch_valid_shape_add[shape0-valid_shape0-${platform}]"
            local cross_core="tests/st/runtime/cross_core/test_cross_core.py::TestCrossCore::test_tpush_tpop_v2c_updown[${platform}]"
            local pypto_smoke_tests=(
              tests/st/examples/00_hello_world/test_hello_world.py
              tests/st/examples/02_intermediate/test_softmax.py::TestTileSoftmax::test_tile_softmax
              tests/st/examples/02_intermediate/test_rms_norm.py::TestRMSNormCore::test_rms_norm_core
              tests/st/runtime/ops/test_elementwise.py
              tests/st/runtime/ops/test_assemble.py
              tests/st/runtime/framework_and_models/test_jit.py::TestJITExecution::test_cache_hit_reuses_compiled_program
              tests/st/runtime/framework_and_models/test_jit.py::TestJITDynamicBatch::test_one_artifact_serves_multiple_batches
              tests/st/runtime/framework_and_models/test_compiled_program.py::TestManualWorkerExtraction::test_block_dim_override_runs
              "${dyn_valid_shape}"
              "${cross_core}"
            )
            run_pypto_pytest "${platform}" core_ptoas "${pypto_smoke_tests[@]}"
          }

          run_pypto_fa_ptoas_smoke() {
            local platform="$1"
            local pypto_fa_tests=(
              tests/st/runtime/ops/test_cast.py::TestCast::test_tile_cast_col_major_narrow
              tests/st/runtime/framework_and_models/test_paged_attention.py::TestPagedAttentionKernels::test_qk_matmul_ptoas[16-128-128]
              tests/st/runtime/framework_and_models/test_paged_attention.py::TestPagedAttentionKernels::test_softmax_prepare_ptoas[16-128]
              tests/st/runtime/framework_and_models/test_paged_attention.py::TestPagedAttentionKernels::test_softmax_prepare_unaligned_ptoas[16-128-100]
              tests/st/runtime/framework_and_models/test_paged_attention.py::TestPagedAttentionKernels::test_pv_matmul_ptoas[16-128-128]
              tests/st/runtime/framework_and_models/test_paged_attention.py::TestPagedAttentionKernels::test_online_update_ptoas[16-128-0-1]
            )
            run_pypto_pytest "${platform}" fa_ptoas "${pypto_fa_tests[@]}"
          }

          run_pypto_int8_codegen_smoke() {
            local platform="$1"
            local pypto_int8_tests=(
              tests/st/codegen/dsl/test_batch_matmul_pipeline.py::test_no_mat_to_mat_tmov
            )
            run_pypto_pytest "${platform}" int8_ptoas_codegen "${pypto_int8_tests[@]}"
          }

          run_pypto_core_smoke a5sim
          run_pypto_fa_ptoas_smoke a5sim
          run_pypto_core_smoke a2a3sim
          run_pypto_fa_ptoas_smoke a2a3sim
          run_pypto_int8_codegen_smoke a2a3sim

      - name: Run VPTO SIM validation
        if: ${{ true }}
        shell: bash
        run: |
          set -euo pipefail
          mkdir -p "${VPTO_SIM_WORKSPACE}"
          WORK_SPACE="${VPTO_SIM_WORKSPACE}" \
          ASCEND_HOME_PATH="${ASCEND_HOME_PATH}" \
          PTOAS_BIN="${PTOAS_BIN}" \
          DEVICE=SIM \
          JOBS="${JOBS:-32}" \
          bash test/vpto/scripts/run_host_vpto_validation_parallel.sh

      - name: Run TileLang DSL CI
        shell: bash
        run: |
          set -euo pipefail
          mkdir -p "${TILELANG_DSL_WORKSPACE}"
          export LLVM_BUILD_DIR="${LLVM_DIR}"
          export MLIR_PYTHON_ROOT="${MLIR_PYTHONPATH}"
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            ASCEND_HOME_PATH="${ASCEND_HOME_PATH}" \
            PTOAS_BIN="${PTOAS_BIN}" \
            bash test/tilelang_st/script/run_ci.sh -r sim -v a5 --jobs 64 --smoke \
              2>&1 | tee "${TILELANG_DSL_WORKSPACE}/run_ci.log"
          else
            ASCEND_HOME_PATH="${ASCEND_HOME_PATH}" \
            PTOAS_BIN="${PTOAS_BIN}" \
            bash test/tilelang_st/script/run_ci.sh -r sim -v a5 --jobs 64 \
              2>&1 | tee "${TILELANG_DSL_WORKSPACE}/run_ci.log"
          fi

      - name: Run PTODSL DSL ST CI
        shell: bash
        run: |
          set -euo pipefail
          mkdir -p "${TILELANG_DSL_WORKSPACE}"
          export LLVM_BUILD_DIR="${LLVM_DIR}"
          export PYTHON_BIN="python3"
          ASCEND_HOME_PATH="${ASCEND_HOME_PATH}" \
          PTOAS_BIN="${PTOAS_BIN}" \
          scripts/sim_dsl.sh test/dsl-st \
            2>&1 | tee "${TILELANG_DSL_WORKSPACE}/ptodsl-dsl-st.log"

      - name: Upload TileLang DSL logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: tilelang-dsl-ci-${{ github.run_id }}
          path: |
            ${{ env.TILELANG_DSL_WORKSPACE }}/run_ci.log
            ${{ env.TILELANG_DSL_WORKSPACE }}/ptodsl-dsl-st.log
          if-no-files-found: warn

      - name: Run TileLang DSL unit tests
        shell: bash
        run: |
          set -euo pipefail
          mkdir -p "${TILELANG_DSL_UT_WORKSPACE}"
          cd tilelang-dsl
          PYTHONPATH=python python3 -m unittest discover -s tests -p 'test_*.py' \
            2>&1 | tee "${TILELANG_DSL_UT_WORKSPACE}/unittest.log"

      - name: Upload TileLang DSL unit test logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: tilelang-dsl-ut-ci-${{ github.run_id }}
          path: |
            ${{ env.TILELANG_DSL_UT_WORKSPACE }}/unittest.log
          if-no-files-found: warn

      - name: Upload VPTO SIM logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: vpto-sim-validation-${{ github.run_id }}
          path: |
            ${{ env.VPTO_SIM_WORKSPACE }}/parallel-runner.log
            ${{ env.VPTO_SIM_WORKSPACE }}/parallel-summary.tsv
          if-no-files-found: warn

      - name: Upload PyPTO SIM logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: pypto-sim-smoke-${{ github.run_id }}
          path: |
            ${{ env.PYPTO_RUN_WORKSPACE }}/*.log
          if-no-files-found: warn