From e2435208f3b1ea1a057d2c9a8d5528e3b4bde2d5 Mon Sep 17 00:00:00 2001 From: Simon Powell Date: Fri, 10 Nov 2023 10:57:55 +0000 Subject: [PATCH] [DEBUGINFRA-724] Migrate to GutHub Actions Migrate to GutHub Actions from Jenkins --- .github/workflows/build_test.yaml | 231 +++++++++++++ .github/workflows/post_weekly_release.yaml | 334 +++++++++++++++++++ .github/workflows/release_body_template.md | 13 + .gitignore | 3 + jenkins/build-image.sh | 51 --- jenkins/build.Dockerfile | 33 -- jenkins/dockerimage.Jenkinsfile | 49 --- jenkins/nightly.Jenkinsfile | 365 -------------------- jenkins/release.Jenkinsfile | 370 --------------------- 9 files changed, 581 insertions(+), 868 deletions(-) create mode 100644 .github/workflows/build_test.yaml create mode 100644 .github/workflows/post_weekly_release.yaml create mode 100644 .github/workflows/release_body_template.md delete mode 100755 jenkins/build-image.sh delete mode 100644 jenkins/build.Dockerfile delete mode 100644 jenkins/dockerimage.Jenkinsfile delete mode 100644 jenkins/nightly.Jenkinsfile delete mode 100644 jenkins/release.Jenkinsfile diff --git a/.github/workflows/build_test.yaml b/.github/workflows/build_test.yaml new file mode 100644 index 000000000..25d925ab0 --- /dev/null +++ b/.github/workflows/build_test.yaml @@ -0,0 +1,231 @@ +name: build-test +run-name: Build and test + +on: + workflow_dispatch: + pull_request: + branches: + - main + +jobs: + build-ubuntu: + name: Build and test on Ubuntu + runs-on: ubuntu-22.04 + steps: + - name: Git checkout + uses: actions/checkout@v4 + with: + submodules: 'true' + + - name: Build R + run: | + export CXX=clang++ + mkdir build_rel + cd build_rel + cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_ISA_NONE=ON -DASTCENC_UNITTEST=ON -DASTCENC_PACKAGE=x64 .. + make install package -j4 + + - name: Build D + run: | + export CXX=clang++ + mkdir build_dbg + cd build_dbg + cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Debug -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_ISA_NONE=ON .. + make -j4 + + - name: Upload binaries + uses: actions/upload-artifact@v4 + with: + name: astcenc-linux-x64 + path: | + build_rel/*.zip + build_rel/*.zip.sha256 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Get Python modules + run: | + python -m pip install --upgrade pip + pip install numpy Pillow + + - name: Python Tests + run: | + python ./Test/astc_test_functional.py --encoder=none + python ./Test/astc_test_functional.py --encoder=sse2 + python ./Test/astc_test_functional.py --encoder=sse4.1 + python ./Test/astc_test_functional.py --encoder=avx2 + python ./Test/astc_test_image.py --encoder=none --test-set Small --test-quality medium + python ./Test/astc_test_image.py --encoder=all-x86 --test-set Small --test-quality medium + + - name: ctest + run: ctest + working-directory: build_rel + + build-macos: + name: Build and test on MacOS + runs-on: macos-12 + steps: + - name: Git checkout + uses: actions/checkout@v4 + with: + submodules: 'true' + + - name: Build R + run: | + mkdir build_rel + cd build_rel + cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_UNIVERSAL_BUILD=OFF -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_PACKAGE=x64 .. + make install package -j4 + + - name: Build D + run: | + mkdir build_dbg + cd build_dbg + cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Debug -DASTCENC_UNIVERSAL_BUILD=OFF -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_ISA_NONE=ON .. + make -j4 + + - name: Upload binaries + uses: actions/upload-artifact@v4 + with: + name: astcenc-macos-x64 + path: | + build_rel/*.zip + build_rel/*.zip.sha256 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Get Python modules + run: | + python -m pip install --upgrade pip + pip install numpy Pillow + + - name: Python Tests + run: | + python ./Test/astc_test_image.py --test-set Small --test-quality medium + + build-windows-msvc: + name: Build and test on Windows MSVC + runs-on: windows-2022 + steps: + - name: Git checkout + uses: actions/checkout@v4 + with: + submodules: 'true' + + - name: Setup Visual Studio x86_6 + uses: ilammy/msvc-dev-cmd@v1 + - name: Build R + run: | + mkdir build_rel + cd build_rel + cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_PACKAGE=x64-cl .. + nmake install package + shell: cmd + - name: Build D + run: | + mkdir build_dbg + cd build_dbg + cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Debug -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_ISA_NONE=ON .. + nmake + shell: cmd + + - name: Upload binaries + uses: actions/upload-artifact@v4 + with: + name: astcenc-windows-x64-cl + path: build_rel/*.zip + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Get Python modules + run: | + python -m pip install --upgrade pip + pip install numpy Pillow + shell: cmd + + - name: Python Tests + run: | + python ./Test/astc_test_image.py --test-set Small --test-quality medium + shell: cmd + + build-windows-ClangCL: + name: Build and test on Windows ClangCL + runs-on: windows-2022 + steps: + - name: Git checkout + uses: actions/checkout@v4 + with: + submodules: 'true' + + - name: Setup Visual Studio x86_6 + uses: ilammy/msvc-dev-cmd@v1 + - name: Build R + run: | + mkdir build_rel + cd build_rel + cmake -G "Visual Studio 17 2022" -T ClangCL -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_PACKAGE=x64-clangcl .. + msbuild astcencoder.sln -property:Configuration=Release + msbuild PACKAGE.vcxproj -property:Configuration=Release + msbuild INSTALL.vcxproj -property:Configuration=Release + shell: cmd + - name: Build D + run: | + mkdir build_dbg + cd build_dbg + cmake -G "Visual Studio 17 2022" -T ClangCL -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON .. + msbuild astcencoder.sln -property:Configuration=Debug + shell: cmd + + - name: Setup Visual Studio arm64 + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x86_arm64 + - name: Build R + run: | + mkdir build_rel_arm64 + cd build_rel_arm64 + cmake -G "Visual Studio 17 2022" -A ARM64 -T ClangCL -DASTCENC_ISA_NEON=ON -DASTCENC_PACKAGE=arm64-clangcl .. + msbuild astcencoder.sln -property:Configuration=Release + msbuild PACKAGE.vcxproj -property:Configuration=Release + msbuild INSTALL.vcxproj -property:Configuration=Release + shell: cmd + - name: Build D + run: | + mkdir build_dbg_arm64 + cd build_dbg_arm64 + cmake -G "Visual Studio 17 2022" -A ARM64 -T ClangCL -DASTCENC_ISA_NEON=ON .. + msbuild astcencoder.sln -property:Configuration=Debug + shell: cmd + + - name: Upload binaries + uses: actions/upload-artifact@v4 + with: + name: astcenc-windows-multi-clangcl + path: | + build_rel/*.zip + build_rel_arm64/*.zip + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Get Python modules + run: | + python -m pip install --upgrade pip + pip install numpy Pillow + shell: cmd + + - name: Python Tests + run: | + python ./Test/astc_test_image.py --test-set Small --test-quality medium + shell: cmd diff --git a/.github/workflows/post_weekly_release.yaml b/.github/workflows/post_weekly_release.yaml new file mode 100644 index 000000000..77de2acac --- /dev/null +++ b/.github/workflows/post_weekly_release.yaml @@ -0,0 +1,334 @@ +name: post-weekly-release +run-name: Build, test, generate signed artefacts and optionally prepare release + +on: + workflow_dispatch: + push: + branches: + - main + tags: + - '*' + schedule: + - cron: '17 2 * * 1' + +jobs: + + coverity: + if: ${{ !startsWith(github.event.ref, 'refs/tags/') }} + name: Run Coverity Static Analysis + runs-on: [self-hosted-ubuntu-latest-x64] + steps: + - name: Clean workspace + uses: AutoModality/action-clean@v1 + - name: Git checkout + uses: actions/checkout@v4 + with: + submodules: 'true' + + - name: Coverity Preparation + run: | + export PATH=$PATH:/usr/local/cov-analysis/bin + mkdir build_cov + cd build_cov + cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX2=ON .. + cov-configure --config ${GITHUB_WORKSPACE}/coverity.conf --template --compiler cc --comptype gcc + cov-configure --config ${GITHUB_WORKSPACE}/coverity.conf --template --compiler c++ --comptype g++ + + - name: Coverity Build + run: | + export PATH=$PATH:/usr/local/cov-analysis/bin + cd build_cov + cov-build --config ${GITHUB_WORKSPACE}/coverity.conf --dir ${GITHUB_WORKSPACE}/intermediate make install + + - name: Coverity Analyse + run: | + export PATH=$PATH:/usr/local/cov-analysis/bin + cd build_cov + cov-analyze --dir ${GITHUB_WORKSPACE}/intermediate + + - name: Coverity Upload + env: + COVERITY_KEY: ${{ secrets.COVERITY_KEY }} + run: | + export PATH=$PATH:/usr/local/cov-analysis/bin + echo "${COVERITY_KEY}" > coverity.key + chmod 400 coverity.key + cd build_cov + cov-commit-defects \ + --dir ${GITHUB_WORKSPACE}/intermediate \ + --stream astcenc-master \ + --url https://coverity.cambridge.arm.com \ + --auth-key-file ../coverity.key \ + --strip-path ${GITHUB_WORKSPACE} + + build-ubuntu: + name: Build and test on Ubuntu + runs-on: ubuntu-22.04 + steps: + - name: Git checkout + uses: actions/checkout@v4 + with: + submodules: 'true' + + - name: Build R + run: | + export CXX=clang++ + mkdir build_rel + cd build_rel + cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_PACKAGE=x64 .. + make install package -j4 + + - name: Build D + run: | + export CXX=clang++ + mkdir build_dbg + cd build_dbg + cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_DECOMPRESSOR=ON .. + make -j4 + + - name: Upload binaries + uses: actions/upload-artifact@v4 + with: + name: astcenc-linux-x86_64 + path: | + build_rel/*.zip + build_rel/*.zip.sha256 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Get Python modules + run: | + python -m pip install --upgrade pip + pip install numpy Pillow + + - name: Python Tests + run: | + python ./Test/astc_test_functional.py + python ./Test/astc_test_image.py --encoder=all-x86 --test-set Small + + build-macos: + name: Build and test on MacOS + runs-on: macos-12 + steps: + - name: Git checkout + uses: actions/checkout@v4 + with: + submodules: 'true' + + - name: Build R + run: | + mkdir build_rel + cd build_rel + cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_PACKAGE=universal .. + make install package -j4 + + - name: Upload binaries + uses: actions/upload-artifact@v4 + with: + name: astcenc-macos-universal + path: | + build_rel/*.zip + build_rel/*.zip.sha256 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Get Python modules + run: | + python -m pip install --upgrade pip + pip install numpy Pillow + + - name: Python Tests + run: | + python ./Test/astc_test_image.py --test-set Small --encoder universal + + build-windows-msvc-ClangCL: + name: Build and test on Windows MSVC + runs-on: windows-2022 + steps: + - name: Git checkout + uses: actions/checkout@v4 + with: + submodules: 'true' + + - name: Setup Visual Studio x86_64 + uses: ilammy/msvc-dev-cmd@v1 + - name: Build R x86_64 + run: | + mkdir build_rel + cd build_rel + cmake -G "Visual Studio 17 2022" -T ClangCL -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_PACKAGE=x64 .. + msbuild astcencoder.sln -property:Configuration=Release + msbuild PACKAGE.vcxproj -property:Configuration=Release + msbuild INSTALL.vcxproj -property:Configuration=Release + shell: cmd + + - name: Setup Visual Studio arm64 + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x86_arm64 + - name: Build R arm64 + run: | + mkdir build_rel_arm64 + cd build_rel_arm64 + cmake -G "Visual Studio 17 2022" -A ARM64 -T ClangCL -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_NEON=ON -DASTCENC_PACKAGE=arm64 .. + msbuild astcencoder.sln -property:Configuration=Release + msbuild PACKAGE.vcxproj -property:Configuration=Release + msbuild INSTALL.vcxproj -property:Configuration=Release + shell: cmd + + - name: Upload binaries + uses: actions/upload-artifact@v4 + with: + name: astcenc-windows-multi-cl + path: | + build_rel/*.zip + build_rel/*.zip.sha256 + build_rel_arm64/*.zip + build_rel_arm64/*.zip.sha256 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Get Python modules + run: | + python -m pip install --upgrade pip + pip install numpy Pillow + shell: cmd + + - name: Python Tests + run: | + python ./Test/astc_test_image.py --test-set Small + shell: cmd + + sign-binaries: + name: Sign Windows and Mac + runs-on: [self-hosted-ubuntu-latest-x64] + needs: [build-macos, build-windows-msvc-ClangCL] + steps: + - name: Clean workspace + uses: AutoModality/action-clean@v1 + - name: Checkout signing code + env: + SIGNING_REPO_URL: ${{ secrets.SIGNING_REPO_URL }} + run: | + git clone --depth 1 ${SIGNING_REPO_URL} + + - name: Install code sign v2 client + env: + ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }} + ARTIFACTORY_APIKEY: ${{ secrets.ARTIFACTORY_APIKEY }} + ARTIFACTORY_FQDN: ${{ secrets.ARTIFACTORY_FQDN }} + run: | + python3.11 -m venv cs + . ./cs/bin/activate + pip install -i https://${ARTIFACTORY_USER}:${ARTIFACTORY_APIKEY}@${ARTIFACTORY_FQDN}/artifactory/api/pypi/dsgcore.pypi/simple code-signer-client + + - name: Download Windows binaries + uses: actions/download-artifact@v4 + with: + name: astcenc-windows-multi-cl + path: windows + + - name: Download Mac binaries + uses: actions/download-artifact@v4 + with: + name: astcenc-macos-universal + path: mac + + - name: Sign and notarize Mac + env: + CODESIGNER_USER: ${{ secrets.CODESIGNER_USER }} + run: | + . ./cs/bin/activate + cd mac + python3 ${GITHUB_WORKSPACE}/signing/macos-client-wrapper.py ${CODESIGNER_USER} *.zip + + - name: Sign Windows binaries + env: + ARTIFACTORY_APIKEY: ${{ secrets.ARTIFACTORY_APIKEY }} + run: | + . ./cs/bin/activate + cd windows + for FILENAME in */*; do mv ${FILENAME} .; done + for ZIPFILE in *.zip; do python3 ../signing/windows-client-wrapper.py -b ${GITHUB_RUN_NUMBER} -t ${ARTIFACTORY_APIKEY} ${ZIPFILE}; done + + - name: Upload signed binaries + uses: actions/upload-artifact@v4 + with: + name: signed-binaries + path: | + windows/* + mac/* + + - name: Tidy intermediate artefacts + uses: geekyeggo/delete-artifact@v5 + with: + name: | + astcenc-windows-multi-cl + astcenc-macos-universal + + prepare-release: + if: ${{ startsWith(github.event.ref, 'refs/tags/') }} + name: Prepare release + runs-on: ubuntu-22.04 + needs: [sign-binaries, build-ubuntu] + steps: + - name: Git checkout + uses: actions/checkout@v4 + + - name: Download signed binaries + uses: actions/download-artifact@v4 + with: + name: signed-binaries + path: prepare-release + - name: Download Linux binaries + uses: actions/download-artifact@v4 + with: + name: astcenc-linux-x86_64 + path: prepare-release + + - name: Flatten file structure + run: | + cd prepare-release + for FILENAME in */*; do mv ${FILENAME} .; done + rmdir windows + rmdir mac + + - name: Create checksum file + run: | + cd prepare-release + cat *.sha256 > release-sha256.txt + rm *.sha256 + + - name: Create release body + run: | + export STATUS_DATE=$(date "+%B %Y") + GITHUB_REF=${{ github.ref }} ; export RELEASE_VERSION=${GITHUB_REF##*/} + export SHA_CHECKSUMS=$(cat prepare-release/release-sha256.txt) + envsubst < .github/workflows/release_body_template.md > prepare-release/release_body.md + + - name: Create release + id: create_release + uses: comnoco/create-release-action@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: ${{ github.ref }} + body_path: prepare-release/release_body.md + draft: true + + - name: Attach artefacts + uses: AButler/upload-release-assets@v3.0 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + release-id: ${{ steps.create_release.outputs.id }} + files: "prepare-release/astcenc-*-*-*.zip;prepare-release/release-sha256.txt" diff --git a/.github/workflows/release_body_template.md b/.github/workflows/release_body_template.md new file mode 100644 index 000000000..0f81edce3 --- /dev/null +++ b/.github/workflows/release_body_template.md @@ -0,0 +1,13 @@ +**Status:** ${STATUS_DATE} + +The ${RELEASE_VERSION} release is a minor/major maintenance release. + +* **General:** + * **Bug fix:** Text here + * **Feature:** Text here + +## Binary release sha256 checksums + +``` +${SHA_CHECKSUMS} +``` diff --git a/.gitignore b/.gitignore index 70a806670..ee2ba210a 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,6 @@ TestOutput astc_reference-main* Docs/Profiling.md Source/astcenccli_version.h + +# Do not ignore workflows +!.github/workflows/ diff --git a/jenkins/build-image.sh b/jenkins/build-image.sh deleted file mode 100755 index 5c0875809..000000000 --- a/jenkins/build-image.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash -ARTIFACTORY_URL=eu-west-1.artifactory.aws.arm.com -DOCKER_REGISTRY=mobile-studio--docker.${ARTIFACTORY_URL} -IMAGE_NAME=astcenc -IMAGE_VERSION=3.2.0 - -# Check Artifactory credentials are set -if [[ -z "${ARTIFACTORY_CREDENTIALS}" ]] -then - echo "ARTIFACTORY_CREDENTIALS not set" - echo "e.g." - echo " export ARTIFACTORY_CREDENTIALS=my.name@arm.com:API-KEY" - exit 1 -fi - -echo "Preparation" -rm -fr tmp -mkdir -p tmp - -echo "Get static analysis tools" -curl -L --user ${ARTIFACTORY_CREDENTIALS} https://${ARTIFACTORY_URL}/artifactory/mobile-studio.tools/coverity/cov-analysis-linux64-2023.3.0.sh --output tmp/coverity_install.sh -curl -L --user ${ARTIFACTORY_CREDENTIALS} https://${ARTIFACTORY_URL}/artifactory/mobile-studio.tools/coverity/license.dat --output tmp/coverity_license.dat - -echo "Building image" -docker build --progress=plain \ - -f jenkins/build.Dockerfile \ - -t $IMAGE_NAME:latest \ - -t $IMAGE_NAME:$IMAGE_VERSION \ - -t $DOCKER_REGISTRY/$IMAGE_NAME:latest \ - -t $DOCKER_REGISTRY/$IMAGE_NAME:$IMAGE_VERSION \ - tmp/ - -echo "Clean up temp files" -rm -rf tmp - -if [ "${1}" = "push" ] -then - echo "Pushing to $DOCKER_REGISTRY" - docker login -u ${ARTIFACTORY_CREDENTIALS%:*} -p ${ARTIFACTORY_CREDENTIALS#*:} $DOCKER_REGISTRY - docker push $DOCKER_REGISTRY/$IMAGE_NAME:$IMAGE_VERSION - # docker push $DOCKER_REGISTRY/$IMAGE_NAME:latest - echo "Clean up images" - docker rmi $IMAGE_NAME:latest $IMAGE_NAME:$IMAGE_VERSION $DOCKER_REGISTRY/$IMAGE_NAME:latest $DOCKER_REGISTRY/$IMAGE_NAME:$IMAGE_VERSION -else - echo "Build complete. To manually push to registry, run:" - echo " docker login -u ${ARTIFACTORY_CREDENTIALS%:*} -p ${ARTIFACTORY_CREDENTIALS#*:} $DOCKER_REGISTRY" - echo " docker push \"$DOCKER_REGISTRY/$IMAGE_NAME:$IMAGE_VERSION\"" - # echo " docker push \"$DOCKER_REGISTRY/$IMAGE_NAME:latest\"" -fi - -echo "Script Completed" diff --git a/jenkins/build.Dockerfile b/jenkins/build.Dockerfile deleted file mode 100644 index 64793ae19..000000000 --- a/jenkins/build.Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM ubuntu:22.04 - -RUN useradd -u 1000 -U -m -c Jenkins jenkins - -RUN apt update && apt -y upgrade \ - && apt install -y \ - software-properties-common \ - clang \ - gcc \ - g++ \ - git \ - cmake \ - imagemagick \ - make \ - python3 \ - python3-pip \ - python3-venv \ - python3-numpy \ - python3-pil \ - ca-certificates \ - gnupg \ - wget \ - && rm -rf /var/lib/apt/lists/* - -# Install python modules -RUN pip3 install requests - -# Install Coverity static analysis tools -COPY coverity_* /tmp/ -RUN chmod 555 /tmp/coverity_install.sh && \ - /tmp/coverity_install.sh -q --license.region=6 --license.agreement=agree --license.cov.path=/tmp/coverity_license.dat -dir /usr/local/cov-analysis && \ - rm /tmp/coverity_* -ENV PATH="/usr/local/cov-analysis/bin:$PATH" diff --git a/jenkins/dockerimage.Jenkinsfile b/jenkins/dockerimage.Jenkinsfile deleted file mode 100644 index e46addbbd..000000000 --- a/jenkins/dockerimage.Jenkinsfile +++ /dev/null @@ -1,49 +0,0 @@ -pipeline { - agent { - kubernetes { - yaml ''' -apiVersion: v1 -kind: Pod -spec: - imagePullSecrets: - - name: artifactory-ms-docker - containers: - - name: dind - image: mobile-studio--docker.eu-west-1.artifactory.aws.arm.com/docker:dind - tty: true - resources: - requests: - cpu: 2 - memory: 4Gi - securityContext: - privileged: true - volumeMounts: - - name: dind-storage - mountPath: /var/lib/docker - volumes: - - name: dind-storage - emptyDir: {} -''' - defaultContainer 'dind' - } - } - environment { - ARTIFACTORY_CREDENTIALS = credentials('cepe-artifactory-jenkins') - } - options { - ansiColor('xterm') - timestamps() - } - stages { - stage('Build and Push Image') { - steps { - sh 'docker info' - sh ''' - apk add --no-cache bash curl - chmod u+x ./jenkins/build-image.sh - ./jenkins/build-image.sh push - ''' - } - } - } -} diff --git a/jenkins/nightly.Jenkinsfile b/jenkins/nightly.Jenkinsfile deleted file mode 100644 index 148be6c94..000000000 --- a/jenkins/nightly.Jenkinsfile +++ /dev/null @@ -1,365 +0,0 @@ -/* This pipeline is used for post-commit testing, so it runs frequently. - * - * Test objectives for this pipeline are: - * - * - Run the entire pipeline in less than 10 minutes. - * - Test builds on all supported operating systems. - * - Test builds on all supported compilers. - * - Test release and debug build variants. - * - Run functional smoke tests. - * - Run image quality smoke tests. - * - * The test matrix is not fully covered; e.g. we can assume compilers behave - * similarly on different operating systems, so we test one compiler per OS. - */ - -@Library('hive-infra-library@changes/86/295486/1') _ - -pipeline { - agent none - - options { - ansiColor('xterm') - timestamps() - } - - stages { - stage('Build All') { - parallel { - /* Build for Linux on x86-64 using GCC */ - stage('Linux') { - agent { - kubernetes { - yaml '''\ - apiVersion: v1 - kind: Pod - spec: - securityContext: - runAsUser: 1000 - runAsGroup: 1000 - imagePullSecrets: - - name: artifactory-ms-docker - containers: - - name: astcenc - image: mobile-studio--docker.eu-west-1.artifactory.aws.arm.com/astcenc:3.2.0 - command: - - sleep - args: - - infinity - resources: - requests: - cpu: 4 - memory: 8Gi - limits: - cpu: 8 - memory: 16Gi - '''.stripIndent() - defaultContainer 'astcenc' - } - } - stages { - stage('Clean') { - steps { - sh ''' - git clean -ffdx - git submodule init - git submodule update - ''' - } - } - stage('Build R x64') { - steps { - sh ''' - mkdir build_rel - cd build_rel - cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_ISA_NONE=ON -DASTCENC_UNITTEST=ON -DASTCENC_PACKAGE=x64 .. - make install package -j4 - ''' - } - } - stage('Build D x64') { - steps { - sh ''' - mkdir build_dbg - cd build_dbg - cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Debug -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_ISA_NONE=ON .. - make -j4 - ''' - } - } - stage('Stash') { - steps { - dir('build_rel') { - stash name: 'astcenc-linux-x64', includes: '*.zip' - } - } - } - stage('Test') { - steps { - sh ''' - python3 ./Test/astc_test_functional.py --encoder=none - python3 ./Test/astc_test_functional.py --encoder=sse2 - python3 ./Test/astc_test_functional.py --encoder=sse4.1 - python3 ./Test/astc_test_functional.py --encoder=avx2 - python3 ./Test/astc_test_image.py --encoder=none --test-set Small --test-quality medium - python3 ./Test/astc_test_image.py --encoder=all-x86 --test-set Small --test-quality medium - ''' - dir('build_rel') { - sh 'ctest' - } - } - } - } - } - /* Build for Windows on x86-64 using MSVC */ - stage('Windows MSVC') { - agent { - label 'Windows' - } - stages { - stage('Clean') { - steps { - bat 'git clean -ffdx' - } - } - stage('Build R x64') { - steps { - bat ''' - call c:\\progra~2\\micros~1\\2022\\buildtools\\vc\\auxiliary\\build\\vcvars64.bat - mkdir build_rel - cd build_rel - cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_PACKAGE=x64-cl .. - nmake install package - ''' - } - } - stage('Build D x64') { - steps { - bat ''' - call c:\\progra~2\\micros~1\\2022\\buildtools\\vc\\auxiliary\\build\\vcvars64.bat - mkdir build_dbg - cd build_dbg - cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Debug -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_ISA_NONE=ON .. - nmake - ''' - } - } - stage('Stash') { - steps { - dir('build_rel') { - stash name: 'astcenc-windows-x64-cl', includes: '*.zip' - } - } - } - stage('Test') { - steps { - bat ''' - set Path=c:\\Python3;c:\\Python3\\Scripts;%Path% - call python ./Test/astc_test_image.py --test-set Small --test-quality medium - ''' - } - } - } - } - /* Build for Windows on x86-64 using MSVC + ClangCL */ - stage('Windows ClangCL') { - agent { - label 'Windows' - } - stages { - stage('Clean') { - steps { - bat 'git clean -ffdx' - } - } - stage('Build R x64') { - steps { - bat ''' - call c:\\progra~2\\micros~1\\2022\\buildtools\\vc\\auxiliary\\build\\vcvars64.bat - mkdir build_rel - cd build_rel - cmake -G "Visual Studio 17 2022" -T ClangCL -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_PACKAGE=x64-clangcl .. - msbuild astcencoder.sln -property:Configuration=Release - msbuild PACKAGE.vcxproj -property:Configuration=Release - msbuild INSTALL.vcxproj -property:Configuration=Release - ''' - } - } - stage('Build D x64') { - steps { - bat ''' - call c:\\progra~2\\micros~1\\2022\\buildtools\\vc\\auxiliary\\build\\vcvars64.bat - mkdir build_dbg - cd build_dbg - cmake -G "Visual Studio 17 2022" -T ClangCL -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON .. - msbuild astcencoder.sln -property:Configuration=Debug - ''' - } - } - stage('Build R Arm64') { - steps { - bat ''' - call c:\\progra~2\\micros~1\\2022\\buildtools\\vc\\auxiliary\\build\\vcvarsall.bat x64_arm64 - mkdir build_rel_arm64 - cd build_rel_arm64 - cmake -G "Visual Studio 17 2022" -A ARM64 -T ClangCL -DASTCENC_ISA_NEON=ON -DASTCENC_PACKAGE=arm64-clangcl .. - msbuild astcencoder.sln -property:Configuration=Release - msbuild PACKAGE.vcxproj -property:Configuration=Release - msbuild INSTALL.vcxproj -property:Configuration=Release - ''' - } - } - stage('Build D Arm64') { - steps { - bat ''' - call c:\\progra~2\\micros~1\\2022\\buildtools\\vc\\auxiliary\\build\\vcvarsall.bat x64_arm64 - mkdir build_dbg_arm64 - cd build_dbg_arm64 - cmake -G "Visual Studio 17 2022" -A ARM64 -T ClangCL -DASTCENC_ISA_NEON=ON .. - msbuild astcencoder.sln -property:Configuration=Debug - ''' - } - } - stage('Stash') { - steps { - dir('build_rel') { - stash name: 'astcenc-windows-x64-clangcl', includes: '*.zip' - } - dir('build_rel_arm64') { - stash name: 'astcenc-windows-arm64-clangcl', includes: '*.zip' - } - } - } - stage('Test') { - steps { - bat ''' - set Path=c:\\Python3;c:\\Python3\\Scripts;%Path% - call python ./Test/astc_test_image.py --test-set Small --test-quality medium - ''' - } - } - } - } - /* Build for macOS on x86-64 using Clang */ - stage('macOS') { - agent { - label 'mac && x86_64' - } - stages { - stage('Clean') { - steps { - sh 'git clean -ffdx' - } - } - stage('Build R') { - steps { - sh ''' - mkdir build_rel - cd build_rel - cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_UNIVERSAL_BUILD=OFF -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_PACKAGE=x64 .. - make install package -j4 - ''' - } - } - stage('Build D') { - steps { - sh ''' - mkdir build_dbg - cd build_dbg - cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Debug -DASTCENC_UNIVERSAL_BUILD=OFF -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_ISA_NONE=ON .. - make -j4 - ''' - } - } - stage('Stash') { - steps { - dir('build_rel') { - stash name: 'astcenc-macos-x64', includes: '*.zip' - } - } - } - stage('Test') { - steps { - sh ''' - export PATH=/usr/local/bin:$PATH - python3 ./Test/astc_test_image.py --test-set Small --test-quality medium - ''' - } - } - } - } - } - } - stage('Artifactory') { - agent { - kubernetes { - yaml ''' -apiVersion: v1 -kind: Pod -spec: - securityContext: - runAsUser: 1000 - runAsGroup: 1000 - imagePullSecrets: - - name: artifactory-ms-docker - containers: - - name: astcenc - image: mobile-studio--docker.eu-west-1.artifactory.aws.arm.com/astcenc:3.2.0 - command: - - sleep - args: - - infinity - resources: - requests: - cpu: 1 - memory: 4Gi -''' - defaultContainer 'astcenc' - } - } - options { - skipDefaultCheckout true - } - stages { - stage('Unstash') { - steps { - dir('upload/linux-x64') { - unstash 'astcenc-linux-x64' - } - dir('upload/windows-x64-cl') { - unstash 'astcenc-windows-x64-cl' - } - dir('upload/windows-x64-clangcl') { - unstash 'astcenc-windows-x64-clangcl' - } - dir('upload/windows-arm64-clangcl') { - unstash 'astcenc-windows-arm64-clangcl' - } - dir('upload/macos-x64') { - unstash 'astcenc-macos-x64' - } - } - } - stage('Upload') { - steps { - zip zipFile: 'astcenc.zip', dir: 'upload', archive: false - cepeArtifactoryUpload(sourcePattern: 'astcenc.zip') - } - } - } - post { - always { - deleteDir() - } - } - } - } - - post { - failure { - script { - slackSend channel: '#dsg-eng-astcenc', color: 'danger', message: "Build ${JOB_NAME} ${BUILD_NUMBER} failed. (<${BUILD_URL}|Open>)", teamDomain: 'arm-dsg', tokenCredentialId: 'jenkins-slack', username: 'jenkins' - } - } - } -} diff --git a/jenkins/release.Jenkinsfile b/jenkins/release.Jenkinsfile deleted file mode 100644 index 230a8855c..000000000 --- a/jenkins/release.Jenkinsfile +++ /dev/null @@ -1,370 +0,0 @@ -/* This pipeline is used for release testing, so it runs rarely. - * - * Test objectives for this pipeline are: - * - * - Run the entire pipeline in less than 60 minutes. - * - Test builds on all supported operating systems. - * - Test builds on optimized compiler choices (i.e. prefer Clang over GCC). - * - Build only release variants. - * - Run full functional tests. - * - Run full image quality tests. - * - Code sign the binaries on supported operating systems. - * - Build the release package. - * - * The test matrix is not fully covered; e.g. we can assume compilers behave - * similarly on different operating systems, so we test one compiler per OS. - */ - -@Library('hive-infra-library@changes/86/295486/1') _ - -pipeline { - agent none - - options { - ansiColor('xterm') - timestamps() - } - - stages { - stage('Build All') { - parallel { - /* Run static analysis on Linux */ - stage('Coverity') { - agent { - kubernetes { - yaml ''' -apiVersion: v1 -kind: Pod -spec: - securityContext: - runAsUser: 1000 - runAsGroup: 1000 - imagePullSecrets: - - name: artifactory-ms-docker - containers: - - name: astcenc - image: mobile-studio--docker.eu-west-1.artifactory.aws.arm.com/astcenc:3.2.0 - command: - - sleep - args: - - infinity - resources: - requests: - cpu: 4 - memory: 8Gi -''' - defaultContainer 'astcenc' - } - } - stages { - stage('Clean') { - steps { - sh 'git clean -fdx' - } - } - stage('Coverity') { - steps { - withCredentials([usernamePassword(credentialsId: 'jenkins-password', - usernameVariable: 'USERNAME', - passwordVariable: 'PASSWORD')]) { - sh script: '''#!/bin/bash - mkdir -p ${WORKSPACE}/occonfig - - mkdir build_cov - cd build_cov - - cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX2=ON .. - - cov-configure --config ${WORKSPACE}/coverity.conf --template --compiler cc --comptype gcc - cov-configure --config ${WORKSPACE}/coverity.conf --template --compiler c++ --comptype g++ - cov-build --config ${WORKSPACE}/coverity.conf --dir ${WORKSPACE}/intermediate make install - cov-analyze --dir ${WORKSPACE}/intermediate - cov-commit-defects --dir ${WORKSPACE}/intermediate \\ - --stream astcenc-master \\ - --url https://coverity.cambridge.arm.com \\ - --user jenkins@arm.com --password ${PASSWORD} \\ - --strip-path ${WORKSPACE} - ''' - } - } - } - } - } - /* Build for Linux on x86-64 using Clang */ - stage('Linux') { - agent { - kubernetes { - yaml '''\ - apiVersion: v1 - kind: Pod - spec: - securityContext: - runAsUser: 1000 - runAsGroup: 1000 - imagePullSecrets: - - name: artifactory-ms-docker - containers: - - name: astcenc - image: mobile-studio--docker.eu-west-1.artifactory.aws.arm.com/astcenc:3.2.0 - command: - - sleep - args: - - infinity - resources: - requests: - cpu: 4 - memory: 8Gi - limits: - cpu: 8 - memory: 16Gi - '''.stripIndent() - defaultContainer 'astcenc' - } - } - stages { - stage('Clean') { - steps { - sh 'git clean -ffdx' - } - } - stage('Build astcenc R x64') { - steps { - sh ''' - export CXX=clang++ - mkdir build_rel - cd build_rel - cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_PACKAGE=x64 .. - make install package -j4 - ''' - } - } - stage('Build astcdec R x64') { - steps { - sh ''' - export CXX=clang++ - mkdir build_reldec - cd build_reldec - cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_DECOMPRESSOR=ON .. - make -j4 - ''' - } - } - stage('Stash') { - steps { - dir('build_rel') { - stash name: 'astcenc-linux-x64', includes: '*.zip' - stash name: 'astcenc-linux-x64-hash', includes: '*.zip.sha256' - } - } - } - stage('Test') { - steps { - sh ''' - python3 ./Test/astc_test_functional.py - python3 ./Test/astc_test_image.py --encoder=all-x86 --test-set Small - ''' - } - } - } - } - /* Build for Windows on x86-64 using MSVC ClangCL */ - stage('Windows') { - agent { - label 'Windows' - } - stages { - stage('Clean') { - steps { - bat 'git clean -ffdx' - } - } - stage('Build R x64') { - steps { - bat ''' - call c:\\progra~2\\micros~1\\2022\\buildtools\\vc\\auxiliary\\build\\vcvars64.bat - mkdir build_rel - cd build_rel - cmake -G "Visual Studio 17 2022" -T ClangCL -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_PACKAGE=x64 .. - msbuild astcencoder.sln -property:Configuration=Release - msbuild PACKAGE.vcxproj -property:Configuration=Release - msbuild INSTALL.vcxproj -property:Configuration=Release - ''' - } - } - stage('Build R Arm64') { - steps { - bat ''' - call c:\\progra~2\\micros~1\\2022\\buildtools\\vc\\auxiliary\\build\\vcvarsall.bat x64_arm64 - mkdir build_rel_arm64 - cd build_rel_arm64 - cmake -G "Visual Studio 17 2022" -A ARM64 -T ClangCL -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_NEON=ON -DASTCENC_PACKAGE=arm64 .. - msbuild astcencoder.sln -property:Configuration=Release - msbuild PACKAGE.vcxproj -property:Configuration=Release - msbuild INSTALL.vcxproj -property:Configuration=Release - ''' - } - } - stage('Sign') { - steps { - dir('sign_tools') { - checkout changelog: false, - poll: false, - scm: [$class: 'GitSCM', - branches: [[name: '*/main']], - doGenerateSubmoduleConfigurations: false, - extensions: [], - submoduleCfg: [], - userRemoteConfigs: [[credentialsId: 'gerrit-jenkins-ssh', - url: 'ssh://mirror.eu-west-1.gerrit-eu01.aws.arm.com:29418/Hive/shared/signing']]] - } - withCredentials([usernamePassword(credentialsId: 'cepe-artifactory-jenkins', - usernameVariable: 'AF_USER', - passwordVariable: 'APIKEY')]) { - powershell 'C:\\Python311\\python.exe .\\sign_tools\\windows-client-wrapper.py -b $Env:BUILD_NUMBER -t $Env:APIKEY (Get-ChildItem -Filter build_rel\\*.zip)[0].FullName' - powershell 'C:\\Python311\\python.exe .\\sign_tools\\windows-client-wrapper.py -b $Env:BUILD_NUMBER -t $Env:APIKEY (Get-ChildItem -Filter build_rel_arm64\\*.zip)[0].FullName' - } - } - } - stage('Stash') { - steps { - dir('build_rel') { - stash name: 'astcenc-windows-x64', includes: '*.zip' - stash name: 'astcenc-windows-x64-hash', includes: '*.zip.sha256' - } - dir('build_rel_arm64') { - stash name: 'astcenc-windows-arm64', includes: '*.zip' - stash name: 'astcenc-windows-arm64-hash', includes: '*.zip.sha256' - } - } - } - stage('Test') { - steps { - bat ''' - set Path=c:\\Python38;c:\\Python38\\Scripts;%Path% - call python ./Test/astc_test_image.py --test-set Small - ''' - } - } - } - } - /* Build for macOS on x86-64 using Clang */ - stage('macOS') { - agent { - label 'mac && x86_64 && notarizer' - } - stages { - stage('Clean') { - steps { - sh 'git clean -ffdx' - } - } - stage('Build R') { - steps { - sh ''' - mkdir build_rel - cd build_rel - cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_PACKAGE=universal .. - make install package -j4 - ''' - } - } - stage('Sign and notarize') { - environment { - NOTARIZATION_CREDS = credentials('notarization-account') - } - steps { - dir('build_rel') { - sh 'git clone ssh://eu-gerrit-1.euhpc.arm.com:29418/Hive/shared/signing' - withCredentials([usernamePassword(credentialsId: 'win-signing', - usernameVariable: 'USERNAME', - passwordVariable: 'PASSWORD')]) { - sh 'python3 ./signing/macos-client-wrapper.py ${USERNAME} *.zip' - sh 'rm -rf ./signing' - } - } - } - } - stage('Stash') { - steps { - dir('build_rel') { - stash name: 'astcenc-macos-universal', includes: '*.zip' - stash name: 'astcenc-macos-universal-hash', includes: '*.zip.sha256' - } - } - } - stage('Test') { - steps { - sh ''' - export PATH=/usr/local/bin:$PATH - python3 ./Test/astc_test_image.py --test-set Small --encoder universal - ''' - } - } - } - } - } - } - stage('Artifactory') { - agent { - kubernetes { - yaml ''' -apiVersion: v1 -kind: Pod -spec: - securityContext: - runAsUser: 1000 - runAsGroup: 1000 - imagePullSecrets: - - name: artifactory-ms-docker - containers: - - name: astcenc - image: mobile-studio--docker.eu-west-1.artifactory.aws.arm.com/astcenc:3.2.0 - command: - - sleep - args: - - infinity - resources: - requests: - cpu: 1 - memory: 4Gi -''' - defaultContainer 'astcenc' - } - } - options { - skipDefaultCheckout true - } - stages { - stage('Unstash') { - steps { - dir('upload') { - unstash 'astcenc-windows-x64-hash' - unstash 'astcenc-windows-arm64-hash' - unstash 'astcenc-linux-x64-hash' - unstash 'astcenc-macos-universal-hash' - - unstash 'astcenc-windows-x64' - unstash 'astcenc-windows-arm64' - unstash 'astcenc-linux-x64' - unstash 'astcenc-macos-universal' - - sh 'cat *.sha256 > release-sha256.txt' - sh 'rm *.sha256' - } - } - } - stage('Upload') { - steps { - zip zipFile: 'astcenc.zip', dir: 'upload', archive: false - cepeArtifactoryUpload(sourcePattern: 'astcenc.zip') - } - } - } - post { - always { - deleteDir() - } - } - } - } -}