From 0a69186fd13188985586c971ca63a4eb079618e9 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Mon, 25 Nov 2024 12:52:17 -0700 Subject: [PATCH] chore(NODE-6578): add misc tooling to ZSTD repo (#36) --- .github/docker/Dockerfile.glibc | 5 + .github/docker/Dockerfile.musl | 8 +- .github/workflows/build.yml | 121 +++++++++++++++++++++++ .github/workflows/codeql.yml | 54 +++++++++++ .release-please-manifest.json | 3 + etc/install-zstd.sh | 3 +- package.json | 8 +- release-please-config.json | 14 +++ sbom.json | 167 +++----------------------------- 9 files changed, 223 insertions(+), 160 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/codeql.yml create mode 100644 .release-please-manifest.json create mode 100644 release-please-config.json diff --git a/.github/docker/Dockerfile.glibc b/.github/docker/Dockerfile.glibc index 1a6dc02..b20e124 100644 --- a/.github/docker/Dockerfile.glibc +++ b/.github/docker/Dockerfile.glibc @@ -17,6 +17,11 @@ RUN python3 --version RUN npm run install-zstd RUN npm install +RUN npm run prebuild ARG RUN_TEST RUN if [ -n "$RUN_TEST" ]; then npm test ; else echo "skipping tests" ; fi + +FROM scratch + +COPY --from=build /zstd/prebuilds/ / \ No newline at end of file diff --git a/.github/docker/Dockerfile.musl b/.github/docker/Dockerfile.musl index e571c43..8fca6ed 100644 --- a/.github/docker/Dockerfile.musl +++ b/.github/docker/Dockerfile.musl @@ -2,13 +2,19 @@ ARG PLATFORM=arm64 ARG NODE_VERSION=16.20.1 -FROM ${PLATFORM}/node:${NODE_VERSION}-alpine AS node +FROM ${PLATFORM}/node:${NODE_VERSION}-alpine AS build WORKDIR /zstd COPY . . RUN apk --no-cache add make g++ libc-dev curl bash python3 py3-pip vim cmake RUN npm run install-zstd && npm i +RUN npm run prebuild ARG RUN_TEST RUN if [ -n "$RUN_TEST" ]; then npm test ; else echo "skipping tests" ; fi + + +FROM scratch + +COPY --from=build /zstd/prebuilds/ / \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..a3d038b --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,121 @@ +on: + pull_request: + branches: [main] + workflow_dispatch: {} + workflow_call: {} + +name: Build + +jobs: + host_tests: + strategy: + matrix: + os: [macos-latest, windows-2019] + fail-fast: false + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 16.20.1 + cache: "npm" + registry-url: "https://registry.npmjs.org" + + - name: Install zstd + run: npm run install-zstd + shell: bash + + - name: install dependencies and compile + run: npm install --loglevel verbose && npm run prebuild + shell: bash + + - id: upload + name: Upload prebuild + uses: actions/upload-artifact@v4 + with: + name: build-${{ matrix.os }} + path: prebuilds/ + if-no-files-found: "error" + retention-days: 1 + compression-level: 0 + + container_tests_glibc: + runs-on: ubuntu-latest + strategy: + matrix: + linux_arch: [s390x, arm64, amd64] + fail-fast: false + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Run Buildx + run: | + docker buildx create --name builder --bootstrap --use + docker buildx build \ + --platform linux/${{ matrix.linux_arch }} \ + --build-arg="NODE_ARCH=${{ matrix.linux_arch == 'amd64' && 'x64' || matrix.linux_arch }}" \ + --build-arg="NODE_VERSION=16.20.1" \ + --output type=local,dest=./prebuilds,platform-split=false \ + -f ./.github/docker/Dockerfile.glibc \ + . + + - id: upload + name: Upload prebuild + uses: actions/upload-artifact@v4 + with: + name: build-linux-glibc-${{ matrix.linux_arch }} + path: prebuilds/ + if-no-files-found: "error" + retention-days: 1 + compression-level: 0 + + container_tests_musl: + runs-on: ubuntu-latest + strategy: + matrix: + linux_arch: [amd64, arm64] + fail-fast: false + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Run Buildx + run: | + docker buildx create --name builder --bootstrap --use + docker --debug buildx build --progress=plain --no-cache \ + --platform linux/${{ matrix.linux_arch }} \ + --build-arg="PLATFORM=${{ matrix.linux_arch == 'arm64' && 'arm64v8' || matrix.linux_arch }}" \ + --build-arg="NODE_VERSION=16.20.1" \ + --output type=local,dest=./prebuilds,platform-split=false \ + -f ./.github/docker/Dockerfile.musl \ + . + + - id: upload + name: Upload prebuild + uses: actions/upload-artifact@v4 + with: + name: build-linux-musl-${{ matrix.linux_arch }} + path: prebuilds/ + if-no-files-found: "error" + retention-days: 1 + compression-level: 0 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..2c292b6 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,54 @@ +name: "CodeQL" + +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + runs-on: "ubuntu-latest" + timeout-minutes: 360 + permissions: + # required for all workflows + security-events: write + + # required to fetch internal or private CodeQL packs + packages: read + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + include: + - language: "c-cpp" + build-mode: "manual" + sourceDirectory: "./addon" + - language: "javascript-typescript" + build-mode: "none" + sourceDirectory: "./lib" + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + source-root: ${{ matrix.sourceDirectory }} + + - if: matrix.build-mode == 'manual' + shell: bash + run: npm i --ignore-scripts && npm run install-zstd && npm run compile + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 0000000..b832e7e --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "2.0.0-beta.0" +} \ No newline at end of file diff --git a/etc/install-zstd.sh b/etc/install-zstd.sh index 12c0c22..62a129f 100644 --- a/etc/install-zstd.sh +++ b/etc/install-zstd.sh @@ -8,8 +8,9 @@ clean_deps() { download_zstd() { rm -rf deps mkdir -p deps/zstd + ZSTD_VERSION=$(node -p "require('./package.json')['mongodb:zstd_version']") - curl -L "https://github.com/facebook/zstd/releases/download/v1.5.6/zstd-1.5.6.tar.gz" \ + curl -L "https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-$ZSTD_VERSION.tar.gz" \ | tar -zxf - -C deps/zstd --strip-components 1 } diff --git a/package.json b/package.json index 6fcb06a..92dfb38 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,8 @@ "install-zstd": "bash etc/install-zstd.sh", "check:eslint": "ESLINT_USE_FLAT_CONFIG=false eslint *ts lib/*.js test/*.js .*.json", "clang-format": "clang-format --style=file:.clang-format --Werror -i addon/*", - "check:clang-format": "clang-format --style=file:.clang-format --dry-run --Werror addon/*" + "check:clang-format": "clang-format --style=file:.clang-format --dry-run --Werror addon/*", + "prebuild": "prebuild --runtime napi --strip --verbose --all" }, "overrides": { "prebuild": { @@ -49,5 +50,6 @@ "napi_versions": [ 4 ] - } -} + }, + "mongodb:zstd_version": "1.5.6" +} \ No newline at end of file diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 0000000..c68f124 --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,14 @@ +{ + "pull-request-header": "Please run the release_notes action before releasing to generate release highlights", + "packages": { + ".": { + "include-component-in-tag": false, + "changelog-path": "HISTORY.md", + "release-type": "node", + "bump-minor-pre-major": false, + "bump-patch-for-minor-pre-major": false, + "draft": false + } + }, + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json" +} \ No newline at end of file diff --git a/sbom.json b/sbom.json index a9bb67b..c949d51 100644 --- a/sbom.json +++ b/sbom.json @@ -1,175 +1,31 @@ { "components": [ { - "bom-ref": "pkg:cargo/futures@0.3", + "bom-ref": "pkg:github/facebook/zstd@1.5.6", "externalReferences": [ { "type": "distribution", - "url": "https://crates.io/api/v1/crates/futures/0.3/download" - }, - { - "type": "vcs", - "url": "https://github.com/rust-lang/futures-rs" - }, - { - "type": "website", - "url": "https://crates.io/crates/futures/0.3" - } - ], - "licenses": [ - { - "license": { - "name": "Apache-2.0" - } - }, - { - "license": { - "name": "MIT" - } - } - ], - "name": "futures", - "purl": "pkg:cargo/futures@0.3", - "type": "library", - "version": "0.3" - }, - { - "bom-ref": "pkg:cargo/napi@2.4.3", - "externalReferences": [ - { - "type": "distribution", - "url": "https://crates.io/api/v1/crates/napi/2.4.3/download" - }, - { - "type": "vcs", - "url": "https://github.com/napi-rs/napi-rs" - }, - { - "type": "website", - "url": "https://crates.io/crates/napi/2.4.3" - } - ], - "licenses": [ - { - "license": { - "name": "MIT" - } - } - ], - "name": "napi", - "purl": "pkg:cargo/napi@2.4.3", - "type": "library", - "version": "2.4.3" - }, - { - "bom-ref": "pkg:cargo/napi-derive@2.4.1", - "externalReferences": [ - { - "type": "distribution", - "url": "https://crates.io/api/v1/crates/napi-derive/2.4.1/download" - }, - { - "type": "vcs", - "url": "https://github.com/napi-rs/napi-rs" - }, - { - "type": "website", - "url": "https://crates.io/crates/napi-derive/2.4.1" - } - ], - "licenses": [ - { - "license": { - "name": "MIT" - } - } - ], - "name": "napi-derive", - "purl": "pkg:cargo/napi-derive@2.4.1", - "type": "library", - "version": "2.4.1" - }, - { - "bom-ref": "pkg:cargo/zstd@0.11", - "externalReferences": [ - { - "type": "distribution", - "url": "https://crates.io/api/v1/crates/zstd/0.11/download" - }, - { - "type": "vcs", - "url": "https://github.com/gyscos/zstd-rs" + "url": "https://github.com/facebook/zstd/archive/refs/tags/1.5.6.tar.gz" }, { "type": "website", - "url": "https://crates.io/crates/zstd/0.11" - } - ], - "licenses": [ - { - "license": { - "name": "MIT" - } + "url": "https://github.com/facebook/zstd/tree/1.5.6" } ], + "group": "facebook", "name": "zstd", - "purl": "pkg:cargo/zstd@0.11", - "type": "library", - "version": "0.11" - }, - { - "bom-ref": "pkg:cargo/zstd-sys@2.0.8", - "externalReferences": [ - { - "type": "distribution", - "url": "https://crates.io/api/v1/crates/zstd-sys/2.0.8/download" - }, - { - "type": "vcs", - "url": "https://github.com/gyscos/zstd-rs" - }, - { - "type": "website", - "url": "https://crates.io/crates/zstd-sys/2.0.8" - } - ], - "licenses": [ - { - "license": { - "name": "Apache-2.0" - } - }, - { - "license": { - "name": "MIT" - } - } - ], - "name": "zstd-sys", - "purl": "pkg:cargo/zstd-sys@2.0.8", + "purl": "pkg:github/facebook/zstd@1.5.6", "type": "library", - "version": "2.0.8" + "version": "1.5.6" } ], "dependencies": [ { - "ref": "pkg:cargo/futures@0.3" - }, - { - "ref": "pkg:cargo/napi-derive@2.4.1" - }, - { - "ref": "pkg:cargo/napi@2.4.3" - }, - { - "ref": "pkg:cargo/zstd-sys@2.0.8" - }, - { - "ref": "pkg:cargo/zstd@0.11" + "ref": "pkg:github/facebook/zstd@1.5.6" } ], "metadata": { - "timestamp": "2024-05-01T21:03:39.164915+00:00", + "timestamp": "2024-11-22T18:19:11.589830+00:00", "tools": [ { "externalReferences": [ @@ -212,9 +68,10 @@ } ] }, - "serialNumber": "urn:uuid:a8980ce9-1205-4b26-bed3-a0ad798988a6", + "serialNumber": "urn:uuid:09d58a28-c1dc-43da-ad85-67c806e16b87", "version": 1, "$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json", "bomFormat": "CycloneDX", - "specVersion": "1.5" -} + "specVersion": "1.5", + "vulnerabilities": [] +} \ No newline at end of file