diff --git a/.github/workflows/00_pull-request-entry-point.yml b/.github/workflows/00_pull-request-entry-point.yml index a8f10ef..7534d75 100644 --- a/.github/workflows/00_pull-request-entry-point.yml +++ b/.github/workflows/00_pull-request-entry-point.yml @@ -26,6 +26,7 @@ jobs: go-projects-changes: ${{ steps.go-projects-changes.outputs.changes }} go-mod-changes: ${{ steps.go-projects-changes.outputs.go-mod }} common-libs-changes: ${{ steps.go-projects-changes.outputs.common-libs }} + any-common-changes: ${{ fromJson(steps.go-projects-changes.outputs.go-mod) || fromJson(steps.go-projects-changes.outputs.common-libs) }} cmd-changes: ${{ steps.per-component-changes.outputs.cmd-changes }} tests-changes: ${{ steps.per-component-changes.outputs.tests-changes }} cmd-unchanged: ${{ steps.per-component-changes.outputs.cmd-unchanged }} @@ -68,37 +69,55 @@ jobs: echo "cmd-unchanged=${cmd_unchanged}" >> "${GITHUB_OUTPUT}" + debug-projects-changed: + needs: [ projects-changed ] + runs-on: [ self-hosted, small ] + steps: + - name: show outputs + env: + CHANGES_GO_PROJECTS: ${{ needs.projects-changed.outputs.go-projects-changes }} + CHANGES_GO_MOD: ${{ needs.projects-changed.outputs.go-mod-changes }} + CHANGES_COMMON_LIB: ${{ needs.projects-changed.outputs.common-libs-changes }} + CHANGES_ANY_COMMON: ${{ needs.projects-changed.outputs.any-common-changes }} + CHANGES_CMD_CHANGES: ${{ needs.projects-changed.outputs.cmd-changes }} + CHANGES_TEST_CHANGES: ${{ needs.projects-changed.outputs.tests-changes }} + CHANGES_CMD_UNCHANGED: ${{ needs.projects-changed.outputs.cmd-unchanged }} + run: | + env | grep -E "^CHANGES" + build-or-retag-go-projects: needs: [ check-permissions, projects-changed ] strategy: matrix: go-component: ${{ fromJson('["gh-action", "history-exporter"]') }} - uses: ./.github/workflows/50_go_build-one-component.yml + uses: ./.github/workflows/50_go_build-component.yml secrets: inherit name: 'Build: ${{ matrix.go-component }}' with: - component-name: ${{ matrix.go-component }} + cmd-path: './cmd/${{ matrix.go-component }}' base-sha: ${{ github.event.pull_request.base.sha }} head-sha: ${{ github.event.pull_request.head.sha }} docker-image-base: neondatabase/${{ matrix.go-component }} - retag-base-image: ${{ contains(needs.projects-changed.outputs.cmd-unchanged, matrix.go-component) }} - run-build: ${{ contains(needs.projects-changed.outputs.cmd-changes, matrix.go-component) }} + retag-base-image: ${{ contains(needs.projects-changed.outputs.cmd-unchanged, matrix.go-component) && (needs.projects-changed.outputs.any-common-changes != 'true') }} pack-to-docker-image: true + goos: linux + goarch: amd64 test-go-projects: needs: [ check-permissions, projects-changed ] strategy: matrix: go-component: ${{ fromJson('["gh-action", "history-exporter"]') }} - uses: ./.github/workflows/50_go_build-one-component.yml + uses: ./.github/workflows/50_go_test-component.yml secrets: inherit name: 'Unit Tests: ${{ matrix.go-component }}' with: - component-name: ${{ matrix.go-component }} + cmd-path: './cmd/${{ matrix.go-component }}' base-sha: ${{ github.event.pull_request.base.sha }} head-sha: ${{ github.event.pull_request.head.sha }} - docker-image-base: neondatabase/${{ matrix.go-component }} - run-tests: true + skip-tests: ${{ contains(needs.projects-changed.outputs.cmd-unchanged, matrix.go-component) && (! contains(needs.projects-changed.outputs.tests-changes, format('{0}-tests', matrix.go-component))) && (needs.projects-changed.outputs.any-common-changes != 'true') }} + goos: linux + goarch: amd64 e2e-tests: needs: [ build-or-retag-go-projects ] @@ -111,7 +130,17 @@ jobs: needs: [ test-go-projects, e2e-tests ] if: ${{ always() }} runs-on: [ self-hosted, small ] + env: + GO_TESTS_RESULT: ${{ needs.test-go-projects.result }} + E2E_TESTS_RESULT: ${{ needs.e2e-tests.result }} steps: - name: Print final message run: | echo Thats it. + echo "We can add here labels to PR to make it visible if any parts are failing" + - name: Conclusion + run: | + exit_code=0 + if [ "${GO_TESTS_RESULT}" != "success" ]; then exit_code=$(( exit_code + 1 )); fi + if [ "${E2E_TESTS_RESULT}" != "success" ]; then exit_code=$(( exit_code + 2 )); fi + exit "${exit_code}" diff --git a/.github/workflows/00_push-entry-point.yml b/.github/workflows/00_push-entry-point.yml index 4b4e183..2938d8c 100644 --- a/.github/workflows/00_push-entry-point.yml +++ b/.github/workflows/00_push-entry-point.yml @@ -15,11 +15,11 @@ jobs: strategy: matrix: go-component: [ 'gh-action', 'history-exporter' ] - uses: ./.github/workflows/50_go_build-one-component.yml + uses: ./.github/workflows/50_go_build-component.yml secrets: inherit name: 'Golang: ${{ matrix.go-component }}' with: - component-name: ${{ matrix.go-component }} + cmd-path: './cmd/${{ matrix.go-component}}' base-sha: ${{ github.event.after }} head-sha: ${{ github.event.after }} docker-image-base: neondatabase/${{ matrix.go-component }} @@ -27,20 +27,22 @@ jobs: pack-to-docker-image: true go-build-cache-restore-or-update: 'update' go-mod-cache-restore-or-update: 'update' + goos: linux + goarch: amd64 test-go-projects: strategy: matrix: go-component: ${{ fromJson('["gh-action", "history-exporter"]') }} - uses: ./.github/workflows/50_go_build-one-component.yml + uses: ./.github/workflows/50_go_test-component.yml secrets: inherit name: 'Unit Tests: ${{ matrix.go-component }}' with: - component-name: ${{ matrix.go-component }} + cmd-path: './cmd/${{ matrix.go-component}}' base-sha: ${{ github.event.after }} head-sha: ${{ github.event.after }} - docker-image-base: neondatabase/${{ matrix.go-component }} - run-tests: true + goos: linux + goarch: amd64 e2e-tests: needs: [ build-go-projects ] diff --git a/.github/workflows/50_go_build-component.yml b/.github/workflows/50_go_build-component.yml new file mode 100644 index 0000000..946bfbd --- /dev/null +++ b/.github/workflows/50_go_build-component.yml @@ -0,0 +1,195 @@ +name: Build Go cmd + +on: + workflow_call: + inputs: + cmd-path: + required: true + type: string + go-build-cache-restore-or-update: + required: false + type: string + default: 'restore' + go-mod-path: + required: false + type: string + default: 'go.mod' + go-mod-cache-restore-or-update: + required: false + type: string + default: 'restore' + goarch: + required: false + type: string + default: '' # default is to take it from ${{ runner.arch }} + goos: + required: false + type: string + default: '' # default is to take it from ${{ runner.os }} + docker-image-base: + required: false + type: string + default: '' + retag-base-image: + required: false + type: boolean + default: false + run-build: + required: false + type: boolean + default: false + run-tests: + required: false + type: boolean + default: false + pack-to-docker-image: + required: false + type: boolean + default: false + base-sha: + required: false + type: string + default: 'anysha' + head-sha: + required: false + type: string + default: 'anysha' + + +jobs: + prepare-running-env: + name: Build ${{ inputs.cmd-path}} for "${{ inputs.goos }}/${{ inputs.goarch }}" + runs-on: [ self-hosted, small ] + outputs: + goarch: ${{ steps.goarch-goos.outputs.goarch }} + goos: ${{ steps.goarch-goos.outputs.goos }} + go-mod-cache-key: ${{ steps.set-cache-keys.outputs.go-mod-cache-key }} + go-build-cache-key-cmd: ${{ steps.set-cache-keys.outputs.go-build-cache-key-cmd }} + go-build-cache-key-test: ${{ steps.set-cache-keys.outputs.go-build-cache-key-test }} + prepare-runner-os: ${{ runner.os }} + prepare-runner-arch: ${{ runner.arch }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + id: setup-go + with: + cache: false + go-version-file: ${{ inputs.go-mod-path }} + - name: Set GOARCH and GOOS + id: goarch-goos + run: | + GOARCH_INPUT="${{ inputs.goarch }}" + GOOS_INPUT="${{ inputs.goos }}" + declare -A runner_to_go_os=( ["Linux"]="linux" ["Windows"]="windows" ["macOS"]="darwin" ) + declare -A runner_to_go_arch=( ["X86"]="386" ["X64"]="amd64" ["ARM"]="arm" ["ARM64"]="arm64" ) + ARCH_FROM_RUNNER="${runner_to_go_arch[${{ runner.arch }}]}" + OS_FROM_RUNNER="${runner_to_go_os[${{ runner.os }}]}" + echo "goarch=${GOARCH_INPUT:-$ARCH_FROM_RUNNER}" >> "$GITHUB_OUTPUT" + echo "goos=${GOOS_INPUT:-$OS_FROM_RUNNER}" >> "$GITHUB_OUTPUT" + - id: set-cache-keys + name: Set keys needed for caching + env: + OS: "${{ steps.goarch-goos.outputs.goos }}" + ARCH: "${{ steps.goarch-goos.outputs.goarch }}" + GO_VER: "${{ steps.setup-go.outputs.go-version }}" + GO_SUM_HASH: "${{ hashFiles('**/go.sum') }}" + GH_REPO: "${{ github.event.repository.full_name }}" + BASE_SHA: "${{ inputs.base-sha }}" + HEAD_SHA: ${{ inputs.head-sha }} + CMD_PATH: ${{ inputs.cmd-path }} + run: | + GH_REPO="${GH_REPO//\//_}" # '//\//_' wat? + # here how it works: + # ${var//pattern/replace}, -- to replace all the occurencise of 'pattern' with 'replace' + # and pattern is '/', so escape it: '\/'. + # and replace is just '_', easy. + CMD_PATH="${CMD_PATH//\//_}" # well, now you know what '//\//_' means and how it works, right? :) + + echo "go-mod-cache-key=go-mod-cache-${OS}-${ARCH}-go-${GO_VER}-${GO_SUM_HASH}" >> "$GITHUB_OUTPUT" + echo "go-build-cache-key-base=go-build-cache-${OS}-${ARCH}-go-${GO_VER}-gh-${GH_REPO}-git-${BASE_SHA}-build-${CMD_PATH}" >> "$GITHUB_OUTPUT" + echo "go-build-cache-key-head=go-build-cache-${OS}-${ARCH}-go-${GO_VER}-gh-${GH_REPO}-git-${HEAD_SHA}-build-${CMD_PATH}" >> "$GITHUB_OUTPUT" + - id: docker-login + if: ${{ inputs.retag-base-image || inputs.pack-to-docker-image }} + uses: docker/login-action@v3 + with: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + - id: docker-retag + if: ${{ inputs.retag-base-image }} + env: + DOCKER_IMAGE_BASE: ${{ inputs.docker-image-base }} + BASE_SHA: ${{ inputs.base-sha }} + HEAD_SHA: ${{ inputs.head-sha }} + BASE_TAG: arch-${{ steps.goarch-goos.outputs.goarch }}-commit-${{ inputs.base-sha }} + HEAD_TAG: arch-${{ steps.goarch-goos.outputs.goarch }}-commit-${{ inputs.head-sha }} + PLATFORM: ${{ steps.goarch-goos.outputs.goarch }} + continue-on-error: true + run: | + docker buildx imagetools create \ + --tag "${DOCKER_IMAGE_BASE}:${HEAD_TAG}" \ + "${DOCKER_IMAGE_BASE}:${BASE_TAG}" + - name: Go Mod Cache + uses: tespkg/actions-cache@v1 + if: ${{ (inputs.go-build-cache-restore-or-update == 'update') && (steps.docker-retag.outcome != 'success') }} + with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false + path: | + ${{ steps.set-cache-keys.outputs.go-mod-cache }} + key: ${{ steps.set-cache-keys.outputs.go-mod-cache-key }} + - name: Go Mod Cache Restore + uses: tespkg/actions-cache/restore@v1 + if: ${{ (inputs.go-build-cache-restore-or-update == 'restore') && (steps.docker-retag.outcome != 'success') }} + with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false + path: | + ${{ steps.set-cache-keys.outputs.go-mod-cache }} + key: ${{ steps.set-cache-keys.outputs.go-mod-cache-key }} + - name: Go Build Cache + uses: tespkg/actions-cache@v1 + if: ${{ (inputs.go-build-cache-restore-or-update == 'update') && (steps.docker-retag.outcome != 'success') }} + with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false + path: | + ${{ steps.set-cache-keys.outputs.go-build }} + key: ${{ steps.set-cache-keys.outputs.go-build-cache-key-head }} + - name: Go Build + if: ${{ steps.docker-retag.outcome != 'success' }} + env: + GOARCH: ${{ steps.goarch-goos.outputs.goarch }} + GOOS: ${{ steps.goarch-goos.outputs.goos }} + CMD_PATH: ${{ inputs.cmd-path }} + run: | + go build -v -o "build/$(basename "${CMD_PATH}")" "${CMD_PATH}" + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Pack as Docker image + if: ${{ steps.docker-retag.outcome != 'success' }} + env: + DOCKER_IMAGE_BASE: ${{ inputs.docker-image-base }} + BASE_TAG: arch-${{ steps.goarch-goos.outputs.goarch }}-commit-${{ inputs.base-sha }} + HEAD_TAG: arch-${{ steps.goarch-goos.outputs.goarch }}-commit-${{ inputs.head-sha }} + PLATFORM: ${{ steps.goarch-goos.outputs.goarch }} + CMD_PATH: ${{ inputs.cmd-path }} + run: | + echo "+++ BUILD:" + ls -laht build/ + docker build \ + --tag "${DOCKER_IMAGE_BASE}:${HEAD_TAG}" \ + --platform "${PLATFORM}" \ + --build-arg BINARY_TO_ADD="$(basename "${CMD_PATH}")" \ + --file Dockerfiles/go-app-common.Dockerfile \ + build/ + docker push \ + "${DOCKER_IMAGE_BASE}:${HEAD_TAG}" diff --git a/.github/workflows/50_go_build-one-arch-one-os-one-path.yml b/.github/workflows/50_go_build-one-arch-one-os-one-path.yml deleted file mode 100644 index d7ad96f..0000000 --- a/.github/workflows/50_go_build-one-arch-one-os-one-path.yml +++ /dev/null @@ -1,331 +0,0 @@ -name: Build Go cmd - -on: - workflow_call: - inputs: - cmd-path: - required: true - type: string - go-build-cache-restore-or-update: - required: false - type: string - default: 'restore' - go-mod-path: - required: false - type: string - default: 'go.mod' - go-mod-cache-restore-or-update: - required: false - type: string - default: 'restore' - goarch: - required: false - type: string - default: '' # default is to take it from ${{ runner.arch }} - goos: - required: false - type: string - default: '' # default is to take it from ${{ runner.os }} - docker-image-base: - required: false - type: string - default: '' - retag-base-image: - required: false - type: boolean - default: false - run-build: - required: false - type: boolean - default: false - run-tests: - required: false - type: boolean - default: false - pack-to-docker-image: - required: false - type: boolean - default: false - base-sha: - required: false - type: string - default: 'anysha' - head-sha: - required: false - type: string - default: 'anysha' - - -jobs: - prepare-running-env: - name: Prepare env for "${{ inputs.goos }}/${{ inputs.goarch }}" - runs-on: [ self-hosted, small ] - if: ${{ inputs.retag-base-image || inputs.run-build || inputs.run-tests }} - outputs: - goarch: ${{ steps.goarch-goos.outputs.goarch }} - goos: ${{ steps.goarch-goos.outputs.goos }} - go-mod-cache-key: ${{ steps.set-cache-keys.outputs.go-mod-cache-key }} - go-build-cache-key-cmd: ${{ steps.set-cache-keys.outputs.go-build-cache-key-cmd }} - go-build-cache-key-test: ${{ steps.set-cache-keys.outputs.go-build-cache-key-test }} - prepare-runner-os: ${{ runner.os }} - prepare-runner-arch: ${{ runner.arch }} - steps: - - uses: actions/checkout@v4 - with: - sparse-checkout: | - go.mod - go.sum - sparse-checkout-cone-mode: false - - uses: actions/setup-go@v5 - id: setup-go - with: - cache: false - go-version-file: ${{ inputs.go-mod-path }} - - name: Set GOARCH and GOOS - id: goarch-goos - run: | - GOARCH_INPUT="${{ inputs.goarch }}" - GOOS_INPUT="${{ inputs.goos }}" - declare -A runner_to_go_os=( ["Linux"]="linux" ["Windows"]="windows" ["macOS"]="darwin" ) - declare -A runner_to_go_arch=( ["X86"]="386" ["X64"]="amd64" ["ARM"]="arm" ["ARM64"]="arm64" ) - ARCH_FROM_RUNNER="${runner_to_go_arch[${{ runner.arch }}]}" - OS_FROM_RUNNER="${runner_to_go_os[${{ runner.os }}]}" - echo "goarch=${GOARCH_INPUT:-$ARCH_FROM_RUNNER}" >> "$GITHUB_OUTPUT" - echo "goos=${GOOS_INPUT:-$OS_FROM_RUNNER}" >> "$GITHUB_OUTPUT" - - id: set-cache-keys - name: Set keys needed for caching - run: | - OS="${{ steps.goarch-goos.outputs.goos }}" - ARCH="${{ steps.goarch-goos.outputs.goarch }}" - GO_VER="${{ steps.setup-go.outputs.go-version }}" - GO_SUM_HASH="${{ hashFiles('**/go.sum') }}" - GH_REPO="${{ github.event.repository.full_name }}" - GH_REPO="${GH_REPO//\//_}" # '//\//_' wat? - # here how it works: - # ${var//pattern/replace}, -- to replace all the occurencise of 'pattern' with 'replace' - # and pattern is '/', so escape it: '\/'. - # and replace is just '_', easy. - GIT_SHA="${{ inputs.base-sha }}" - CMD_PATH="${{ inputs.cmd-path }}" - CMD_PATH="${CMD_PATH//\//_}" # well, now you know what '//\//_' means and how it works, right? :) - - echo "go-mod-cache-key=go-mod-cache-${OS}-${ARCH}-go-${GO_VER}-${GO_SUM_HASH}" >> "$GITHUB_OUTPUT" - for run_target in cmd test; do - echo "go-build-cache-key-${run_target}=go-build-cache-${OS}-${ARCH}-go-${GO_VER}-gh-${GH_REPO}-git-${GIT_SHA}-${run_target}-${CMD_PATH}" >> "$GITHUB_OUTPUT" - done - - build: - name: "Build for the ${{ inputs.goos }}/${{ inputs.goarch }} ${{ inputs.pack-to-docker-image && 'and pack to the Docker image'}}" - needs: [ prepare-running-env ] - runs-on: [ self-hosted, small ] - if: ${{ inputs.retag-base-image || inputs.run-build }} - steps: - - id: keys-sanity-check - run: | - exit_code=0 - if [ -z "${{ inputs.goos }}" ] && [ "${{ needs.prepare-running-env.outputs.prepare-runner-os }}" != "${{ runner.os }}" ]; then - echo "Failed sanity check, different OS for the prepare step and build step." - echo "Either use the same runner OS there, or defines GOOS explicitly." - exit_code=$(( exit_code + 1 )) - fi - if [ -z "${{ inputs.goarch }}" ] && [ "${{ needs.prepare-running-env.outputs.prepare-runner-arch }}" != "${{ runner.arch }}" ]; then - echo "Failed sanity check, different ARCH for the prepare step and build step." - echo "Either use the same runner ARCH there, or defines GOARCH explicitly." - exit_code=$(( exit_code + 2 )) - fi - exit ${exit_code} - - id: docker-login - if: ${{ inputs.retag-base-image || inputs.pack-to-docker-image }} - uses: docker/login-action@v3 - with: - username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} - password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - id: setup-go - with: - cache: false - go-version-file: ${{ inputs.go-mod-path }} - - id: setup-keys - run: | - echo "go-build=$(go env GOCACHE)" >> "$GITHUB_OUTPUT" - echo "go-mod-cache=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" - echo "goarch=${{ needs.prepare-running-env.outputs.goarch }}" >> "$GITHUB_OUTPUT" - echo "goos=${{ needs.prepare-running-env.outputs.goos }}" >> "$GITHUB_OUTPUT" - echo "go-mod-cache-key=${{ needs.prepare-running-env.outputs.go-mod-cache-key }}" >> "$GITHUB_OUTPUT" - echo "go-build-cache-key-cmd=${{ needs.prepare-running-env.outputs.go-build-cache-key-cmd }}" >> "$GITHUB_OUTPUT" - - id: docker-retag - if: ${{ inputs.retag-base-image }} - env: - DOCKER_IMAGE_BASE: ${{ inputs.docker-image-base }} - BASE_SHA: ${{ inputs.base-sha }} - HEAD_SHA: ${{ inputs.head-sha }} - BASE_TAG: arch-${{ steps.setup-keys.outputs.goarch }}-commit-${{ inputs.base-sha }} - HEAD_TAG: arch-${{ steps.setup-keys.outputs.goarch }}-commit-${{ inputs.head-sha }} - PLATFORM: ${{ steps.setup-keys.outputs.goarch }} - continue-on-error: true - run: | - docker buildx imagetools create \ - --tag "${DOCKER_IMAGE_BASE}:${HEAD_TAG}" \ - "${DOCKER_IMAGE_BASE}:${BASE_TAG}" - - name: Go Mod Cache - uses: tespkg/actions-cache@v1 - if: ${{ inputs.go-mod-cache-restore-or-update == 'update' }} - with: - endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} - bucket: ${{ vars.HETZNER_CACHE_BUCKET }} - accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} - secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} - use-fallback: false - path: | - ${{ steps.setup-keys.outputs.go-mod-cache }} - key: ${{ steps.setup-keys.outputs.go-mod-cache-key }} - - name: Go Mod Cache Restore - uses: tespkg/actions-cache/restore@v1 - if: ${{ inputs.go-mod-cache-restore-or-update == 'restore' }} - with: - endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} - bucket: ${{ vars.HETZNER_CACHE_BUCKET }} - accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} - secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} - use-fallback: false - path: | - ${{ steps.setup-keys.outputs.go-mod-cache }} - key: ${{ steps.setup-keys.outputs.go-mod-cache-key }} - - name: Go Build Cache - uses: tespkg/actions-cache@v1 - if: ${{ inputs.go-build-cache-restore-or-update == 'update' }} - with: - endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} - bucket: ${{ vars.HETZNER_CACHE_BUCKET }} - accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} - secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} - use-fallback: false - path: | - ${{ steps.setup-keys.outputs.go-build }} - key: ${{ steps.setup-keys.outputs.go-build-cache-key-cmd }} - - name: Go Build Cache Restore - uses: tespkg/actions-cache/restore@v1 - if: ${{ inputs.go-build-cache-restore-or-update == 'restore' }} - with: - endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} - bucket: ${{ vars.HETZNER_CACHE_BUCKET }} - accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} - secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} - use-fallback: false - path: | - ${{ steps.setup-keys.outputs.go-build }} - key: ${{ steps.setup-keys.outputs.go-build-cache-key-cmd }} - - name: Go Build - if: ${{ inputs.run-build || (inputs.retag-base-image && steps.docker-retag.outcome != 'success') }} - env: - GOARCH: ${{ steps.setup-keys.outputs.goarch }} - GOOS: ${{ steps.setup-keys.outputs.goos }} - run: | - GOARCH=${{ steps.setup-keys.outputs.goarch }} \ - GOOS=${{ steps.setup-keys.outputs.goos }} \ - go build -v -o "build/$(basename ${{ inputs.cmd-path }})" ${{ inputs.cmd-path }} - - name: Pack as Docker image - if: ${{ inputs.pack-to-docker-image || (inputs.retag-base-image && steps.docker-retag.outcome != 'success') }} - env: - DOCKER_IMAGE_BASE: ${{ inputs.docker-image-base }} - BASE_TAG: arch-${{ steps.setup-keys.outputs.goarch }}-commit-${{ inputs.base-sha }} - HEAD_TAG: arch-${{ steps.setup-keys.outputs.goarch }}-commit-${{ inputs.head-sha }} - PLATFORM: ${{ steps.setup-keys.outputs.goarch }} - run: | - echo "+++ BUILD:" - ls -laht build/ - docker build \ - --tag "${DOCKER_IMAGE_BASE}:${HEAD_TAG}" \ - --platform "${PLATFORM}" \ - --build-arg BINARY_TO_ADD="$(basename ${{ inputs.cmd-path }})" \ - --file Dockerfiles/go-app-common.Dockerfile \ - build/ - docker image ls - docker push \ - "${DOCKER_IMAGE_BASE}:${HEAD_TAG}" - - test: - name: "Test for the ${{ inputs.goos }}/${{ inputs.goarch }}" - needs: [ prepare-running-env ] - runs-on: [ self-hosted, small ] - if: ${{ inputs.run-tests }} - steps: - - id: keys-sanity-check - run: | - exit_code=0 - if [ -z "${{ inputs.goos }}" ] && [ "${{ needs.prepare-running-env.outputs.prepare-runner-os }}" != "${{ runner.os }}" ]; then - echo "Failed sanity check, different OS for the prepare step and build step." - echo "Either use the same runner OS there, or defines GOOS explicitly." - exit_code=$(( exit_code + 1 )) - fi - if [ -z "${{ inputs.goarch }}" ] && [ "${{ needs.prepare-running-env.outputs.prepare-runner-arch }}" != "${{ runner.arch }}" ]; then - echo "Failed sanity check, different ARCH for the prepare step and build step." - echo "Either use the same runner ARCH there, or defines GOARCH explicitly." - exit_code=$(( exit_code + 2 )) - fi - exit ${exit_code} - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - id: setup-go - with: - cache: false - go-version-file: ${{ inputs.go-mod-path }} - - id: setup-keys - run: | - echo "go-build=$(go env GOCACHE)" >> "$GITHUB_OUTPUT" - echo "go-mod-cache=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" - echo "goarch=${{ needs.prepare-running-env.outputs.goarch }}" >> "$GITHUB_OUTPUT" - echo "goos=${{ needs.prepare-running-env.outputs.goos }}" >> "$GITHUB_OUTPUT" - echo "go-mod-cache-key=${{ needs.prepare-running-env.outputs.go-mod-cache-key }}" >> "$GITHUB_OUTPUT" - echo "go-build-cache-key-cmd=${{ needs.prepare-running-env.outputs.go-build-cache-key-cmd }}" >> "$GITHUB_OUTPUT" - - name: Go Mod Cache - uses: tespkg/actions-cache@v1 - if: ${{ inputs.go-mod-cache-restore-or-update == 'update' }} - with: - endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} - bucket: ${{ vars.HETZNER_CACHE_BUCKET }} - accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} - secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} - use-fallback: false - path: | - ${{ steps.setup-keys.outputs.go-mod-cache }} - key: ${{ steps.setup-keys.outputs.go-mod-cache-key }} - - name: Go Mod Cache Restore - uses: tespkg/actions-cache/restore@v1 - if: ${{ inputs.go-mod-cache-restore-or-update == 'restore' }} - with: - endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} - bucket: ${{ vars.HETZNER_CACHE_BUCKET }} - accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} - secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} - use-fallback: false - path: | - ${{ steps.setup-keys.outputs.go-mod-cache }} - key: ${{ steps.setup-keys.outputs.go-mod-cache-key }} - - name: Go Build Cache - uses: tespkg/actions-cache@v1 - if: ${{ inputs.go-build-cache-restore-or-update == 'update' }} - with: - endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} - bucket: ${{ vars.HETZNER_CACHE_BUCKET }} - accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} - secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} - use-fallback: false - path: | - ${{ steps.setup-keys.outputs.go-build }} - key: ${{ steps.setup-keys.outputs.go-build-cache-key-cmd }} - - name: Go Build Cache Restore - uses: tespkg/actions-cache/restore@v1 - if: ${{ inputs.go-build-cache-restore-or-update == 'restore' }} - with: - endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} - bucket: ${{ vars.HETZNER_CACHE_BUCKET }} - accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} - secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} - use-fallback: false - path: | - ${{ steps.setup-keys.outputs.go-build }} - key: ${{ steps.setup-keys.outputs.go-build-cache-key-cmd }} - - name: Go Test - run: GOARCH=${{ steps.setup-keys.outputs.goarch }} GOOS=${{ steps.setup-keys.outputs.goos }} go test -v ${{ inputs.cmd-path }} diff --git a/.github/workflows/50_go_test-component.yml b/.github/workflows/50_go_test-component.yml new file mode 100644 index 0000000..dec6b6f --- /dev/null +++ b/.github/workflows/50_go_test-component.yml @@ -0,0 +1,136 @@ +name: Test Go cmd + +on: + workflow_call: + inputs: + cmd-path: + required: true + type: string + skip-tests: + required: false + type: boolean + default: false + go-cache-restore-or-update: + required: false + type: string + default: 'restore' + go-mod-path: + required: false + type: string + default: 'go.mod' + go-mod-cache-restore-or-update: + required: false + type: string + default: 'restore' + goarch: + required: false + type: string + default: '' # default is to take it from ${{ runner.arch }} + goos: + required: false + type: string + default: '' # default is to take it from ${{ runner.os }} + base-sha: + required: false + type: string + default: 'anysha' + head-sha: + required: false + type: string + default: 'anysha' + + +jobs: + go-test: + name: Test ${{ inputs.cmd-path }} for "${{ inputs.goos }}/${{ inputs.goarch }}" + if: ${{ ! inputs.skip-tests }} + runs-on: [ self-hosted, small ] + outputs: + goarch: ${{ steps.goarch-goos.outputs.goarch }} + goos: ${{ steps.goarch-goos.outputs.goos }} + go-mod-cache-key: ${{ steps.set-cache-keys.outputs.go-mod-cache-key }} + go-build-cache-key-cmd: ${{ steps.set-cache-keys.outputs.go-build-cache-key-cmd }} + go-build-cache-key-test: ${{ steps.set-cache-keys.outputs.go-build-cache-key-test }} + prepare-runner-os: ${{ runner.os }} + prepare-runner-arch: ${{ runner.arch }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + id: setup-go + with: + cache: false + go-version-file: ${{ inputs.go-mod-path }} + - name: Set GOARCH and GOOS + id: goarch-goos + run: | + GOARCH_INPUT="${{ inputs.goarch }}" + GOOS_INPUT="${{ inputs.goos }}" + declare -A runner_to_go_os=( ["Linux"]="linux" ["Windows"]="windows" ["macOS"]="darwin" ) + declare -A runner_to_go_arch=( ["X86"]="386" ["X64"]="amd64" ["ARM"]="arm" ["ARM64"]="arm64" ) + ARCH_FROM_RUNNER="${runner_to_go_arch[${{ runner.arch }}]}" + OS_FROM_RUNNER="${runner_to_go_os[${{ runner.os }}]}" + echo "goarch=${GOARCH_INPUT:-$ARCH_FROM_RUNNER}" >> "$GITHUB_OUTPUT" + echo "goos=${GOOS_INPUT:-$OS_FROM_RUNNER}" >> "$GITHUB_OUTPUT" + - id: set-cache-keys + name: Set keys needed for caching + env: + OS: "${{ steps.goarch-goos.outputs.goos }}" + ARCH: "${{ steps.goarch-goos.outputs.goarch }}" + GO_VER: "${{ steps.setup-go.outputs.go-version }}" + GO_SUM_HASH: "${{ hashFiles('**/go.sum') }}" + GH_REPO: "${{ github.event.repository.full_name }}" + BASE_SHA: "${{ inputs.base-sha }}" + HEAD_SHA: ${{ inputs.head-sha }} + CMD_PATH: ${{ inputs.cmd-path }} + run: | + GH_REPO="${GH_REPO//\//_}" # '//\//_' wat? + # here how it works: + # ${var//pattern/replace}, -- to replace all the occurencise of 'pattern' with 'replace' + # and pattern is '/', so escape it: '\/'. + # and replace is just '_', easy. + CMD_PATH="${CMD_PATH//\//_}" # well, now you know what '//\//_' means and how it works, right? :) + + echo "go-mod-cache-key=go-mod-cache-${OS}-${ARCH}-go-${GO_VER}-${GO_SUM_HASH}" >> "$GITHUB_OUTPUT" + echo "go-build-cache-key-base=go-build-cache-${OS}-${ARCH}-go-${GO_VER}-gh-${GH_REPO}-git-${BASE_SHA}-tests-${CMD_PATH}" >> "$GITHUB_OUTPUT" + echo "go-build-cache-key-head=go-build-cache-${OS}-${ARCH}-go-${GO_VER}-gh-${GH_REPO}-git-${HEAD_SHA}-tests-${CMD_PATH}" >> "$GITHUB_OUTPUT" + - name: Go Mod Cache + uses: tespkg/actions-cache@v1 + if: ${{ inputs.go-mod-cache-restore-or-update == 'update' }} + with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false + path: | + ${{ steps.set-cache-keys.outputs.go-mod-cache }} + key: ${{ steps.set-cache-keys.outputs.go-mod-cache-key-head }} + - name: Go Mod Cache Restore + uses: tespkg/actions-cache/restore@v1 + if: ${{ inputs.go-mod-cache-restore-or-update == 'restore' }} + with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false + path: | + ${{ steps.set-cache-keys.outputs.go-mod-cache }} + key: ${{ steps.set-cache-keys.outputs.go-mod-cache-key }} + - name: Go Build Cache + uses: tespkg/actions-cache@v1 + with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false + path: | + ${{ steps.set-cache-keys.outputs.go-build }} + key: ${{ steps.set-cache-keys.outputs.go-build-cache-key-head }} + - name: Go Test + env: + GOARCH: ${{ steps.goarch-goos.outputs.goarch }} + GOOS: ${{ steps.goarch-goos.outputs.goos }} + CMD_PATH: ${{ inputs.cmd-path }} + run: go test -v "${CMD_PATH}" diff --git a/Dockerfiles/go-app-common.Dockerfile b/Dockerfiles/go-app-common.Dockerfile index d3df06a..63f9b37 100644 --- a/Dockerfiles/go-app-common.Dockerfile +++ b/Dockerfiles/go-app-common.Dockerfile @@ -1,6 +1,14 @@ -ARG BASE_IMAGE=debian:bookworm-slim +ARG DEBIAN_VERSION=bookworm +ARG DEBIAN_FLAVOR=${DEBIAN_VERSION}-slim -FROM --platform=$TARGETPLATFORM ${BASE_IMAGE} +ARG BOOKWORM_SLIM_SHA=sha256:40b107342c492725bc7aacbe93a49945445191ae364184a6d24fedb28172f6f7 +ARG BULLSEYE_SLIM_SHA=sha256:e831d9a884d63734fe3dd9c491ed9a5a3d4c6a6d32c5b14f2067357c49b0b7e1 + +ARG BASE_IMAGE_SHA=debian@$BOOKWORM_SLIM_SHA +# ARG BASE_IMAGE_SHA=${BASE_IMAGE_SHA/debian:bookworm-slim/debian@$BOOKWORM_SLIM_SHA} +# ARG BASE_IMAGE_SHA=${BASE_IMAGE_SHA/debian:bullseye-slim/debian@$BULLSEYE_SLIM_SHA} + +FROM --platform=$TARGETPLATFORM ${BASE_IMAGE_SHA} ARG BINARY_TO_ADD ADD ${BINARY_TO_ADD} /usr/local/bin/ diff --git a/cmd/history-exporter/main.go b/cmd/history-exporter/main.go index 9b807c2..8f56d39 100644 --- a/cmd/history-exporter/main.go +++ b/cmd/history-exporter/main.go @@ -1,7 +1,5 @@ package main -// Trigger changes - import ( "context" "flag" diff --git a/pkg/db/db.go b/pkg/db/db.go index a7959b1..971560b 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -1,5 +1,7 @@ package db +// Trigger + import ( "context" "fmt"