From 7cde7e8328d25411d4ae6d79fb1259113c5911c4 Mon Sep 17 00:00:00 2001 From: Trevor Vaughan Date: Fri, 8 May 2026 17:42:42 -0400 Subject: [PATCH 1/2] test!: add Ginkgo integration tests and migrate CI to UBI10 containers BREAKING CHANGE: Container base images switch from Alpine/distroless to UBI10 (ubi-minimal for certs, ubi-micro for runtime). CA cert path changes to /etc/pki/tls/certs/ca-bundle.crt. CI jobs run inside UBI10 containers. hack/ directory contents moved to configs/ and deploy/. - Add Ginkgo v2 E2E tests at tests/integration/ with per-layer sub-packages (base, storage, storage-tls, enrichment), mock Compass TLS server, and shared Go helpers - Containerize all ci_local.yml jobs on UBI10 with new integration-test job using vfs storage driver for nested podman - Add compose profiles (storage, enrichment, debug) with COLLECTOR_CONFIG env var for layer-specific collector config selection - Move hack/ contents to configs/ (collector configs, loki, openssl.cnf) and deploy/ (terraform) - Centralize tool management in .taskfiles/tools.yml with SHA256 verification via .tool_checksums - Bump go-gemara to v0.4.0; add Ginkgo CLI as tool directive in go.mod - Drop unused ctx variable in proofwatch/cmd/validate-logs/main.go Details: Integration tests: - Mock Compass at tests/integration/mock-compass/ serves fixture-driven /v1/enrich and /healthz responses over TLS - Typed Go helpers (postEvidence, queryLoki, listS3Objects, execInContainer) replace Venom HTTP and exec executors - RustFS test bucket uses anonymous public access for plain HTTP S3 ListObjectsV2 queries - Ginkgo runs via `go tool ginkgo` (tool directive, no global install) - CI passes --succinct; local runs use -vv for full hierarchy output CI (ci_local.yml): - All jobs run in registry.access.redhat.com/ubi10/ubi:latest with safe.directory workaround for actions/checkout container job issue - golangci-lint sets GOWORK=off and GOFLAGS=-buildvcs=false to lint modules independently in the workspace repo - integration-test job pins podman-compose with --require-hashes Compose: - RustFS healthcheck uses /health (not /minio/health/live) - Collector volumes use :U flag for podman rootless UID remapping - Loki queries use indexed resource attributes directly, replacing the removed {exporter="OTLP"} label - Fix S3_DISABLE_SSL env var to drop unsupported ${VAR:-default} syntax in collector config Refs: complytime/complytime-collector-components#181 Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/ci_local.yml | 129 +++++- .github/workflows/ci_version_sync.yml | 32 +- .gitignore | 13 +- .gitleaks.toml | 2 + .taskfiles/dev.yml | 4 +- .taskfiles/infra.yml | 48 ++- .taskfiles/integration.yml | 188 +++++++++ .taskfiles/quality.yml | 10 - .../scripts/check-go-mod-consistency.sh | 95 ----- .taskfiles/scripts/check-go-version.sh | 47 --- .taskfiles/scripts/check-otel-versions.sh | 47 --- .taskfiles/scripts/go-module-runner.sh | 24 +- .taskfiles/scripts/sync-all-otel-versions.sh | 99 ----- .taskfiles/scripts/sync-manifest-versions.sh | 37 -- .taskfiles/scripts/version-check.sh | 177 +++++++++ .taskfiles/scripts/version-sync.sh | 137 +++++++ .taskfiles/tools.yml | 31 ++ .taskfiles/version.yml | 34 +- .tool_checksums | 8 + AGENTS.md | 371 +++--------------- README.md | 19 +- Taskfile.yml | 11 +- beacon-distro/Containerfile.collector | 2 +- compose.yaml | 105 ++++- configs/collector-base.yaml | 65 +++ .../collector-enrichment.yaml | 2 +- configs/collector-storage-tls.yaml | 191 +++++++++ configs/collector-storage.yaml | 190 +++++++++ .../loki-config.yaml => configs/loki.yaml | 0 .../self-signed-cert => configs}/openssl.cnf | 2 +- {hack/demo => deploy}/terraform/.gitignore | 0 {hack/demo => deploy}/terraform/README.md | 0 {hack/demo => deploy}/terraform/main.tf | 0 {hack/demo => deploy}/terraform/outputs.tf | 0 deploy/terraform/variables.tf | 16 + docs/DESIGN.md | 2 + docs/DEVELOPMENT.md | 73 ++-- go.mod | 56 +++ go.sum | 140 +++++++ hack/demo/terraform/variables.tf | 12 - proofwatch/cmd/validate-logs/main.go | 24 +- proofwatch/go.mod | 8 +- proofwatch/go.sum | 8 +- tests/integration/README.md | 112 ++++++ tests/integration/base/base_suite_test.go | 43 ++ tests/integration/base/base_test.go | 76 ++++ .../enrichment/enrichment_suite_test.go | 43 ++ .../integration/enrichment/enrichment_test.go | 47 +++ .../fixtures/compass-responses.json | 30 ++ .../integration/fixtures/evidence-fail.json | 24 +- .../fixtures/evidence-malformed.json | 1 + .../integration/fixtures/evidence-pass.json | 22 +- .../fixtures/evidence-unknown.json | 35 ++ tests/integration/go.mod | 23 ++ tests/integration/go.sum | 69 ++++ tests/integration/helpers.go | 189 +++++++++ tests/integration/mock-compass/Containerfile | 13 + tests/integration/mock-compass/go.mod | 3 + tests/integration/mock-compass/main.go | 148 +++++++ .../integration/storage/storage_suite_test.go | 47 +++ tests/integration/storage/storage_test.go | 52 +++ .../storage_tls/storage_tls_suite_test.go | 43 ++ .../storage_tls/storage_tls_test.go | 55 +++ truthbeam/go.mod | 7 +- truthbeam/go.sum | 7 +- 65 files changed, 2698 insertions(+), 850 deletions(-) create mode 100644 .gitleaks.toml create mode 100644 .taskfiles/integration.yml delete mode 100755 .taskfiles/scripts/check-go-mod-consistency.sh delete mode 100755 .taskfiles/scripts/check-go-version.sh delete mode 100755 .taskfiles/scripts/check-otel-versions.sh delete mode 100755 .taskfiles/scripts/sync-all-otel-versions.sh delete mode 100755 .taskfiles/scripts/sync-manifest-versions.sh create mode 100755 .taskfiles/scripts/version-check.sh create mode 100755 .taskfiles/scripts/version-sync.sh create mode 100644 .taskfiles/tools.yml create mode 100644 .tool_checksums create mode 100644 configs/collector-base.yaml rename hack/demo/demo-config.yaml => configs/collector-enrichment.yaml (99%) create mode 100644 configs/collector-storage-tls.yaml create mode 100644 configs/collector-storage.yaml rename hack/demo/loki-config.yaml => configs/loki.yaml (100%) rename {hack/self-signed-cert => configs}/openssl.cnf (91%) rename {hack/demo => deploy}/terraform/.gitignore (100%) rename {hack/demo => deploy}/terraform/README.md (100%) rename {hack/demo => deploy}/terraform/main.tf (100%) rename {hack/demo => deploy}/terraform/outputs.tf (100%) create mode 100644 deploy/terraform/variables.tf create mode 100644 go.mod create mode 100644 go.sum delete mode 100644 hack/demo/terraform/variables.tf create mode 100644 tests/integration/README.md create mode 100644 tests/integration/base/base_suite_test.go create mode 100644 tests/integration/base/base_test.go create mode 100644 tests/integration/enrichment/enrichment_suite_test.go create mode 100644 tests/integration/enrichment/enrichment_test.go create mode 100644 tests/integration/fixtures/compass-responses.json rename hack/sampledata/evidence.json => tests/integration/fixtures/evidence-fail.json (76%) create mode 100644 tests/integration/fixtures/evidence-malformed.json rename hack/sampledata/evidence-code-review.json => tests/integration/fixtures/evidence-pass.json (76%) create mode 100644 tests/integration/fixtures/evidence-unknown.json create mode 100644 tests/integration/go.mod create mode 100644 tests/integration/go.sum create mode 100644 tests/integration/helpers.go create mode 100644 tests/integration/mock-compass/Containerfile create mode 100644 tests/integration/mock-compass/go.mod create mode 100644 tests/integration/mock-compass/main.go create mode 100644 tests/integration/storage/storage_suite_test.go create mode 100644 tests/integration/storage/storage_test.go create mode 100644 tests/integration/storage_tls/storage_tls_suite_test.go create mode 100644 tests/integration/storage_tls/storage_tls_test.go diff --git a/.github/workflows/ci_local.yml b/.github/workflows/ci_local.yml index 31e88cdc..ce9fea57 100644 --- a/.github/workflows/ci_local.yml +++ b/.github/workflows/ci_local.yml @@ -15,17 +15,30 @@ permissions: security-events: none env: - GO_VERSION: 1.25 - GOLANGCI_LINT_VERSION: v2.5 + GO_VERSION: 1.26 + GOLANGCI_LINT_VERSION: v2.12 WEAVER_VERSION: v0.21.2 + UBI_BASE_PACKAGES: git-core jobs: detect-modules: runs-on: ubuntu-latest + container: + image: registry.access.redhat.com/ubi10/ubi:10.2@sha256:f56ffeed0a79e53594be6164adfb199a4c4ed8925098f5905b0875f0edb46aa1 outputs: modules: "${{ steps.set-modules.outputs.modules }}" steps: + - name: Install prerequisites + run: dnf install -y --nodocs ${{ env.UBI_BASE_PACKAGES }} jq + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + # Workaround: actions/checkout set-safe-directory does not reliably + # persist in container jobs — the gitconfig it writes can land in a + # HOME that subsequent tools don't read. + # See: https://github.com/actions/checkout/issues/1048 + # https://github.com/actions/runner/issues/2033 + - name: Trust workspace directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: go-version: "${{ env.GO_VERSION }}" @@ -44,11 +57,25 @@ jobs: golangci-lint: needs: detect-modules runs-on: ubuntu-latest + container: + image: registry.access.redhat.com/ubi10/ubi:10.2@sha256:f56ffeed0a79e53594be6164adfb199a4c4ed8925098f5905b0875f0edb46aa1 strategy: matrix: modules: "${{ fromJSON(needs.detect-modules.outputs.modules) }}" + env: + # Each module is linted independently via the matrix; disable workspace + # mode so golangci-lint does not pass -mod=mod (forbidden in workspace). + GOWORK: "off" + # VCS stamping is irrelevant for linting; disable it so golangci-lint + # never shells out to git for build metadata. + GOFLAGS: "-buildvcs=false" steps: + - name: Install prerequisites + run: dnf install -y --nodocs ${{ env.UBI_BASE_PACKAGES }} + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Trust workspace directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: go-version: "${{ env.GO_VERSION }}" @@ -60,8 +87,15 @@ jobs: test: runs-on: ubuntu-latest + container: + image: registry.access.redhat.com/ubi10/ubi:10.2@sha256:f56ffeed0a79e53594be6164adfb199a4c4ed8925098f5905b0875f0edb46aa1 steps: + - name: Install prerequisites + run: dnf install -y --nodocs ${{ env.UBI_BASE_PACKAGES }} + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Trust workspace directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: go-version: "${{ env.GO_VERSION }}" @@ -77,11 +111,22 @@ jobs: weave-check: runs-on: ubuntu-latest + container: + image: registry.access.redhat.com/ubi10/ubi:10.2@sha256:f56ffeed0a79e53594be6164adfb199a4c4ed8925098f5905b0875f0edb46aa1 permissions: contents: read steps: + - name: Install prerequisites + run: dnf install -y --nodocs ${{ env.UBI_BASE_PACKAGES }} xz + - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Trust workspace directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version: "${{ env.GO_VERSION }}" - name: Install Task uses: arduino/setup-task@b91d5d2c96a56797b48ac1e0e89220bf64044611 # v2.0.0 @@ -90,15 +135,7 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install weaver - run: | - echo "Installing weaver ${{ env.WEAVER_VERSION }}" - curl -fL -o /tmp/weaver.tar.xz "https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver-x86_64-unknown-linux-gnu.tar.xz" - mkdir -p "${HOME}/bin" - tar -xJf /tmp/weaver.tar.xz - mv weaver-x86_64-unknown-linux-gnu/weaver "${HOME}/bin/weaver" - chmod +x "${HOME}/bin/weaver" - echo "${HOME}/bin" >> "${GITHUB_PATH}" - rm -rf "weaver-x86_64-unknown-linux-gnu/" + run: task tools:install-weaver - name: Setup Go workspace run: task workspace @@ -111,8 +148,15 @@ jobs: verify-codegen: runs-on: ubuntu-latest + container: + image: registry.access.redhat.com/ubi10/ubi:10.2@sha256:f56ffeed0a79e53594be6164adfb199a4c4ed8925098f5905b0875f0edb46aa1 steps: + - name: Install prerequisites + run: dnf install -y --nodocs ${{ env.UBI_BASE_PACKAGES }} xz + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Trust workspace directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: go-version: "${{ env.GO_VERSION }}" @@ -124,15 +168,7 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install weaver - run: | - echo "Installing weaver ${{ env.WEAVER_VERSION }}" - curl -fL -o /tmp/weaver.tar.xz "https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver-x86_64-unknown-linux-gnu.tar.xz" - mkdir -p "${HOME}/bin" - tar -xJf /tmp/weaver.tar.xz - mv weaver-x86_64-unknown-linux-gnu/weaver "${HOME}/bin/weaver" - chmod +x "${HOME}/bin/weaver" - echo "${HOME}/bin" >> "${GITHUB_PATH}" - rm -rf weaver-x86_64-unknown-linux-gnu/ + run: task tools:install-weaver - name: Install oapi-codegen run: go install github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen@v2.5.0 @@ -168,3 +204,56 @@ jobs: else echo "SUCCESS: No diffs detected. Code generation is up to date." fi + + integration-test: + runs-on: ubuntu-latest + container: + image: registry.access.redhat.com/ubi10/ubi:10.2@sha256:f56ffeed0a79e53594be6164adfb199a4c4ed8925098f5905b0875f0edb46aa1 + # SECURITY: --privileged required for container-in-container scenario + # GitHub Actions containers use overlayfs which conflicts with Podman's + # default overlay driver. VFS storage + privileged mode enables Podman + # to manage container networking and storage inside the CI container. + options: --privileged + needs: [test] + steps: + - name: Install prerequisites + run: | + dnf install -y --nodocs ${{ env.UBI_BASE_PACKAGES }} podman openssl python3-pip python3-pyyaml perl + pip install --require-hashes --no-deps -r /dev/stdin <<'EOF' + python-dotenv==1.2.2 --hash=sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a + podman-compose==1.5.0 --hash=sha256:f0b9d35f4da1b309172adf208a5cb7a882b532a834c2202666c1988b6f147546 + EOF + + # Podman's default overlay driver cannot stack on the overlayfs + # backing the GitHub Actions container job — use vfs instead. + - name: Configure Podman storage for nested containers + run: | + mkdir -p /etc/containers + printf '[storage]\ndriver = "vfs"\n' > /etc/containers/storage.conf + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Trust workspace directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version: "${{ env.GO_VERSION }}" + + - name: Install Task + uses: arduino/setup-task@b91d5d2c96a56797b48ac1e0e89220bf64044611 # v2.0.0 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup workspace + run: task workspace + + - name: Run integration tests + run: task integration:test + + - name: Upload test results + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: integration-test-results + path: .test-output/integration/ + retention-days: 7 diff --git a/.github/workflows/ci_version_sync.yml b/.github/workflows/ci_version_sync.yml index 69b50707..a30aa654 100644 --- a/.github/workflows/ci_version_sync.yml +++ b/.github/workflows/ci_version_sync.yml @@ -1,19 +1,23 @@ # Version Synchronization Check # ============================= -# Ensures beacon-distro manifest.yaml stays in sync with truthbeam OTel versions +# Ensures Go and OTel versions are aligned across all modules, +# Containerfiles, manifest, and CI workflows. name: Version Sync Check on: pull_request: paths: - - 'truthbeam/go.mod' + - 'go.work' + - '**/go.mod' - 'beacon-distro/manifest.yaml' - 'beacon-distro/Containerfile.collector' + - 'tests/integration/mock-compass/Containerfile' push: branches: [main] paths: - - 'truthbeam/go.mod' + - 'go.work' + - '**/go.mod' - 'beacon-distro/manifest.yaml' # Minimal permissions at workflow level @@ -32,23 +36,5 @@ jobs: version: 3.x repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Check Go version alignment - run: task version:check-go-version - - - name: Check OTel version alignment - run: task version:check-otel-versions - - - name: Verify manifest can be synced - run: | - # Run sync in dry-run mode (save original first) - cp beacon-distro/manifest.yaml /tmp/manifest.yaml.orig - task version:sync-otel-versions - - # Check if there are differences - if ! diff -q beacon-distro/manifest.yaml /tmp/manifest.yaml.orig; then - echo "ERROR: manifest.yaml is out of sync with truthbeam" - echo "Run 'task version:sync-otel-versions' locally and commit the changes" - exit 1 - fi - - echo "✓ manifest.yaml is in sync" + - name: Check version alignment + run: task version:check diff --git a/.gitignore b/.gitignore index 708b1195..153b04ea 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ coverage.* *.coverprofile profile.cov .test-output/ +*.log # Dependency directories (remove the comment below to include it) vendor/ @@ -41,9 +42,8 @@ megalinter-reports/ bin/ data/ -# self-signed certificate -hack/self-signed-cert/compass* -hack/self-signed-cert/truthbeam* +# self-signed certificates (generated by task infra:generate-self-signed-cert) +certs/ # Unbound Force — managed by uf init # Runtime data under .uf/ (databases, caches, locks, logs) @@ -67,10 +67,3 @@ hack/self-signed-cert/truthbeam* .unbound-force/ .muti-mind/ .mx-f/ - -# Unbound Force convention packs not applicable to this repository -.opencode/uf/packs/typescript.md -.opencode/uf/packs/typescript-custom.md - -# MegaLinter generated reports -megalinter-reports/ diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 00000000..54748573 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,2 @@ +[extend] +useDefault = true diff --git a/.taskfiles/dev.yml b/.taskfiles/dev.yml index 7eb73f27..0378e84f 100644 --- a/.taskfiles/dev.yml +++ b/.taskfiles/dev.yml @@ -25,9 +25,7 @@ tasks: test: desc: Run unit tests with coverage and version checks for all modules cmds: - - task: :version:check-go-version - - task: :version:check-go-mod-consistency - - task: :version:check-otel-versions + - task: :version:check - mkdir -p {{.TEST_DIR}} - bash {{.ROOT_DIR}}/.taskfiles/scripts/go-module-runner.sh test diff --git a/.taskfiles/infra.yml b/.taskfiles/infra.yml index 054cf6e5..47271b64 100644 --- a/.taskfiles/infra.yml +++ b/.taskfiles/infra.yml @@ -1,8 +1,8 @@ version: '3' vars: - CERT_DIR: '{{.CERT_DIR | default (printf "%s/hack/self-signed-cert" .ROOT_DIR)}}' - OPENSSL_CNF: '{{.CERT_DIR}}/openssl.cnf' + CERT_DIR: '{{.CERT_DIR | default (printf "%s/certs" .ROOT_DIR)}}' + OPENSSL_CNF: '{{.ROOT_DIR}}/configs/openssl.cnf' IMAGE: '{{.IMAGE | default "complybeacon/collector"}}' TAG: '{{.TAG | default "latest"}}' @@ -15,13 +15,53 @@ tasks: desc: Build the beacon collector container image cmds: - task: :dev:deps - - task: :version:sync-otel-versions + - task: :version:sync - podman build -f Containerfile.collector -t {{.IMAGE}}:{{.TAG}} {{.ROOT_DIR}}/beacon-distro + generate-self-signed-cert: + desc: Generate self-signed certificates for compass, truthbeam, and rustfs + cmds: + - rm -rf {{.CERT_DIR}} && mkdir -p {{.CERT_DIR}} + - echo "--- Generating self-signed certificates in {{.CERT_DIR}} ---" + - openssl genrsa -out {{.CERT_DIR}}/truthbeam.key 2048 + - cmd: | + openssl req -x509 -new -nodes -key {{.CERT_DIR}}/truthbeam.key -sha256 -days 365 \ + -subj "/CN=ComplyBeacon Root CA" \ + -extensions v3_ca -config {{.OPENSSL_CNF}} \ + -out {{.CERT_DIR}}/truthbeam.crt + - openssl genrsa -out {{.CERT_DIR}}/compass.key 2048 + - openssl req -new -key {{.CERT_DIR}}/compass.key -out {{.CERT_DIR}}/compass.csr -config {{.OPENSSL_CNF}} + - cmd: | + openssl x509 -req -in {{.CERT_DIR}}/compass.csr -CA {{.CERT_DIR}}/truthbeam.crt -CAkey {{.CERT_DIR}}/truthbeam.key -CAcreateserial \ + -out {{.CERT_DIR}}/compass.crt -days 365 -sha256 \ + -extfile {{.OPENSSL_CNF}} -extensions v3_req + - openssl genrsa -out {{.CERT_DIR}}/rustfs.key 2048 + - cmd: | + openssl req -new -key {{.CERT_DIR}}/rustfs.key -out {{.CERT_DIR}}/rustfs.csr \ + -subj "/C=US/ST=California/L=San Francisco/O=ComplyBeacon/OU=Dev/CN=rustfs-tls" + - cmd: | + # Create temporary extension file for RustFS certificate + cat > {{.CERT_DIR}}/rustfs_ext.cnf << EOF + [v3_req] + subjectAltName = @alt_names + basicConstraints = CA:FALSE + + [alt_names] + DNS.1 = rustfs-tls + DNS.2 = localhost + IP.1 = 127.0.0.1 + EOF + openssl x509 -req -in {{.CERT_DIR}}/rustfs.csr -CA {{.CERT_DIR}}/truthbeam.crt -CAkey {{.CERT_DIR}}/truthbeam.key -CAcreateserial \ + -out {{.CERT_DIR}}/rustfs.crt -days 365 -sha256 \ + -extensions v3_req -extfile {{.CERT_DIR}}/rustfs_ext.cnf + rm {{.CERT_DIR}}/rustfs_ext.cnf + - chmod 644 {{.CERT_DIR}}/*.key + - echo "--- Certificates generated successfully ---" + deploy: desc: Deploy infra (auto-syncs OTel versions first) cmds: - - task: :version:sync-otel-versions + - task: :version:sync - podman-compose -f compose.yaml up undeploy: diff --git a/.taskfiles/integration.yml b/.taskfiles/integration.yml new file mode 100644 index 00000000..4bc54122 --- /dev/null +++ b/.taskfiles/integration.yml @@ -0,0 +1,188 @@ +version: '3' + +vars: + GINKGO_OUTPUT_DIR: '{{.TEST_DIR | default (printf "%s/.test-output" .ROOT_DIR)}}/integration' + CERT_DIR: '{{.CERT_DIR | default (printf "%s/certs" .ROOT_DIR)}}' + +tasks: + default: + silent: true + cmd: task integration --list + + ensure-certs: + desc: Generate self-signed certificates if they do not exist + status: + - test -f {{.CERT_DIR}}/compass.crt + - test -f {{.CERT_DIR}}/compass.key + - test -f {{.CERT_DIR}}/truthbeam.crt + - test -f {{.CERT_DIR}}/truthbeam.key + - test -f {{.CERT_DIR}}/rustfs.crt + - test -f {{.CERT_DIR}}/rustfs.key + cmds: + - task: :infra:generate-self-signed-cert + + test: + desc: Run all integration test layers sequentially + cmds: + - task: ensure-certs + - task: test-layer + vars: {PROFILE: base} + - task: test-layer + vars: {PROFILE: storage} + - task: test-layer + vars: {PROFILE: enrichment} + + test-profile: + desc: "Run a single integration test layer (PROFILE=base|storage|enrichment|storage-tls)" + requires: + vars: [PROFILE] + cmds: + - task: ensure-certs + - task: test-layer + vars: {PROFILE: '{{.PROFILE}}'} + + test-layer: + internal: true + requires: + vars: [PROFILE] + vars: + COLLECTOR_CONFIG: + sh: | + case "{{.PROFILE}}" in + base) echo "configs/collector-base.yaml" ;; + storage) echo "configs/collector-storage.yaml" ;; + storage-tls) echo "configs/collector-storage-tls.yaml" ;; + enrichment) echo "configs/collector-enrichment.yaml" ;; + *) echo "INVALID_PROFILE" ;; + esac + COMPOSE_PROFILES: + sh: | + case "{{.PROFILE}}" in + base) echo "" ;; + storage) echo "--profile storage" ;; + storage-tls) echo "--profile storage-tls" ;; + enrichment) echo "--profile storage --profile enrichment" ;; + *) echo "" ;; + esac + GINKGO_FLAGS: + sh: | + if [ "${CI:-}" = "true" ]; then + echo "--succinct" + else + echo "-vv" + fi + TEST_DIR: './tests/integration/{{.PROFILE | replace "-" "_"}}/' + env: + COLLECTOR_CONFIG: '{{.COLLECTOR_CONFIG}}' + GOWORK: "off" + cmds: + - echo "=== Running {{.PROFILE}} layer ===" + - task: :infra:build + - defer: {task: down, vars: {COMPOSE_PROFILES: '{{.COMPOSE_PROFILES}}'}} + - podman-compose -f compose.yaml {{.COMPOSE_PROFILES}} up -d --build + - task: wait + vars: {PROFILE: '{{.PROFILE}}'} + - mkdir -p {{.GINKGO_OUTPUT_DIR}} + - go tool ginkgo run {{.GINKGO_FLAGS}} --output-dir={{.GINKGO_OUTPUT_DIR}} --junit-report=junit-{{.PROFILE}}.xml {{.TEST_DIR}} + + up: + desc: "Start integration stack without running tests (for IDE debugging)" + requires: + vars: [PROFILE] + vars: + COLLECTOR_CONFIG: + sh: | + case "{{.PROFILE}}" in + base) echo "configs/collector-base.yaml" ;; + storage) echo "configs/collector-storage.yaml" ;; + storage-tls) echo "configs/collector-storage-tls.yaml" ;; + enrichment) echo "configs/collector-enrichment.yaml" ;; + *) echo "INVALID_PROFILE" ;; + esac + COMPOSE_PROFILES: + sh: | + case "{{.PROFILE}}" in + base) echo "" ;; + storage) echo "--profile storage" ;; + storage-tls) echo "--profile storage-tls" ;; + enrichment) echo "--profile storage --profile enrichment" ;; + *) echo "" ;; + esac + TEST_DIR: './tests/integration/{{.PROFILE | replace "-" "_"}}/' + env: + COLLECTOR_CONFIG: '{{.COLLECTOR_CONFIG}}' + cmds: + - task: ensure-certs + - task: :infra:build + - podman-compose -f compose.yaml {{.COMPOSE_PROFILES}} up -d --build + - task: wait + vars: {PROFILE: '{{.PROFILE}}'} + - cmd: 'echo "Stack running. Run tests with: GOWORK=off go tool ginkgo run -vv {{.TEST_DIR}}"' + - cmd: 'echo "Tear down with: task integration:down"' + + wait: + internal: true + vars: + PROFILE: '{{.PROFILE | default "base"}}' + cmd: | + dump_container_info() { + local cid="$1" tail="${2:-10}" + if [[ -n "$cid" ]]; then + podman ps -a --filter "name=$cid" --format json 2>/dev/null | \ + python3 -c "import sys,json; [print(n[0],c.get('Status','')) for c in json.load(sys.stdin) for n in [c.get('Names',[])]]" 2>/dev/null || true + podman logs --tail "$tail" "$cid" 2>&1 || true + fi + } + + wait_for_service() { + local name="$1" url="$2" curl_flags="${3:-}" cid="${4:-}" + for i in $(seq 1 30); do + if curl -sf $curl_flags "$url" >/dev/null 2>&1; then + echo "$name: ready" + return 0 + fi + echo "$name: waiting... ($i/30)" + if [[ -n "$cid" ]] && (( i % 5 == 0 )); then + echo "--- $name diagnostics ---" + dump_container_info "$cid" + echo "---" + fi + sleep 2 + done + echo "$name: FAILED after 30 retries" + echo "=== $name final diagnostics ===" + dump_container_info "$cid" 30 + echo "===" + return 1 + } + + # Build service→container name map from podman-compose (project-scoped) + declare -A CONTAINERS + while read -r cname; do + for svc in collector loki rustfs rustfs-tls compass; do + if [[ "$cname" == *"_${svc}_"* ]]; then + CONTAINERS["$svc"]="$cname" + fi + done + done < <(podman-compose -f compose.yaml ps --format json 2>/dev/null | \ + python3 -c "import sys,json; [print(n) for c in json.load(sys.stdin) for n in c.get('Names',[])]" 2>/dev/null || true) + echo "Resolved containers:" + for svc in "${!CONTAINERS[@]}"; do echo " $svc -> ${CONTAINERS[$svc]}"; done + + echo "Waiting for services to be ready..." + wait_for_service "Collector webhook" "http://localhost:8088/eventreceiver/healthcheck" "" "${CONTAINERS[collector]:-}" + wait_for_service "Loki" "http://localhost:3100/ready" "" "${CONTAINERS[loki]:-}" + {{if or (eq .PROFILE "storage") (eq .PROFILE "enrichment")}} + wait_for_service "RustFS" "http://localhost:9000/health" "" "${CONTAINERS[rustfs]:-}" + {{end}} + {{if eq .PROFILE "storage-tls"}} + wait_for_service "RustFS TLS" "https://localhost:9000/health" "-k" "${CONTAINERS[rustfs-tls]:-}" + {{end}} + {{if eq .PROFILE "enrichment"}} + wait_for_service "Mock Compass" "https://localhost:8081/healthz" "-sk" "${CONTAINERS[compass]:-}" + {{end}} + echo "All services ready." + + down: + desc: Tear down integration test stack + cmd: podman-compose -f compose.yaml {{.COMPOSE_PROFILES}} down -v diff --git a/.taskfiles/quality.yml b/.taskfiles/quality.yml index 6fdefdfe..882cb33d 100644 --- a/.taskfiles/quality.yml +++ b/.taskfiles/quality.yml @@ -3,7 +3,6 @@ version: '3' vars: # Task v3 does not inherit root vars in included files; keep in sync with Taskfile.yml. MODULES: '{{.MODULES | default "./proofwatch ./truthbeam"}}' - GAZE_VERSION: '{{.GAZE_VERSION | default "latest"}}' GAZE_COVERPROFILE: '{{.GAZE_COVERPROFILE | default "coverage.out"}}' GAZE_NEW_FUNC_THRESHOLD: '{{.GAZE_NEW_FUNC_THRESHOLD | default "30"}}' @@ -12,29 +11,20 @@ tasks: silent: true cmd: task quality --list - ensure-gaze: - desc: Install gaze if not present - internal: true - cmd: command -v gaze >/dev/null 2>&1 || (echo "Installing gaze..." && go install github.com/unbound-force/gaze/cmd/gaze@{{.GAZE_VERSION}}) - run: once - crapload: desc: Run CRAP and GazeCRAP analysis (human-readable) for all modules - deps: [ensure-gaze] cmds: - task: :dev:test - bash {{.ROOT_DIR}}/.taskfiles/scripts/go-module-runner.sh crapload crapload-baseline: desc: Generate baseline thresholds in .gaze/baseline.json for all modules - deps: [ensure-gaze] cmds: - task: :dev:test - bash {{.ROOT_DIR}}/.taskfiles/scripts/go-module-runner.sh crapload-baseline crapload-check: desc: Check for CRAP regressions against baseline for all modules - deps: [ensure-gaze] cmds: - task: :dev:test - bash {{.ROOT_DIR}}/.taskfiles/scripts/go-module-runner.sh crapload-check diff --git a/.taskfiles/scripts/check-go-mod-consistency.sh b/.taskfiles/scripts/check-go-mod-consistency.sh deleted file mode 100755 index 4369c22c..00000000 --- a/.taskfiles/scripts/check-go-mod-consistency.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash -# Check that all OTel Collector dependencies within each go.mod are at consistent versions -set -euo pipefail - -MODULES="./proofwatch ./truthbeam" - -echo "Checking internal go.mod version consistency..." - -TOTAL_FAILED=0 - -for MODULE in $MODULES; do - GOMOD="$MODULE/go.mod" - - if [ ! -f "$GOMOD" ]; then - echo "WARNING: $GOMOD not found, skipping" - continue - fi - - echo "" - echo "Checking $GOMOD..." - - # OTel Collector uses dual versioning: - # - v1.x.x for stable/core APIs (component, consumer, pdata, processor, etc.) - # - v0.x.x for experimental packages (componenttest, processorhelper, xprocessor, etc.) - # - # We check consistency within each scheme separately - - # Check v0.x.x experimental packages - EXPERIMENTAL_VERSIONS=$(grep -E 'go\.opentelemetry\.io/collector/[^/]+' "$GOMOD" | \ - grep -v 'go.opentelemetry.io/contrib' | \ - grep -oE 'v0\.[0-9]+\.[0-9]+' | \ - sort -u || true) - - if [ -n "$EXPERIMENTAL_VERSIONS" ]; then - EXP_COUNT=$(echo "$EXPERIMENTAL_VERSIONS" | wc -l) - if [ "$EXP_COUNT" -eq 1 ]; then - echo " ✓ All experimental OTel packages at $(echo "$EXPERIMENTAL_VERSIONS" | head -1)" - else - echo " ERROR: Multiple experimental (v0.x) OTel Collector versions detected:" - while IFS= read -r version; do - echo " - $version" - grep -E "go\.opentelemetry\.io/collector/[^/]+.*$version" "$GOMOD" | sed 's/^/ /' - done <<< "$EXPERIMENTAL_VERSIONS" - TOTAL_FAILED=$((TOTAL_FAILED + 1)) - fi - fi - - # Check v1.x.x stable packages - STABLE_VERSIONS=$(grep -E 'go\.opentelemetry\.io/collector/[^/]+' "$GOMOD" | \ - grep -v 'go.opentelemetry.io/contrib' | \ - grep -oE 'v1\.[0-9]+\.[0-9]+' | \ - sort -u || true) - - if [ -n "$STABLE_VERSIONS" ]; then - STABLE_COUNT=$(echo "$STABLE_VERSIONS" | wc -l) - if [ "$STABLE_COUNT" -eq 1 ]; then - echo " ✓ All stable OTel packages at $(echo "$STABLE_VERSIONS" | head -1)" - else - echo " ERROR: Multiple stable (v1.x) OTel Collector versions detected:" - while IFS= read -r version; do - echo " - $version" - grep -E "go\.opentelemetry\.io/collector/[^/]+.*$version" "$GOMOD" | sed 's/^/ /' - done <<< "$STABLE_VERSIONS" - TOTAL_FAILED=$((TOTAL_FAILED + 1)) - fi - fi - - # Check contrib separately (different versioning scheme is acceptable) - CONTRIB_VERSIONS=$(grep -E 'github\.com/open-telemetry/opentelemetry-collector-contrib' "$GOMOD" | \ - grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | \ - sort -u || true) - - if [ -n "$CONTRIB_VERSIONS" ]; then - CONTRIB_COUNT=$(echo "$CONTRIB_VERSIONS" | wc -l) - if [ "$CONTRIB_COUNT" -eq 1 ]; then - echo " ✓ All OTel Collector contrib dependencies at $(echo "$CONTRIB_VERSIONS" | head -1)" - else - echo " ERROR: Multiple OTel Collector contrib versions detected:" - while IFS= read -r version; do - echo " - $version" - grep -E "github\.com/open-telemetry/opentelemetry-collector-contrib.*$version" "$GOMOD" | sed 's/^/ /' - done <<< "$CONTRIB_VERSIONS" - TOTAL_FAILED=$((TOTAL_FAILED + 1)) - fi - fi -done - -echo "" -if [ "$TOTAL_FAILED" -gt 0 ]; then - echo "ERROR: Found version inconsistencies in $TOTAL_FAILED module(s)" - echo "All OTel Collector dependencies within a go.mod should be at the same version." - exit 1 -fi - -echo "✓ All go.mod files have consistent OTel Collector versions" diff --git a/.taskfiles/scripts/check-go-version.sh b/.taskfiles/scripts/check-go-version.sh deleted file mode 100755 index 836b00e5..00000000 --- a/.taskfiles/scripts/check-go-version.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# Check that Containerfile Go version satisfies all module requirements -set -euo pipefail - -CONTAINERFILE="beacon-distro/Containerfile.collector" -MODULES="./proofwatch ./truthbeam" - -# Extract Go version from Containerfile -CF_VERSION=$(sed -n 's/^FROM golang:\([0-9]*\.[0-9]*\.[0-9]*\).*/\1/p' "$CONTAINERFILE" | head -1) -if [ -z "$CF_VERSION" ]; then - echo "ERROR: Could not extract Go version from $CONTAINERFILE" - exit 1 -fi - -CF_MAJOR=$(echo "$CF_VERSION" | cut -d. -f1) -CF_MINOR=$(echo "$CF_VERSION" | cut -d. -f2) -echo "Containerfile Go version: $CF_VERSION (major=$CF_MAJOR minor=$CF_MINOR)" - -# Check each module's go.mod -FAILED=0 -for m in $MODULES; do - MOD_VERSION=$(sed -n 's/^go \([0-9]*\.[0-9]*\).*/\1/p' "$m/go.mod" | head -1) - if [ -z "$MOD_VERSION" ]; then - echo "WARNING: Could not extract go directive from $m/go.mod" - continue - fi - - MOD_MAJOR=$(echo "$MOD_VERSION" | cut -d. -f1) - MOD_MINOR=$(echo "$MOD_VERSION" | cut -d. -f2) - - if [ "$CF_MAJOR" -lt "$MOD_MAJOR" ] || \ - { [ "$CF_MAJOR" -eq "$MOD_MAJOR" ] && [ "$CF_MINOR" -lt "$MOD_MINOR" ]; }; then - echo "FAIL: $m/go.mod requires go $MOD_VERSION but $CONTAINERFILE uses $CF_VERSION" - FAILED=1 - else - echo "OK: $m/go.mod requires go $MOD_VERSION <= $CF_VERSION" - fi -done - -if [ "$FAILED" -ne 0 ]; then - echo "" - echo "ERROR: Containerfile Go version is behind module requirements." - echo "Update the FROM golang:X.Y.Z line in $CONTAINERFILE." - exit 1 -fi - -echo "--- Go version check passed ---" diff --git a/.taskfiles/scripts/check-otel-versions.sh b/.taskfiles/scripts/check-otel-versions.sh deleted file mode 100755 index 1883d6f1..00000000 --- a/.taskfiles/scripts/check-otel-versions.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# Check that beacon-distro/manifest.yaml OTel versions align with truthbeam/go.mod -set -euo pipefail - -MANIFEST="beacon-distro/manifest.yaml" -TRUTHBEAM_GOMOD="truthbeam/go.mod" - -echo "Checking OTel Collector version consistency..." - -# Extract collector version from truthbeam go.mod (using processorhelper as reference) -TRUTHBEAM_VERSION=$(grep 'go.opentelemetry.io/collector/processor/processorhelper' "$TRUTHBEAM_GOMOD" | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1) -if [ -z "$TRUTHBEAM_VERSION" ]; then - echo "ERROR: Could not extract OTel Collector version from $TRUTHBEAM_GOMOD" - exit 1 -fi - -echo "truthbeam requires: $TRUTHBEAM_VERSION" - -# Check manifest.yaml components -FAILED=0 -while IFS= read -r line; do - COMPONENT=$(echo "$line" | grep -oE 'go\.opentelemetry\.io/collector/[^[:space:]]+' || true) - VERSION=$(echo "$line" | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' || true) - - if [ -n "$COMPONENT" ] && [ -n "$VERSION" ]; then - if [ "$VERSION" != "$TRUTHBEAM_VERSION" ]; then - echo "MISMATCH: $COMPONENT is at $VERSION (expected $TRUTHBEAM_VERSION)" - FAILED=1 - fi - fi -done < <(grep -E 'go.opentelemetry.io/collector/(exporter|processor|receiver)' "$MANIFEST") - -# Check builder version in Containerfile -BUILDER_VERSION=$(grep 'go.opentelemetry.io/collector/cmd/builder@' beacon-distro/Containerfile.collector | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+') -if [ "$BUILDER_VERSION" != "$TRUTHBEAM_VERSION" ]; then - echo "MISMATCH: Builder is at $BUILDER_VERSION (expected $TRUTHBEAM_VERSION)" - FAILED=1 -fi - -if [ "$FAILED" -ne 0 ]; then - echo "" - echo "ERROR: OTel Collector version mismatch detected!" - echo "Update manifest.yaml and Containerfile.collector to use $TRUTHBEAM_VERSION" - exit 1 -fi - -echo "✓ All OTel Collector versions aligned at $TRUTHBEAM_VERSION" diff --git a/.taskfiles/scripts/go-module-runner.sh b/.taskfiles/scripts/go-module-runner.sh index bafc2c26..c621980d 100755 --- a/.taskfiles/scripts/go-module-runner.sh +++ b/.taskfiles/scripts/go-module-runner.sh @@ -53,7 +53,7 @@ case "$1" in lint) for m in $MODULES; do echo "Running golangci-lint for $m..." - (cd "$m" && golangci-lint run --config ../.golangci.yml ./...) || { echo "Linting failed for module: $m"; exit 1; } + (cd "$m" && GOWORK=off golangci-lint run --config ../.golangci.yml ./...) || { echo "Linting failed for module: $m"; exit 1; } done echo "--- All linting passed! ---" ;; @@ -69,7 +69,7 @@ case "$1" in echo "=========================================================================================================" echo "CRAP analysis for $m..." echo "=========================================================================================================" - (cd "$m" && gaze crap --format=text --coverprofile="$GAZE_COVERPROFILE" ./...) + (cd "$m" && go tool gaze crap --format=text --coverprofile="$GAZE_COVERPROFILE" ./...) done ;; @@ -78,7 +78,7 @@ case "$1" in echo "Generating baseline for $m..." mkdir -p "$m/.gaze" MODULE_ROOT=$(cd "$m" && pwd) - (cd "$m" && gaze crap --format=json --coverprofile="$GAZE_COVERPROFILE" ./... 2>/dev/null | \ + (cd "$m" && go tool gaze crap --format=json --coverprofile="$GAZE_COVERPROFILE" ./... 2>/dev/null | \ jq --arg root "$MODULE_ROOT/" '(.scores[],.summary.worst_crap[]?,.summary.worst_gaze_crap[]?) |= (.file |= ltrimstr($root))' > .gaze/baseline.json) echo "Baseline written to $m/.gaze/baseline.json" done @@ -91,44 +91,44 @@ case "$1" in echo "Checking CRAP regressions for $m..." echo "=========================================================================================================" BASELINE="$m/.gaze/baseline.json" - if [ ! -f "$BASELINE" ]; then + if [[ ! -f "$BASELINE" ]]; then echo "ERROR: Baseline file $BASELINE not found. Run 'task quality:crapload-baseline' first." exit 1 fi MODULE_ROOT=$(cd "$m" && pwd) - (cd "$m" && gaze crap --format=json --coverprofile="$GAZE_COVERPROFILE" ./... 2>/dev/null | \ + (cd "$m" && go tool gaze crap --format=json --coverprofile="$GAZE_COVERPROFILE" ./... 2>/dev/null | \ jq --arg root "$MODULE_ROOT/" '(.scores[],.summary.worst_crap[]?,.summary.worst_gaze_crap[]?) |= (.file |= ltrimstr($root))' > /tmp/crapload-current.json) echo "Comparing against baseline..." jq -r '.scores[] | "\(.file):\(.function)\t\(.crap)\t\(.gaze_crap // 0)"' "$BASELINE" | sort > /tmp/crapload-baseline.tsv REGRESSIONS=0 while IFS=$'\t' read -r func crap gaze_crap; do baseline_line=$(grep -F "$func " /tmp/crapload-baseline.tsv | head -1 || true) - if [ -z "$baseline_line" ]; then - if [ "$(echo "$crap > $GAZE_NEW_FUNC_THRESHOLD" | bc -l)" = "1" ]; then + if [[ -z "$baseline_line" ]]; then + if [[ "$(echo "$crap > $GAZE_NEW_FUNC_THRESHOLD" | bc -l || true)" = "1" ]]; then echo "NEW FUNCTION VIOLATION: $func CRAP=$crap (threshold=$GAZE_NEW_FUNC_THRESHOLD)" REGRESSIONS=$((REGRESSIONS + 1)) fi else b_crap=$(echo "$baseline_line" | cut -f2) b_gaze=$(echo "$baseline_line" | cut -f3) - if [ "$(echo "$crap > $b_crap" | bc -l)" = "1" ]; then + if [[ "$(echo "$crap > $b_crap" | bc -l || true)" = "1" ]]; then echo "REGRESSION: $func CRAP $b_crap -> $crap" REGRESSIONS=$((REGRESSIONS + 1)) fi - if [ "$(echo "$gaze_crap > $b_gaze" | bc -l)" = "1" ]; then + if [[ "$(echo "$gaze_crap > $b_gaze" | bc -l || true)" = "1" ]]; then echo "REGRESSION: $func GazeCRAP $b_gaze -> $gaze_crap" REGRESSIONS=$((REGRESSIONS + 1)) fi fi - done < <(jq -r '.scores[] | "\(.file):\(.function)\t\(.crap)\t\(.gaze_crap // 0)"' /tmp/crapload-current.json | sort) + done < <(jq -r '.scores[] | "\(.file):\(.function)\t\(.crap)\t\(.gaze_crap // 0)"' /tmp/crapload-current.json | sort || true) TOTAL_REGRESSIONS=$((TOTAL_REGRESSIONS + REGRESSIONS)) - if [ $REGRESSIONS -gt 0 ]; then + if [[ "$REGRESSIONS" -gt 0 ]]; then echo "$m: $REGRESSIONS regression(s) detected" else echo "$m: No regressions detected" fi done - if [ $TOTAL_REGRESSIONS -gt 0 ]; then + if [[ "$TOTAL_REGRESSIONS" -gt 0 ]]; then echo "FAIL: $TOTAL_REGRESSIONS total regression(s) detected" exit 1 else diff --git a/.taskfiles/scripts/sync-all-otel-versions.sh b/.taskfiles/scripts/sync-all-otel-versions.sh deleted file mode 100755 index 31e342c1..00000000 --- a/.taskfiles/scripts/sync-all-otel-versions.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/bash -# Auto-sync all OTel Collector versions to the highest version found across all modules -set -euo pipefail - -MODULES="./proofwatch ./truthbeam" -MANIFEST="beacon-distro/manifest.yaml" -CONTAINERFILE="beacon-distro/Containerfile.collector" - -echo "Finding highest OTel Collector versions across all modules..." - -# Find highest v0.x.x experimental version across all go.mod files -HIGHEST_EXPERIMENTAL="" -for MODULE in $MODULES; do - GOMOD="$MODULE/go.mod" - if [ ! -f "$GOMOD" ]; then - continue - fi - - VERSIONS=$(grep -E 'go\.opentelemetry\.io/collector/[^/]+' "$GOMOD" | \ - grep -oE 'v0\.[0-9]+\.[0-9]+' | \ - sort -V -u || true) - - for VERSION in $VERSIONS; do - if [ -z "$HIGHEST_EXPERIMENTAL" ] || [ "$(printf '%s\n' "$HIGHEST_EXPERIMENTAL" "$VERSION" | sort -V | tail -1)" = "$VERSION" ]; then - HIGHEST_EXPERIMENTAL="$VERSION" - fi - done -done - -# Find highest v1.x.x stable version across all go.mod files -HIGHEST_STABLE="" -for MODULE in $MODULES; do - GOMOD="$MODULE/go.mod" - if [ ! -f "$GOMOD" ]; then - continue - fi - - VERSIONS=$(grep -E 'go\.opentelemetry\.io/collector/[^/]+' "$GOMOD" | \ - grep -oE 'v1\.[0-9]+\.[0-9]+' | \ - sort -V -u || true) - - for VERSION in $VERSIONS; do - if [ -z "$HIGHEST_STABLE" ] || [ "$(printf '%s\n' "$HIGHEST_STABLE" "$VERSION" | sort -V | tail -1)" = "$VERSION" ]; then - HIGHEST_STABLE="$VERSION" - fi - done -done - -if [ -z "$HIGHEST_EXPERIMENTAL" ]; then - echo "ERROR: Could not find any experimental (v0.x) OTel Collector versions" - exit 1 -fi - -if [ -z "$HIGHEST_STABLE" ]; then - echo "ERROR: Could not find any stable (v1.x) OTel Collector versions" - exit 1 -fi - -echo " Highest experimental (v0.x): $HIGHEST_EXPERIMENTAL" -echo " Highest stable (v1.x): $HIGHEST_STABLE" -echo "" - -# Update each go.mod file -for MODULE in $MODULES; do - GOMOD="$MODULE/go.mod" - if [ ! -f "$GOMOD" ]; then - continue - fi - - echo "Updating $GOMOD..." - - # Update experimental packages to highest experimental version - perl -i -pe "s{(go\.opentelemetry\.io/collector/[^/]+) v0\.[0-9]+\.[0-9]+}{\$1 $HIGHEST_EXPERIMENTAL}g" "$GOMOD" - - # Update stable packages to highest stable version - perl -i -pe "s{(go\.opentelemetry\.io/collector/[^/]+) v1\.[0-9]+\.[0-9]+}{\$1 $HIGHEST_STABLE}g" "$GOMOD" - - # Run go mod tidy to ensure dependencies are resolved - echo " Running go mod tidy..." - (cd "$MODULE" && GOWORK=off go mod tidy) - - echo " ✓ $MODULE updated to experimental=$HIGHEST_EXPERIMENTAL stable=$HIGHEST_STABLE" -done - -echo "" -echo "Updating manifest and Containerfile..." - -# Update manifest.yaml -perl -i -pe "s{(go\.opentelemetry\.io/collector/(exporter|processor|receiver)/\w+) v[\d.]+}{\$1 $HIGHEST_EXPERIMENTAL}g" "$MANIFEST" -perl -i -pe "s{(github\.com/open-telemetry/opentelemetry-collector-contrib/(exporter|processor|receiver|connector|extension)/\w+) v[\d.]+}{\$1 $HIGHEST_EXPERIMENTAL}g" "$MANIFEST" -perl -i -pe "s{(go\.opentelemetry\.io/collector/confmap/provider/\w+) v[\d.]+}{\$1 $HIGHEST_STABLE}g" "$MANIFEST" - -# Update Containerfile builder version -perl -i -pe "s{builder\@v[\d.]+}{builder\@$HIGHEST_EXPERIMENTAL}g" "$CONTAINERFILE" -perl -i -pe "s{builder version \(v[\d.]+\)}{builder version ($HIGHEST_EXPERIMENTAL)}g" "$CONTAINERFILE" - -echo " ✓ manifest.yaml and Containerfile.collector updated" -echo "" -echo "✓ All OTel Collector versions synced to experimental=$HIGHEST_EXPERIMENTAL stable=$HIGHEST_STABLE" diff --git a/.taskfiles/scripts/sync-manifest-versions.sh b/.taskfiles/scripts/sync-manifest-versions.sh deleted file mode 100755 index 58302245..00000000 --- a/.taskfiles/scripts/sync-manifest-versions.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -# Sync manifest.yaml versions from truthbeam/go.mod (the idiomatic way) -# This keeps manifest.yaml in sync with Go module versions -set -euo pipefail - -TRUTHBEAM_GOMOD="truthbeam/go.mod" -MANIFEST="beacon-distro/manifest.yaml" - -echo "Syncing OTel Collector versions from truthbeam to manifest..." - -# Extract OTel Collector version from truthbeam (using processorhelper as reference) -OTEL_VERSION=$(grep 'go.opentelemetry.io/collector/processor/processorhelper' "$TRUTHBEAM_GOMOD" | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1) -if [ -z "$OTEL_VERSION" ]; then - echo "ERROR: Could not extract OTel Collector version from $TRUTHBEAM_GOMOD" - exit 1 -fi - -# Extract provider version (different version scheme) -PROVIDER_VERSION=$(grep 'go.opentelemetry.io/collector/component v' "$TRUTHBEAM_GOMOD" | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1) -if [ -z "$PROVIDER_VERSION" ]; then - echo "ERROR: Could not extract provider version from $TRUTHBEAM_GOMOD" - exit 1 -fi - -echo " OTel Collector: $OTEL_VERSION" -echo " Providers: $PROVIDER_VERSION" - -# Update versions using perl (more reliable than sed for complex patterns) -perl -i -pe "s{(go\.opentelemetry\.io/collector/(exporter|processor|receiver)/\w+) v[\d.]+}{\$1 $OTEL_VERSION}g" "$MANIFEST" -perl -i -pe "s{(github\.com/open-telemetry/opentelemetry-collector-contrib/(exporter|processor|receiver|connector|extension)/\w+) v[\d.]+}{\$1 $OTEL_VERSION}g" "$MANIFEST" -perl -i -pe "s{(go\.opentelemetry\.io/collector/confmap/provider/\w+) v[\d.]+}{\$1 $PROVIDER_VERSION}g" "$MANIFEST" - -# Update builder version in Containerfile -perl -i -pe "s{builder\@v[\d.]+}{builder\@$OTEL_VERSION}g" beacon-distro/Containerfile.collector -perl -i -pe "s{builder version \(v[\d.]+\)}{builder version ($OTEL_VERSION)}g" beacon-distro/Containerfile.collector - -echo "✓ Synced all versions to OTel $OTEL_VERSION / Providers $PROVIDER_VERSION" diff --git a/.taskfiles/scripts/version-check.sh b/.taskfiles/scripts/version-check.sh new file mode 100755 index 00000000..a4c7342f --- /dev/null +++ b/.taskfiles/scripts/version-check.sh @@ -0,0 +1,177 @@ +#!/bin/bash +# Read-only version validation: checks that Go and OTel versions are +# consistent across all modules, Containerfiles, and manifest.yaml. +# Exit 0 = all aligned, exit 1 = drift detected. +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)" +cd "$ROOT_DIR" + +FAILED=0 + +GO_WORK="go.work" +TRUTHBEAM_GOMOD="truthbeam/go.mod" +MANIFEST="beacon-distro/manifest.yaml" + +# Auto-discover go.mod files and Containerfiles with Go base images +mapfile -t GO_MODS < <(find . -name go.mod -not -path '*/vendor/*' -print | sort || true) +mapfile -t CONTAINERFILES < <(grep -rl '^FROM golang:' . --include='Containerfile*' --include='Dockerfile*' 2>/dev/null | sort || true) + +# Workspace modules (from go.work use block) — these carry OTel deps +mapfile -t WORKSPACE_MODULES < <(sed -n '/^use (/,/^)/{ s/^[[:space:]]*\.\///p }' "$GO_WORK" || true) + +# ── Go version alignment ──────────────────────────────────────── +echo "=== Go version check ===" + +GO_VERSION=$(sed -n 's/^go \([0-9]*\.[0-9]*\.[0-9]*\)/\1/p' "$GO_WORK" | head -1) +if [[ -z "$GO_VERSION" ]]; then + echo "ERROR: Could not extract Go version from $GO_WORK" + exit 1 +fi +echo " Source of truth ($GO_WORK): $GO_VERSION" + +for GOMOD in "${GO_MODS[@]}"; do + MOD_VERSION=$(sed -n 's/^go \([0-9]*\.[0-9]*\.[0-9]*\)/\1/p' "$GOMOD" | head -1) + if [[ -z "$MOD_VERSION" ]]; then + MOD_VERSION=$(sed -n 's/^go \([0-9]*\.[0-9]*\)/\1/p' "$GOMOD" | head -1) + fi + if [[ "$MOD_VERSION" != "$GO_VERSION" ]]; then + echo " FAIL: $GOMOD has go $MOD_VERSION (expected $GO_VERSION)" + FAILED=1 + else + echo " OK: $GOMOD" + fi +done + +for CF in "${CONTAINERFILES[@]}"; do + CF_VERSION=$(sed -n 's/^FROM golang:\([0-9]*\.[0-9]*\.[0-9]*\).*/\1/p' "$CF" | head -1) + if [[ -z "$CF_VERSION" ]]; then + echo " WARNING: Could not extract Go version from $CF" + continue + fi + if [[ "$CF_VERSION" != "$GO_VERSION" ]]; then + echo " FAIL: $CF uses golang:$CF_VERSION (expected $GO_VERSION)" + FAILED=1 + else + echo " OK: $CF" + fi +done + +for GOMOD in "${GO_MODS[@]}"; do + if grep -q '^toolchain go' "$GOMOD"; then + echo " FAIL: $GOMOD has stale toolchain directive" + FAILED=1 + fi +done + +echo "" + +# ── OTel version consistency ──────────────────────────────────── +echo "=== OTel version check ===" + +OTEL_EXPERIMENTAL=$(grep -E 'go\.opentelemetry\.io/collector/[^/]+' "$TRUTHBEAM_GOMOD" | \ + grep -v 'go.opentelemetry.io/contrib' | \ + grep -oE 'v0\.[0-9]+\.[0-9]+' | \ + sort -V -u | tail -1) + +OTEL_STABLE=$(grep -E 'go\.opentelemetry\.io/collector/[^/]+' "$TRUTHBEAM_GOMOD" | \ + grep -v 'go.opentelemetry.io/contrib' | \ + grep -oE 'v1\.[0-9]+\.[0-9]+' | \ + sort -V -u | tail -1) + +echo " Source of truth ($TRUTHBEAM_GOMOD): experimental=$OTEL_EXPERIMENTAL stable=$OTEL_STABLE" + +for MODULE in "${WORKSPACE_MODULES[@]}"; do + GOMOD="$MODULE/go.mod" + if [[ ! -f "$GOMOD" ]]; then + continue + fi + + EXP_VERSIONS=$(grep -E 'go\.opentelemetry\.io/collector/[^/]+' "$GOMOD" | \ + grep -v 'go.opentelemetry.io/contrib' | \ + grep -oE 'v0\.[0-9]+\.[0-9]+' | \ + sort -u || true) + + if [[ -n "$EXP_VERSIONS" ]]; then + EXP_COUNT=$(echo "$EXP_VERSIONS" | wc -l) + if [[ "$EXP_COUNT" -gt 1 ]]; then + echo " FAIL: $GOMOD has mixed experimental OTel versions:" + echo "${EXP_VERSIONS//$'\n'/$'\n' }" + FAILED=1 + else + FIRST_EXP=$(echo "$EXP_VERSIONS" | head -1 || true) + echo " OK: $GOMOD experimental at $FIRST_EXP" + fi + fi + + STABLE_VERSIONS=$(grep -E 'go\.opentelemetry\.io/collector/[^/]+' "$GOMOD" | \ + grep -v 'go.opentelemetry.io/contrib' | \ + grep -oE 'v1\.[0-9]+\.[0-9]+' | \ + sort -u || true) + + if [[ -n "$STABLE_VERSIONS" ]]; then + STABLE_COUNT=$(echo "$STABLE_VERSIONS" | wc -l) + if [[ "$STABLE_COUNT" -gt 1 ]]; then + echo " FAIL: $GOMOD has mixed stable OTel versions:" + echo "${STABLE_VERSIONS//$'\n'/$'\n' }" + FAILED=1 + else + FIRST_STABLE=$(echo "$STABLE_VERSIONS" | head -1 || true) + echo " OK: $GOMOD stable at $FIRST_STABLE" + fi + fi +done + +MANIFEST_VERSIONS=$(grep -E 'go\.opentelemetry\.io/collector/(exporter|processor|receiver)' "$MANIFEST" | \ + grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | sort -u || true) + +for V in $MANIFEST_VERSIONS; do + if [[ "$V" != "$OTEL_EXPERIMENTAL" ]]; then + echo " FAIL: $MANIFEST has component at $V (expected $OTEL_EXPERIMENTAL)" + FAILED=1 + fi +done + +PROVIDER_VERSIONS=$(grep -E 'go\.opentelemetry\.io/collector/confmap/provider' "$MANIFEST" | \ + grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | sort -u || true) + +for V in $PROVIDER_VERSIONS; do + if [[ "$V" != "$OTEL_STABLE" ]]; then + echo " FAIL: $MANIFEST has provider at $V (expected $OTEL_STABLE)" + FAILED=1 + fi +done + +CONTRIB_VERSIONS=$(grep -E 'github\.com/open-telemetry/opentelemetry-collector-contrib' "$MANIFEST" | \ + grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | sort -u || true) + +for V in $CONTRIB_VERSIONS; do + if [[ "$V" != "$OTEL_EXPERIMENTAL" ]]; then + echo " FAIL: $MANIFEST has contrib at $V (expected $OTEL_EXPERIMENTAL)" + FAILED=1 + fi +done + +COLLECTOR_CF="beacon-distro/Containerfile.collector" +if [[ -f "$COLLECTOR_CF" ]]; then + BUILDER_VERSION=$(grep 'go.opentelemetry.io/collector/cmd/builder@' "$COLLECTOR_CF" | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' || true) + if [[ -n "$BUILDER_VERSION" && "$BUILDER_VERSION" != "$OTEL_EXPERIMENTAL" ]]; then + echo " FAIL: Builder at $BUILDER_VERSION (expected $OTEL_EXPERIMENTAL)" + FAILED=1 + elif [[ -n "$BUILDER_VERSION" ]]; then + echo " OK: Builder at $BUILDER_VERSION" + fi +fi + +if [[ -z "$MANIFEST_VERSIONS" ]] || echo "$MANIFEST_VERSIONS" | grep -q "^${OTEL_EXPERIMENTAL}$"; then + echo " OK: $MANIFEST components aligned" +fi + +echo "" + +if [[ "$FAILED" -ne 0 ]]; then + echo "FAILED: Version drift detected. Run 'task version:sync' to fix." + exit 1 +fi + +echo "All version checks passed." diff --git a/.taskfiles/scripts/version-sync.sh b/.taskfiles/scripts/version-sync.sh new file mode 100755 index 00000000..1197bed0 --- /dev/null +++ b/.taskfiles/scripts/version-sync.sh @@ -0,0 +1,137 @@ +#!/bin/bash +# Unified version sync: aligns Go and OTel versions across all modules, +# Containerfiles, CI workflows, and documentation. +# +# Source of truth: +# Go version — go.work `go` directive +# OTel version — truthbeam/go.mod (experimental v0.x and stable v1.x) +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)" +cd "$ROOT_DIR" + +GO_WORK="go.work" +TRUTHBEAM_GOMOD="truthbeam/go.mod" +MANIFEST="beacon-distro/manifest.yaml" + +# Auto-discover go.mod files and Containerfiles with Go base images +mapfile -t GO_MODS < <(find . -name go.mod -not -path '*/vendor/*' -print | sort || true) +mapfile -t CONTAINERFILES < <(grep -rl '^FROM golang:' . --include='Containerfile*' --include='Dockerfile*' 2>/dev/null | sort || true) + +# Workspace modules (from go.work use block) — these carry OTel deps +mapfile -t WORKSPACE_MODULES < <(sed -n '/^use (/,/^)/{ s/^[[:space:]]*\.\///p }' "$GO_WORK" || true) + +# ── Extract Go version from go.work ────────────────────────────── +GO_VERSION=$(sed -n 's/^go \([0-9]*\.[0-9]*\.[0-9]*\)/\1/p' "$GO_WORK" | head -1) +if [[ -z "$GO_VERSION" ]]; then + echo "ERROR: Could not extract Go version from $GO_WORK" + exit 1 +fi +GO_MINOR="${GO_VERSION%.*}" +echo "=== Go version sync (source: $GO_WORK) ===" +echo " Target: $GO_VERSION (minor: $GO_MINOR)" + +# ── Sync go.mod files ──────────────────────────────────────────── +for GOMOD in "${GO_MODS[@]}"; do + perl -i -pe "s/^go \d+\.\d+(\.\d+)?$/go $GO_VERSION/" "$GOMOD" + perl -i -ne 'print unless /^toolchain go/' "$GOMOD" + echo " go.mod: $GOMOD" +done + +# ── Sync Containerfile Go image tags ───────────────────────────── +for CF in "${CONTAINERFILES[@]}"; do + perl -i -pe "s{FROM golang:\d+\.\d+\.\d+}{FROM golang:$GO_VERSION}" "$CF" + echo " Containerfile: $CF" +done + +# ── Sync CI workflow GO_VERSION ────────────────────────────────── +CI_LOCAL=".github/workflows/ci_local.yml" +if [[ -f "$CI_LOCAL" ]]; then + perl -i -pe "s{^(\s*GO_VERSION:\s*)\S+}{\${1}$GO_MINOR}" "$CI_LOCAL" + echo " CI workflow: $CI_LOCAL (GO_VERSION: $GO_MINOR)" +fi + +# ── Sync documentation ────────────────────────────────────────── +DOCS=( + "docs/DEVELOPMENT.md" + "README.md" + "AGENTS.md" +) +for DOC in "${DOCS[@]}"; do + if [[ ! -f "$DOC" ]]; then + continue + fi + perl -i -pe "s{Go \d+\.\d+\+}{Go ${GO_MINOR}+}g" "$DOC" + perl -i -pe "s{Go \d+\.\d+\.\d+}{Go $GO_VERSION}g" "$DOC" + perl -i -pe "s{golang:\d+\.\d+\.\d+}{golang:$GO_VERSION}g" "$DOC" + echo " Doc: $DOC" +done + +echo "" + +# ── Extract OTel versions from truthbeam ───────────────────────── +echo "=== OTel version sync (source: $TRUTHBEAM_GOMOD) ===" + +OTEL_EXPERIMENTAL=$(grep -E 'go\.opentelemetry\.io/collector/[^/]+' "$TRUTHBEAM_GOMOD" | \ + grep -v 'go.opentelemetry.io/contrib' | \ + grep -oE 'v0\.[0-9]+\.[0-9]+' | \ + sort -V -u | tail -1) + +OTEL_STABLE=$(grep -E 'go\.opentelemetry\.io/collector/[^/]+' "$TRUTHBEAM_GOMOD" | \ + grep -v 'go.opentelemetry.io/contrib' | \ + grep -oE 'v1\.[0-9]+\.[0-9]+' | \ + sort -V -u | tail -1) + +if [[ -z "$OTEL_EXPERIMENTAL" ]]; then + echo "ERROR: Could not extract experimental (v0.x) OTel version from $TRUTHBEAM_GOMOD" + exit 1 +fi +if [[ -z "$OTEL_STABLE" ]]; then + echo "ERROR: Could not extract stable (v1.x) OTel version from $TRUTHBEAM_GOMOD" + exit 1 +fi + +echo " Experimental: $OTEL_EXPERIMENTAL" +echo " Stable: $OTEL_STABLE" + +# ── Sync OTel versions in workspace module go.mod files ────────── +for MODULE in "${WORKSPACE_MODULES[@]}"; do + GOMOD="$MODULE/go.mod" + if [[ ! -f "$GOMOD" ]]; then + continue + fi + perl -i -pe "s{(go\.opentelemetry\.io/collector/[^/\s]+)\s+v0\.\d+\.\d+}{\$1 $OTEL_EXPERIMENTAL}g" "$GOMOD" + perl -i -pe "s{(go\.opentelemetry\.io/collector/[^/\s]+)\s+v1\.\d+\.\d+}{\$1 $OTEL_STABLE}g" "$GOMOD" + echo " go.mod: $GOMOD" +done + +# ── Sync manifest.yaml ────────────────────────────────────────── +perl -i -pe "s{(go\.opentelemetry\.io/collector/(exporter|processor|receiver)/\w+) v[\d.]+}{\$1 $OTEL_EXPERIMENTAL}g" "$MANIFEST" +perl -i -pe "s{(github\.com/open-telemetry/opentelemetry-collector-contrib/(exporter|processor|receiver|connector|extension)/\w+) v[\d.]+}{\$1 $OTEL_EXPERIMENTAL}g" "$MANIFEST" +perl -i -pe "s{(go\.opentelemetry\.io/collector/confmap/provider/\w+) v[\d.]+}{\$1 $OTEL_STABLE}g" "$MANIFEST" +echo " Manifest: $MANIFEST" + +# ── Sync Containerfile builder version ─────────────────────────── +COLLECTOR_CF="beacon-distro/Containerfile.collector" +if [[ -f "$COLLECTOR_CF" ]]; then + perl -i -pe "s{builder\@v[\d.]+}{builder\@$OTEL_EXPERIMENTAL}g" "$COLLECTOR_CF" + echo " Builder: $COLLECTOR_CF" +fi + +echo "" + +# ── Run go mod tidy on all discovered modules ──────────────────── +echo "=== Running go mod tidy ===" +for GOMOD in "${GO_MODS[@]}"; do + MODULE_DIR=$(dirname "$GOMOD") + echo " Tidying $MODULE_DIR..." + (cd "$MODULE_DIR" && go mod tidy 2>&1 | sed 's/^/ /') +done + +echo "" +echo "=== Version sync complete ===" +echo " Go: $GO_VERSION | OTel: experimental=$OTEL_EXPERIMENTAL stable=$OTEL_STABLE" +echo "" +echo "NOTE: Containerfile @sha256: digests are NOT updated automatically." +echo " To update digests, pull the new image and replace the hash:" +echo " podman pull golang:$GO_VERSION && podman inspect golang:$GO_VERSION --format '{{.Digest}}'" diff --git a/.taskfiles/tools.yml b/.taskfiles/tools.yml new file mode 100644 index 00000000..da731d0b --- /dev/null +++ b/.taskfiles/tools.yml @@ -0,0 +1,31 @@ +version: '3' + +# External tool installation with integrity verification +# Uses .tool_checksums file for SHA256 verification + +vars: + TOOLS_DIR: '{{.TOOLS_DIR | default "/usr/local/bin"}}' + CHECKSUMS_FILE: '{{.ROOT_DIR}}/.tool_checksums' + +tasks: + install-weaver: + desc: "Install weaver with SHA256 verification" + vars: + WEAVER_URL: "https://github.com/open-telemetry/weaver/releases/download/{{.WEAVER_VERSION}}/weaver-x86_64-unknown-linux-gnu.tar.xz" + cmds: + - curl -fL -o /tmp/weaver.tar.xz "{{.WEAVER_URL}}" + - | + WEAVER_SHA=$(grep "weaver/{{.WEAVER_VERSION}}/linux/x86_64" {{.CHECKSUMS_FILE}} | cut -d: -f2 | tr -d ' ') + echo "${WEAVER_SHA} /tmp/weaver.tar.xz" | sha256sum -c - + - tar -xJf /tmp/weaver.tar.xz -C /tmp + - install /tmp/weaver-x86_64-unknown-linux-gnu/weaver {{.TOOLS_DIR}}/weaver + - rm -rf /tmp/weaver.tar.xz /tmp/weaver-x86_64-unknown-linux-gnu/ + - weaver --version + status: + - test -x {{.TOOLS_DIR}}/weaver + - weaver --version | grep -q "{{.WEAVER_VERSION}}" + + install-all: + desc: "Install all external tools" + cmds: + - task: install-weaver diff --git a/.taskfiles/version.yml b/.taskfiles/version.yml index c6c5ef93..066153f7 100644 --- a/.taskfiles/version.yml +++ b/.taskfiles/version.yml @@ -1,26 +1,14 @@ version: '3' tasks: - default: - silent: true - cmd: task version --list - - check-go-version: - desc: Check that Containerfile Go version satisfies all module requirements - cmd: bash {{.ROOT_DIR}}/.taskfiles/scripts/check-go-version.sh - - check-otel-versions: - desc: Check that manifest.yaml OTel versions align with truthbeam - cmd: bash {{.ROOT_DIR}}/.taskfiles/scripts/check-otel-versions.sh - - check-go-mod-consistency: - desc: Check that OTel dependencies within each go.mod are consistent - cmd: bash {{.ROOT_DIR}}/.taskfiles/scripts/check-go-mod-consistency.sh - - sync-otel-versions: - desc: Sync manifest.yaml OTel versions from truthbeam (idiomatic Go way) - cmd: bash {{.ROOT_DIR}}/.taskfiles/scripts/sync-manifest-versions.sh - - sync-all-otel-versions: - desc: Sync all OTel versions to highest found across all modules - cmd: bash {{.ROOT_DIR}}/.taskfiles/scripts/sync-all-otel-versions.sh + sync: + desc: Sync Go and OTel versions across all modules, Containerfiles, CI, and docs + deps: + - task: :dev:workspace + cmd: bash {{.ROOT_DIR}}/.taskfiles/scripts/version-sync.sh + + check: + desc: Validate version alignment (read-only, for CI) + deps: + - task: :dev:workspace + cmd: bash {{.ROOT_DIR}}/.taskfiles/scripts/version-check.sh diff --git a/.tool_checksums b/.tool_checksums new file mode 100644 index 00000000..c0fd235b --- /dev/null +++ b/.tool_checksums @@ -0,0 +1,8 @@ +# External Tool Checksums +# Format: ///: + +# Weaver v0.21.2 +weaver/v0.21.2/linux/x86_64/weaver-x86_64-unknown-linux-gnu.tar.xz: b219f2de44ea3667c28356cd4860084983d146b694342a4ee52f349b38e7389f + +# Add new tools here following the same pattern +# Format: tool_name/version/platform/architecture/filename: sha256_hash diff --git a/AGENTS.md b/AGENTS.md index cb450aa5..35dd5869 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,350 +1,103 @@ # ComplyBeacon -## Project Overview +Open-source observability toolkit that collects, normalizes, and enriches compliance evidence by extending the OpenTelemetry standard. Uses a Go workspace monorepo with two active modules (`proofwatch`, `truthbeam`) and an OTel Collector distribution (`beacon-distro`). -ComplyBeacon is an open-source observability toolkit that collects, -normalizes, and enriches compliance evidence by extending the -OpenTelemetry (OTEL) standard. It bridges raw policy scanner output -and modern logging pipelines into a unified, auditable data stream. - -- **Type**: Go monorepo (multi-module workspace) -- **License**: Apache-2.0 -- **Go version**: 1.25.8 (toolchain 1.25.9) -- **Modules**: `proofwatch` (instrumentation library), - `truthbeam` (OTel Collector processor) -- **Mission**: Provide enriched, standards-based compliance - evidence as an OpenTelemetry log stream - -## Build & Test Commands - -All commands run from the repository root via `task`. - -### Build & Deploy - -```sh -task workspace # Setup Go workspace with all modules -task build # Build beacon collector container image -task infra:deploy # Deploy full stack via podman-compose -task infra:undeploy # Tear down container stack — DESTRUCTIVE -task version:sync-otel-versions # Sync beacon-distro manifest from truthbeam -``` - -### Test - -```sh -task test # Unit tests with coverage for all modules - # (includes go-version, go-mod-consistency, - # and otel-version drift checks) -task test-race # Tests with -race detection -task dev:coverage-report # Generate HTML coverage reports -task quality:crapload # CRAP + GazeCRAP analysis (human-readable) -task quality:crapload-baseline # Generate .gaze/baseline.json per module -task quality:crapload-check # Check for CRAP regressions vs baseline -``` - -### Lint - -```sh -task lint # golangci-lint for all modules - # (config: .golangci.yml) -task mega-lint # MegaLinter (config: .mega-linter.yml) -task mega-lint-fix # MegaLinter with auto-fix enabled -``` - -### Code Generation & Semantic Conventions - -```sh -task codegen:api-codegen # go generate for all modules -task codegen:weaver-codegen # Generate Go code from model/ -task codegen:weaver-docsgen # Generate docs from model/ -task codegen:weaver-check # Validate model schema -task codegen:weaver-semantic-check # Validate logs against semconv -``` - -### Version Drift Checks - -```sh -task version:check-go-version # Containerfile vs module Go version -task version:check-otel-versions # manifest.yaml vs truthbeam OTel -task version:check-go-mod-consistency # OTel dep consistency within go.mod -task # List all available targets -``` - -### CI Workflow Structure - -| Workflow | File | Purpose | -|----------|------|---------| -| CI | `ci_checks.yml` | Reusable standardized CI (org-infra) | -| Local CI | `ci_local.yml` | golangci-lint, test, weaver-check, verify-codegen | -| CRAP Load | `ci_crapload.yml` | CRAP analysis on PRs | -| Security | `ci_security.yml` | OSV-Scanner, Trivy, OpenSSF Scorecards | -| SonarCloud | `ci_sonarcloud.yml` | SonarQube coverage analysis | -| Version Sync | `ci_version_sync.yml` | Beacon-distro/truthbeam version alignment | -| Go Compat | `ci_go_compat.yml` | Weekly forward-compat build (pinned + latest) | -| Dependencies | `ci_dependencies.yml` | Dependency review + Dependabot auto-approval | -| Publish GHCR | `ci_publish_ghcr.yml` | Build, scan, sign, publish container images | -| Promote Quay | `ci_publish_quay.yml` | Tag-triggered promotion to Quay.io | -| Scheduled | `ci_scheduled.yml` | Daily OSV-Scanner + Scorecards | - -## Project Structure +## Structure ```text proofwatch/ # Go module — evidence collection & emission library - cmd/validate-logs/ # CLI tool for validating log output internal/metrics/ # OTel metrics observer (evidence counters) + cmd/validate-logs/ # CLI tool for validating log output truthbeam/ # Go module — OTel Collector enrichment processor internal/applier/ # Attribute application logic internal/client/ # Generated OpenAPI client + otter cache internal/metadata/ # Component metadata + test fixtures beacon-distro/ # OTel Collector distribution (manifest.yaml + Containerfile) -model/ # Weaver semantic convention definitions (source of truth) - attributes.yaml # Attribute registry - entities.yaml # Entity definitions +model/ # Weaver semantic convention definitions (source of truth for attributes) templates/ # Weaver Jinja2 code generation templates - registry/ # Go code templates +configs/ # Collector and Loki deployment configs (base, storage, enrichment) +certs/ # Generated TLS certificates (gitignored, created by task infra:generate-self-signed-cert) +deploy/ # Deployment infrastructure (Terraform) +tests/ # Test infrastructure + integration/ # E2E Ginkgo tests, mock Compass, evidence fixtures docs/ # Architecture (DESIGN.md), dev guide (DEVELOPMENT.md), attribute docs - attributes/ # Generated attribute docs - integration/ # Integration guides -hack/ # Demo configs, sample data, TLS cert generation - demo/ # Demo infrastructure (compose, Terraform) - sampledata/ # Sample compliance evidence payloads - self-signed-cert/ # TLS certificate generation -openspec/ # OpenSpec specification workflow - changes/ # Active and archived change specs - schemas/ # Spec templates (unbound-force) - specs/ # Standalone specs -Taskfile.yml # Build automation entry point (canonical) -.taskfiles/ # Task modules (dev, codegen, infra, quality, version) + scripts +openspec/ # OpenSpec change proposals and specs .specify/memory/ # ComplyTime constitution (org-wide standards) +Taskfile.yml # Build automation entry point (canonical) +.taskfiles/ # Task modules (dev, codegen, infra, quality, version, integration) + scripts ``` -## Coding Conventions - -### Go - -- **Formatting**: `goimports` and `go fmt` (enforced by golangci-lint) -- **Linter**: golangci-lint v2 with `standard` defaults + `gosec` - (config: `.golangci.yml`) -- **Multi-language CI lint**: MegaLinter (config: `.mega-linter.yml`) -- **File headers**: `// SPDX-License-Identifier: Apache-2.0` -- **File naming**: lowercase with underscores (`my_file.go`) -- **Package names**: short, lowercase, no underscores -- **Error handling**: always check and handle; return to caller - when unresolvable locally -- **Import grouping**: stdlib, external, internal (enforced by - `goimports`) -- **Line length**: 99 characters unless exceeding improves - readability -- **Magic values**: centralize in dedicated constants; no inline - magic strings or numbers - -### Spec Writing - -- Use RFC 2119 language: MUST, SHOULD, MAY -- Scenarios: Given/When/Then format -- Requirement numbering: FR-NNN -- Line length: < 72 characters - -## Testing Conventions - -- **Framework**: Go stdlib `testing` package -- **Assertions**: `stretchr/testify` (`assert` and `require`) -- **Naming**: `TestFunctionName` or - `TestFunctionName_Description` (e.g., `TestProcessLogs`, - `TestNewTruthBeamProcessorWithInvalidConfig`) -- **Test fixtures**: struct-based fixtures with setup helpers - (e.g., `setupProofWatchTest`) -- **Isolation**: `httptest.NewServer` for HTTP mocking, - in-memory exporters for OTel, `t.TempDir()` for filesystem -- **Coverage**: `-coverprofile=coverage.out -covermode=atomic` - per module via `task test` -- **Quality gates**: CRAP load analysis via `gaze` with - baseline regression checks -- **Negative tests**: each scenario MUST include positive and - negative cases (error paths, invalid configs) -- **Module isolation**: tests run with `GOWORK=off` to verify - each module compiles independently +## Commands + +```bash +task # List all available targets +task build # Build the collector container image +task test # Unit tests with coverage (proofwatch + truthbeam) +task test-race # Tests with race detection +task lint # Lint all modules (golangci-lint v2) +task check # Run all quality gates (lint + test) +task deps # Tidy, verify, download deps for all modules +task clean # Remove build artifacts and test output +task version:sync # Sync Go + OTel versions across all modules, Containerfiles, CI, docs +task version:check # Validate version alignment (read-only, for CI) +task infra:deploy # Start local stack (podman-compose) +task infra:undeploy # Stop local stack — DESTRUCTIVE +``` ## Constraints -- **Go workspace, no root go.mod**: This repo uses `go.work` - to link modules. All module-level commands iterate over - `MODULES := ./proofwatch ./truthbeam`. Running - `go test ./...` from root will not work — use `task test`. -- **Build automation**: Use `task` (taskfile.dev), not `make`. - The Makefile has been deleted. +- **Go workspace, no root go.mod**: This repo uses `go.work` to link modules. All module-level commands iterate over `MODULES := ./proofwatch ./truthbeam`. Running `go test ./...` from root will not work — use `task test`. - **Generated files — DO NOT EDIT**: - - `proofwatch/attributes.go` — regenerate with - `task codegen:weaver-codegen` - - `truthbeam/internal/applier/attributes.go` — regenerate - with `task codegen:weaver-codegen` - - `truthbeam/internal/client/client.gen.go` — regenerate - with `task codegen:api-codegen` - - `docs/attributes/*.md` — regenerate with - `task codegen:weaver-docsgen` -- **Podman, not Docker**: Container operations use `podman` - and `podman-compose`. Do not reference `docker` commands. -- **No pre-commit hooks**: Run `task lint` locally before - submitting PRs. -- **Standards**: All coding standards are in - `.specify/memory/constitution.md`. For architecture context, - see `docs/DESIGN.md`. For dev setup, see `docs/DEVELOPMENT.md`. + - `proofwatch/attributes.go` — regenerate with `task codegen:weaver-codegen` + - `truthbeam/internal/applier/attributes.go` — regenerate with `task codegen:weaver-codegen` + - `truthbeam/internal/client/client.gen.go` — regenerate with `task codegen:api-codegen` + - `docs/attributes/*.md` — regenerate with `task codegen:weaver-docsgen` +- **Build automation**: Use `task` (taskfile.dev), not `make`. A deprecated Makefile exists but is not maintained. +- **External tools**: Install development tools with `task tools:install-all` or `task tools:install-weaver`. SHA256 checksums are pinned in `.tool_checksums` for supply chain security. Ginkgo CLI is managed as a `tool` directive in the root `go.mod` and invoked via `go tool ginkgo`. +- **Podman, not Docker**: Container operations use `podman` and `podman-compose`. Do not reference `docker` commands. +- **Lint**: Go linting uses `.golangci.yml` (v2 format). Multi-language CI linting uses `.mega-linter.yml`. No pre-commit hooks — run `task lint` locally. +- **Integration tests**: `tests/integration/` contains Ginkgo E2E tests and a mock Compass HTTP server. Run with `task integration:test` (all layers) or `task integration:test-profile PROFILE=base|storage|enrichment`. Do not recreate mock Compass — it already serves fixture-driven `/v1/enrich` responses. +- **Standards**: All coding standards are in `.specify/memory/constitution.md`. For architecture context, see `docs/DESIGN.md`. For dev setup, see `docs/DEVELOPMENT.md`. ## Local Dev Stack The compose stack (`compose.yaml`) runs the full evidence pipeline locally: -- **Loki** — log aggregation -- **Grafana** — visualization () -- **rustfs** — S3-compatible object storage (API , console , credentials `rustfsadmin`/`rustfsadmin`) -- **collector** — custom OTel Collector distribution with evidence processing +- **Base** (always on): **Loki** (log aggregation), **collector** (OTel Collector with evidence processing) +- **Storage** (`--profile storage`): adds **rustfs** (S3-compatible object storage, API , console , credentials `rustfsadmin`/`rustfsadmin`) +- **Enrichment** (`--profile enrichment`): adds **Compass** (mock enrichment service for TruthBeam) +- **Debug** (`--profile debug`): adds **Grafana** (visualization, ) + +Collector config is selected via `COLLECTOR_CONFIG` env var (defaults to `configs/collector-base.yaml`). All S3 environment variables have inline defaults that target rustfs. The stack works out-of-the-box with `task infra:deploy` — no AWS credentials needed. Override env vars in your shell to target real AWS S3 instead. ## Commits -All commits MUST use Conventional Commits, the `-s` flag -(Signed-off-by), and include an `Assisted-by` trailer when -AI-assisted. See `.specify/memory/constitution.md` for full -details. +All commits MUST use Conventional Commits, the `-s` flag (Signed-off-by), and include an `Assisted-by` trailer when AI-assisted. See `.specify/memory/constitution.md` for full details. + +## Integration Test Gotchas + +- **RustFS is not MinIO**: The health endpoint is `/health`, not `/minio/health/live` (returns 403). The S3 API is compatible but administrative endpoints differ. The `rustfs-init` container uses `rc` (RustFS CLI), not `mc` (MinIO client). +- **Loki 3.x OTLP labels**: Loki no longer auto-creates `{exporter="OTLP"}` from OTLP ingestion. Query indexed labels from `otlp_config.resource_attributes` in `configs/loki.yaml` directly (e.g. `{policy_rule_id="..."}`). Log attributes are stored as structured metadata and can be filtered with `| key="value"` after the stream selector. +- **Podman rootless volume permissions**: The collector runs as UID 10001. Host-mounted volumes need the `:U` flag (podman ownership remapping) or the container can't write. The `:Z` flag alone only handles SELinux relabeling. +- **S3 test portability**: Don't depend on host-installed CLI tools (`aws`, `mc`). The test bucket is configured with anonymous public access via `rc anonymous set public` in `rustfs-init`, so tests can query the S3 ListObjectsV2 API with plain HTTP — no auth headers needed. +- **awss3 exporter partitioning**: When `resource_attrs_to_s3.s3_prefix` is set, it replaces (not appends to) the static `s3_prefix`. Objects land at `{resource_attr_value}/evidence_logs_{uuid}.json`, not `{s3_prefix}/{resource_attr_value}/...`. ## Active Technologies -- Go 1.25.8 (toolchain 1.25.9), multi-module workspace (`go.work`) -- OpenTelemetry Collector SDK v1.57.0 / v0.151.0 (component - framework, pipeline data, processor interfaces) -- OTel Collector Builder v0.151.0 (beacon-distro binary) -- `github.com/gemaraproj/go-gemara` v0.4.0 (compliance - evidence model — Gemara schema) -- `github.com/Santiago-Labs/go-ocsf` (OCSF cybersecurity - schema types) +- Go 1.26.3, multi-module workspace (`go.work`) +- OpenTelemetry Collector SDK v1.57.0 / v0.151.0 (component framework, pipeline data, processor interfaces) +- `github.com/gemaraproj/go-gemara` v0.4.0 (compliance evidence model — Gemara v1 schema) +- `github.com/Santiago-Labs/go-ocsf` (OCSF cybersecurity schema types) - `github.com/maypok86/otter/v2` (in-memory cache, truthbeam) -- `github.com/oapi-codegen` (OpenAPI client generation, - truthbeam) +- `github.com/oapi-codegen` (OpenAPI client generation, truthbeam) - `go.uber.org/zap` (structured logging, truthbeam) - `github.com/stretchr/testify` (test assertions, both modules) - OTel Weaver (semantic convention model — Go constants + docs) -- Task v3 ([taskfile.dev](https://taskfile.dev)) — build automation +- OTel Collector Builder v0.151.0 (beacon-distro binary) - golangci-lint v2, MegaLinter, SonarCloud, gaze (quality tooling) +- Task v3 ([taskfile.dev](https://taskfile.dev)) — build automation - Podman + podman-compose (container runtime) -- Container: UBI10 Minimal (certs), golang:1.25.9 (build), - UBI10 Micro (runtime) +- Container: UBI10 Minimal (certs), golang:1.26.3 (build), UBI10 Micro (runtime) - Registries: `ghcr.io` (primary), Quay.io (secondary) - -## Behavioral Rules - -These rules are non-negotiable. Violations are CRITICAL severity. - -- **Gatekeeping**: MUST NOT modify quality/governance gates - (coverage thresholds, CRAP scores, severity definitions, - CI flags, agent settings, constitution MUST rules, review - limits, workflow markers). Stop and report instead. -- **Phase boundaries**: MUST NOT cross workflow phase boundaries. - Spec phases: spec artifacts only. Implement: source code. - Review: fixes only. Violation = process error, stop immediately. -- **CI parity**: MUST replicate CI checks locally before marking - tasks complete. Derive commands from `.github/workflows/`. -- **Review council**: MUST run `/review-council` before PR - submission. Resolve all REQUEST CHANGES. No code changes - between APPROVE and PR. Exempt: constitution amendments, - docs-only, emergency hotfixes. -- **Branch protection**: MUST NOT commit directly to `main`. - All changes via feature branches and PRs. -- **Documentation gate**: Before marking a task complete, - assess documentation impact: `CHANGELOG.md` for change - entries, `AGENTS.md` for structural updates (project - structure, conventions, build commands), `README.md` for - description changes. -- **Website gate**: MUST file `unbound-force/website` issue - for user-facing changes before PR merge. Exempt: internal - refactoring, test-only, CI-only, spec artifacts. -- **Zero-waste**: No orphaned specs, unused standards, or - aspirational documents that do not map to actionable work. - -### PR Review Commands - -| Command | When | Scope | -|---------|------|-------| -| `/review-council` | Pre-PR (local) | 5+ Divisor agents | -| `/review-pr [N]` | Post-PR (GitHub) | Single agent, CI analysis | - -## Specification Workflow - -All non-trivial changes MUST be preceded by a spec workflow. - -| Tier | Tool | When | Artifacts | -|------|------|------|-----------| -| Strategic | Speckit | >= 3 stories, cross-repo | `specs/NNN-*/` | -| Tactical | OpenSpec | < 3 stories, single-repo | `openspec/changes/*/` | - -Pipeline: `constitution -> specify -> clarify -> plan -> tasks -> -analyze -> checklist -> implement` - -**Ordering**: Constitution before specs. Spec before plan. Plan -before tasks. Tasks before implementation. Spec artifacts MUST -be committed/pushed before implementation begins. - -**Branches**: Speckit: `NNN-`. OpenSpec: `opsx/`. - -**Task bookkeeping**: Mark checkboxes `[x]` immediately on -completion. `[P]` marks parallel-eligible tasks. - -**When in doubt**: Start with OpenSpec. Escalate to Speckit if -scope grows beyond 3 stories or crosses repo boundaries. - -**What requires a spec**: New features, refactoring that changes -signatures, test additions across multiple functions, agent -changes, CI changes, data model changes. - -**Exempt**: Constitution amendments, typo fixes, emergency -hotfixes (retroactively documented). - -## Knowledge Retrieval - -Prefer Dewey MCP tools over grep/glob/read for cross-repo -context and architectural patterns. - -| Intent | Tool | -|--------|------| -| Conceptual | `dewey_semantic_search` | -| Keyword | `dewey_search` | -| Navigation | `dewey_traverse`, `dewey_get_page` | -| Discovery | `dewey_find_connections`, `dewey_similar` | - -**Fallback**: Use Read/Grep/Glob when Dewey is unavailable, -for exact string matching, known file paths, or non-Markdown -content (Go source, JSON, YAML). - -## Convention Packs - -This repository uses convention packs scaffolded by -unbound-force. Agents MUST read the applicable pack(s) -before writing or reviewing code. - -- `.opencode/uf/packs/default.md` -- `.opencode/uf/packs/default-custom.md` -- `.opencode/uf/packs/severity.md` -- `.opencode/uf/packs/content.md` -- `.opencode/uf/packs/content-custom.md` - -## Architecture - -ComplyBeacon follows a pipeline architecture built on the -OpenTelemetry Collector framework: - -- **Multi-module Go workspace**: `proofwatch` and `truthbeam` - are independent Go modules linked via `go.work`, each with - its own `go.mod` and test coverage -- **OTel Collector extension**: TruthBeam is a custom processor - plugin implementing the `processor.Logs` interface -- **Semantic conventions model**: Domain attributes defined in - `model/` using OTel Weaver, with code generation into both - modules via `task codegen:weaver-codegen` -- **Options pattern**: Configuration uses functional options - (e.g., `WithTracerProvider`, `WithMeterProvider`) -- **Version-locked distribution**: `beacon-distro/manifest.yaml` - is version-synced from `truthbeam/go.mod` via - `task version:sync-otel-versions` with CI drift detection -- **External enrichment**: Compass service consumed as a - pre-built container image, accessed via HTTP client with - caching (`otter`) and TLS support diff --git a/README.md b/README.md index 324ba68a..b049b5e6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,15 @@ # ComplyBeacon +---- + +> 🤖 LLM WARNING 🤖 +> +> This project was written with LLM (AI) assistance. +> +> 🤖 LLM WARNING 🤖 + +---- + **ComplyBeacon** is an open-source observability toolkit that collects, normalizes, and enriches compliance evidence by extending the OpenTelemetry (OTel) standard. It bridges the gap between raw policy scanner output and modern logging pipelines, providing a unified, enriched, and auditable data stream for security and compliance analysis. @@ -100,7 +110,7 @@ graph TB ## Prerequisites -- **Go 1.25+** +- **Go 1.26+** - **Task** ([taskfile.dev](https://taskfile.dev/installation/)) - **Podman** - **Git** @@ -135,7 +145,7 @@ task infra:undeploy # Send sample compliance evidence curl -X POST http://localhost:8088/eventsource/receiver \ -H "Content-Type: application/json" \ - -d @hack/sampledata/evidence.json + -d @tests/integration/fixtures/evidence-fail.json # View enriched logs in Grafana at http://localhost:3000 ``` @@ -179,6 +189,7 @@ task test # Run tests with coverage task test-race # Run tests with race detection task lint # Run golangci-lint task check # Run all quality gates (lint + test) +task test:integration # Run end-to-end integration tests (Ginkgo) task deps # Tidy, verify, and download dependencies task workspace # Set up Go workspace task clean # Remove build artifacts and test output @@ -218,13 +229,13 @@ task build IMAGE=ghcr.io/complytime/complybeacon TAG=v1.0.0 # Run standalone (without compose) podman run --rm \ - -v ./hack/demo/demo-config.yaml:/etc/otel-collector.yaml:Z \ + -v ./configs/collector-enrichment.yaml:/etc/otel-collector.yaml:Z \ -p 4317:4317 -p 8088:8088 \ complybeacon/collector:latest \ --config=/etc/otel-collector.yaml ``` -**Grafana Dashboard:** To configure Loki as default datasource, see [hack/demo/terraform/README.md](./hack/demo/terraform/README.md). +**Grafana Dashboard:** To configure Loki as default datasource, see [deploy/terraform/README.md](./deploy/terraform/README.md). ## Additional Resources diff --git a/Taskfile.yml b/Taskfile.yml index 48fc59fd..52c056b0 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -11,13 +11,17 @@ includes: taskfile: .taskfiles/version.yml quality: taskfile: .taskfiles/quality.yml + integration: + taskfile: .taskfiles/integration.yml + tools: + taskfile: .taskfiles/tools.yml vars: APP_NAME: '{{.APP_NAME | default "complybeacon"}}' TEST_DIR: '{{.TEST_DIR | default ".test-output"}}' MODULES: './proofwatch ./truthbeam' BIN_DIR: '{{.BIN_DIR | default "bin"}}' - CERT_DIR: '{{.CERT_DIR | default "hack/self-signed-cert"}}' + CERT_DIR: '{{.CERT_DIR | default "certs"}}' tasks: default: @@ -74,3 +78,8 @@ tasks: cmds: - rm -rf {{.BIN_DIR}} {{.TEST_DIR}} - GOFLAGS= go clean -modcache + + test:integration: + desc: "Run integration tests (all layers, or PROFILE=base|storage|enrichment)" + cmds: + - task: integration:test diff --git a/beacon-distro/Containerfile.collector b/beacon-distro/Containerfile.collector index 78af2bcb..6b074892 100644 --- a/beacon-distro/Containerfile.collector +++ b/beacon-distro/Containerfile.collector @@ -9,7 +9,7 @@ FROM registry.access.redhat.com/ubi10/ubi-minimal@sha256:a5f93d49cd946fe616205b8 RUN microdnf install -y ca-certificates && microdnf clean all # Stage 2: Build the OpenTelemetry Collector -FROM golang:1.25.9 AS build-stage +FROM golang:1.26.3@sha256:e3665e241a474aba30bbfaf177cfa88e1913e970c83bd86889cacfb67d6e7e51 AS build-stage WORKDIR /build # Build the collector using the OpenTelemetry collector builder diff --git a/compose.yaml b/compose.yaml index 838fefec..7f0342bc 100644 --- a/compose.yaml +++ b/compose.yaml @@ -5,11 +5,12 @@ services: ports: - "3100:3100" volumes: - - ./hack/demo/loki-config.yaml:/etc/loki/local-config.yaml:Z + - ./configs/loki.yaml:/etc/loki/local-config.yaml:z command: -config.file=/etc/loki/local-config.yaml grafana: image: "docker.io/grafana/grafana:11.6.0" + profiles: [debug] environment: - GF_FEATURE_TOGGLES_ENABLE=grafanaManagedRecordingRules - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin @@ -38,7 +39,8 @@ services: /run.sh rustfs: - image: "docker.io/rustfs/rustfs:latest" + image: "docker.io/rustfs/rustfs@sha256:d441111efe3af5bbd6e29eba7cd124f96bb7d49a0612390907f9216d5f970382" # v1.0.0-beta.3 + profiles: [storage] environment: - RUSTFS_ACCESS_KEY=${AWS_ACCESS_KEY_ID:-rustfsadmin} - RUSTFS_SECRET_KEY=${AWS_SECRET_ACCESS_KEY:-rustfsadmin} @@ -49,13 +51,35 @@ services: - rustfs-data:/data - rustfs-logs:/logs healthcheck: - test: ["CMD", "curl", "-sf", "http://localhost:9000/minio/health/live"] + test: ["CMD", "curl", "-sf", "http://localhost:9000/health"] + interval: 5s + timeout: 3s + retries: 5 + + rustfs-tls: + image: "docker.io/rustfs/rustfs@sha256:d441111efe3af5bbd6e29eba7cd124f96bb7d49a0612390907f9216d5f970382" # v1.0.0-beta.3 + profiles: [storage-tls] + environment: + - RUSTFS_ACCESS_KEY=${AWS_ACCESS_KEY_ID:-rustfsadmin} + - RUSTFS_SECRET_KEY=${AWS_SECRET_ACCESS_KEY:-rustfsadmin} + - RUSTFS_CERT_FILE=/certs/rustfs.crt + - RUSTFS_CERT_KEY_FILE=/certs/rustfs.key + ports: + - "9000:9000" + - "9001:9001" + volumes: + - rustfs-data:/data + - rustfs-logs:/logs + - ./certs:/certs:z + healthcheck: + test: ["CMD", "curl", "-sf", "https://localhost:9000/health", "--insecure"] interval: 5s timeout: 3s retries: 5 rustfs-init: - image: "docker.io/rustfs/rc:latest" + image: "docker.io/rustfs/rc@sha256:ca85fbee9f96425446915cb08b41d5e246c782eeb3f6a00d3b8294fce5219891" + profiles: [storage] depends_on: - rustfs entrypoint: ["/bin/sh", "-c"] @@ -66,18 +90,58 @@ services: sleep 2 done rc mb --ignore-existing local/$$S3_BUCKETNAME + # SECURITY: Limited anonymous access for integration tests only + # RustFS test bucket configured with anonymous public access so + # tests can query S3 ListObjectsV2 via plain HTTP + rc anonymous set public local/$$S3_BUCKETNAME + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-rustfsadmin} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-rustfsadmin} + - S3_BUCKETNAME=${S3_BUCKETNAME:-complybeacon-evidence} + restart: "no" + + rustfs-init-tls: + image: "docker.io/rustfs/rc@sha256:ca85fbee9f96425446915cb08b41d5e246c782eeb3f6a00d3b8294fce5219891" + profiles: [storage-tls] + depends_on: + - rustfs-tls + entrypoint: ["/bin/sh", "-c"] + command: + - | + # Wait for TLS endpoint + until rc alias set local https://rustfs-tls:9000 $$AWS_ACCESS_KEY_ID $$AWS_SECRET_ACCESS_KEY --insecure 2>/dev/null; do + echo "waiting for rustfs-tls..." + sleep 2 + done + rc mb --ignore-existing local/$$S3_BUCKETNAME --insecure + # NO ANONYMOUS ACCESS: TLS tests use authenticated rc client calls + echo "TLS bucket configured with authenticated access only" environment: - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-rustfsadmin} - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-rustfsadmin} - S3_BUCKETNAME=${S3_BUCKETNAME:-complybeacon-evidence} restart: "no" + compass: + build: + context: ./tests/integration/mock-compass + dockerfile: Containerfile + profiles: [enrichment] + volumes: + - ./tests/integration/fixtures/compass-responses.json:/fixtures/compass-responses.json:z + - ./certs:/certs:z + ports: + - "8081:8081" + healthcheck: + test: ["CMD", "/mock-compass", "-healthcheck"] + interval: 5s + timeout: 3s + retries: 5 + collector: build: context: ./beacon-distro dockerfile: Containerfile.collector - depends_on: - - rustfs-init environment: - AWS_REGION=${AWS_REGION:-us-east-1} - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-rustfsadmin} @@ -85,11 +149,36 @@ services: - S3_BUCKETNAME=${S3_BUCKETNAME:-complybeacon-evidence} - S3_OBJ_DIR=${S3_OBJ_DIR:-evidence} - S3_ENDPOINT=${S3_ENDPOINT:-http://rustfs:9000} + - S3_DISABLE_SSL=${S3_DISABLE_SSL:-true} + restart: unless-stopped + command: ["--config=/etc/otel-collector.yaml"] + volumes: + - ./${COLLECTOR_CONFIG:-configs/collector-base.yaml}:/etc/otel-collector.yaml:z + - ./data:/data:z,U + - ./certs:/certs:z + ports: + - "4317:4317" + - "8088:8088" + + collector-tls: + build: + context: ./beacon-distro + dockerfile: Containerfile.collector + profiles: [storage-tls] + environment: + - AWS_REGION=${AWS_REGION:-us-east-1} + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-rustfsadmin} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-rustfsadmin} + - S3_BUCKETNAME=${S3_BUCKETNAME:-complybeacon-evidence} + - S3_OBJ_DIR=${S3_OBJ_DIR:-evidence} + - S3_ENDPOINT=${S3_ENDPOINT:-https://rustfs-tls:9000} + - S3_DISABLE_SSL=${S3_DISABLE_SSL:-false} restart: unless-stopped command: ["--config=/etc/otel-collector.yaml"] volumes: - - ./hack/demo/demo-config.yaml:/etc/otel-collector.yaml:Z - - ./data:/data:Z + - ./${COLLECTOR_CONFIG:-configs/collector-storage-tls.yaml}:/etc/otel-collector.yaml:z + - ./data:/data:z,U + - ./certs:/certs:z ports: - "4317:4317" - "8088:8088" diff --git a/configs/collector-base.yaml b/configs/collector-base.yaml new file mode 100644 index 00000000..55ee26aa --- /dev/null +++ b/configs/collector-base.yaml @@ -0,0 +1,65 @@ +receivers: + webhookevent: + endpoint: 0.0.0.0:8088 + read_timeout: "500ms" + path: "/eventsource/receiver" + health_path: "/eventreceiver/healthcheck" + split_logs_at_newline: false + + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + # For logs that are received from or something similar filelog instead of OTLP. + # These are expected to be in OCSF format before entering the pipeline. + transform/ocsf: + error_mode: ignore + log_statements: + - context: log + conditions: + - body != nil + statements: + - set(observed_time, Now()) where observed_time_unix_nano == 0 + - set(time, observed_time) where time_unix_nano == 0 + # Extract policy.rule.id from OCSF policy.uid field (procedure ID for Compass mapper lookup) + - set(attributes["policy.rule.id"], ParseJSON(body)["policy"]["uid"]) where ParseJSON(body)["policy"]["uid"] != nil + # Extract control ID from policy.data.sources[0].name as a separate attribute (don't overwrite policy.rule.id) + # Note: policy.rule.id must match procedure.id in plan.yml for Compass mapper to work correctly + - set(attributes["policy.control.id"], ParseJSON(ParseJSON(body)["policy"]["data"])["sources"][0]["name"]) where ParseJSON(body)["policy"]["data"] != nil and ParseJSON(ParseJSON(body)["policy"]["data"])["sources"] != nil and ParseJSON(ParseJSON(body)["policy"]["data"])["sources"][0]["name"] != nil + # Extract policy.data.config.include (first element of array) as attribute + - set(attributes["policy.config.include"], ParseJSON(ParseJSON(body)["policy"]["data"])["sources"][0]["config"]["include"][0]) where ParseJSON(body)["policy"]["data"] != nil and ParseJSON(ParseJSON(body)["policy"]["data"])["sources"] != nil and ParseJSON(ParseJSON(body)["policy"]["data"])["sources"][0]["config"] != nil and ParseJSON(ParseJSON(body)["policy"]["data"])["sources"][0]["config"]["include"] != nil and Len(ParseJSON(ParseJSON(body)["policy"]["data"])["sources"][0]["config"]["include"]) > 0 + # Extract policy.engine.name from OCSF metadata.product.name field + - set(attributes["policy.engine.name"], ParseJSON(body)["metadata"]["product"]["name"]) where ParseJSON(body)["metadata"]["product"]["name"] != nil + # Extract policy.evaluation.result from OCSF status field (mapped to Compass enum values) + - set(attributes["policy.evaluation.result"], "Passed") where ParseJSON(body)["status"] == "success" + - set(attributes["policy.evaluation.result"], "Failed") where ParseJSON(body)["status"] == "failure" + - set(attributes["policy.evaluation.result"], "Not Run") where ParseJSON(body)["status"] == "not_run" + - set(attributes["policy.evaluation.result"], "Needs Review") where ParseJSON(body)["status"] == "needs_review" + - set(attributes["policy.evaluation.result"], "Not Applicable") where ParseJSON(body)["status"] == "not_applicable" + - set(attributes["policy.evaluation.result"], "Unknown") where ParseJSON(body)["status"] == "unknown" or ParseJSON(body)["status"] == "error" or ParseJSON(body)["status"] == "timeout" + # Set default Unknown if status is not recognized + - set(attributes["policy.evaluation.result"], "Unknown") where ParseJSON(body)["status"] != nil and attributes["policy.evaluation.result"] == nil + # Extract severity from OCSF evidence + - set(attributes["severity"], ParseJSON(body)["severity"]) where ParseJSON(body)["severity"] != nil + # Set policy.rule.id as resource attribute for S3 partitioning + - set(resource.attributes["policy.rule.id"], attributes["policy.rule.id"]) where attributes["policy.rule.id"] != nil + +exporters: + debug: + verbosity: detailed + otlphttp/logs: + endpoint: "http://loki:3100/otlp" + tls: + insecure: true + +service: + pipelines: + logs/analysis_pipeline: + receivers: [webhookevent, otlp] + processors: [batch, transform/ocsf] + exporters: [debug, otlphttp/logs] diff --git a/hack/demo/demo-config.yaml b/configs/collector-enrichment.yaml similarity index 99% rename from hack/demo/demo-config.yaml rename to configs/collector-enrichment.yaml index 1015a5d2..52916219 100644 --- a/hack/demo/demo-config.yaml +++ b/configs/collector-enrichment.yaml @@ -165,7 +165,7 @@ exporters: s3_prefix: ${S3_OBJ_DIR} endpoint: ${S3_ENDPOINT} s3_force_path_style: true - disable_ssl: true + disable_ssl: ${S3_DISABLE_SSL} unique_key_func_name: uuidv7 s3_partition_format: "" file_prefix: "evidence_" diff --git a/configs/collector-storage-tls.yaml b/configs/collector-storage-tls.yaml new file mode 100644 index 00000000..297413e1 --- /dev/null +++ b/configs/collector-storage-tls.yaml @@ -0,0 +1,191 @@ +receivers: + webhookevent: + endpoint: 0.0.0.0:8088 + read_timeout: "500ms" + path: "/eventsource/receiver" + health_path: "/eventreceiver/healthcheck" + split_logs_at_newline: false + + # Filelog receiver: reads metrics JSON and sends to Loki as logs + filelog/metrics: + include: + - /data/metrics.jsonl + start_at: end + poll_interval: 250ms + operators: + - type: json_parser + id: parser-metrics + output: extract_metric_data + - type: add + id: extract_metric_data + field: attributes.service_name + value: "signaltometrics" + + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + batch/metrics: + timeout: 1s + send_batch_size: 10 + # Transform processor: extracts metric data from OTLP JSON and converts to log attributes + transform/metrics_to_logs: + error_mode: ignore + log_statements: + - context: log + conditions: + - attributes["service_name"] == "signaltometrics" + statements: + # Parse OTLP JSON: resourceMetrics[0].scopeMetrics[0].metrics[0] + - set(attributes["_metric_parsed"], true) where IsMap(ParseJSON(body)) + - set(attributes["metric_name"], ParseJSON(body)["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]["name"]) where IsMap(ParseJSON(body)) and ParseJSON(body)["resourceMetrics"] != nil + # Extract metric value (asDouble or asInt) + - set(attributes["metric_value"], ParseJSON(body)["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]["sum"]["dataPoints"][0]["asDouble"]) where ParseJSON(body)["resourceMetrics"] != nil and ParseJSON(body)["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]["sum"] != nil and ParseJSON(body)["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]["sum"]["dataPoints"][0]["asDouble"] != nil + - set(attributes["metric_value"], ParseJSON(body)["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]["sum"]["dataPoints"][0]["asInt"]) where ParseJSON(body)["resourceMetrics"] != nil and ParseJSON(body)["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]["sum"] != nil and ParseJSON(body)["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]["sum"]["dataPoints"][0]["asInt"] != nil + # Extract metric labels from dataPoints and resource attributes + - merge_maps(attributes, ParseJSON(body)["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]["sum"]["dataPoints"][0]["attributes"], "upsert") where ParseJSON(body)["resourceMetrics"] != nil and ParseJSON(body)["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]["sum"]["dataPoints"][0]["attributes"] != nil + - merge_maps(attributes, ParseJSON(body)["resourceMetrics"][0]["resource"]["attributes"], "upsert") where ParseJSON(body)["resourceMetrics"] != nil and ParseJSON(body)["resourceMetrics"][0]["resource"]["attributes"] != nil + - set(body, attributes) where attributes["_metric_parsed"] == true + + # For logs that are received from or something similar filelog instead of OTLP. + # These are expected to be in OCSF format before entering the pipeline. + transform/ocsf: + error_mode: ignore + log_statements: + - context: log + conditions: + - body != nil + statements: + - set(observed_time, Now()) where observed_time_unix_nano == 0 + - set(time, observed_time) where time_unix_nano == 0 + # Extract policy.rule.id from OCSF policy.uid field (procedure ID for Compass mapper lookup) + - set(attributes["policy.rule.id"], ParseJSON(body)["policy"]["uid"]) where ParseJSON(body)["policy"]["uid"] != nil + # Extract control ID from policy.data.sources[0].name as a separate attribute (don't overwrite policy.rule.id) + # Note: policy.rule.id must match procedure.id in plan.yml for Compass mapper to work correctly + - set(attributes["policy.control.id"], ParseJSON(ParseJSON(body)["policy"]["data"])["sources"][0]["name"]) where ParseJSON(body)["policy"]["data"] != nil and ParseJSON(ParseJSON(body)["policy"]["data"])["sources"] != nil and ParseJSON(ParseJSON(body)["policy"]["data"])["sources"][0]["name"] != nil + # Extract policy.data.config.include (first element of array) as attribute + - set(attributes["policy.config.include"], ParseJSON(ParseJSON(body)["policy"]["data"])["sources"][0]["config"]["include"][0]) where ParseJSON(body)["policy"]["data"] != nil and ParseJSON(ParseJSON(body)["policy"]["data"])["sources"] != nil and ParseJSON(ParseJSON(body)["policy"]["data"])["sources"][0]["config"] != nil and ParseJSON(ParseJSON(body)["policy"]["data"])["sources"][0]["config"]["include"] != nil and Len(ParseJSON(ParseJSON(body)["policy"]["data"])["sources"][0]["config"]["include"]) > 0 + # Extract policy.engine.name from OCSF metadata.product.name field + - set(attributes["policy.engine.name"], ParseJSON(body)["metadata"]["product"]["name"]) where ParseJSON(body)["metadata"]["product"]["name"] != nil + # Extract policy.evaluation.result from OCSF status field (mapped to Compass enum values) + - set(attributes["policy.evaluation.result"], "Passed") where ParseJSON(body)["status"] == "success" + - set(attributes["policy.evaluation.result"], "Failed") where ParseJSON(body)["status"] == "failure" + - set(attributes["policy.evaluation.result"], "Not Run") where ParseJSON(body)["status"] == "not_run" + - set(attributes["policy.evaluation.result"], "Needs Review") where ParseJSON(body)["status"] == "needs_review" + - set(attributes["policy.evaluation.result"], "Not Applicable") where ParseJSON(body)["status"] == "not_applicable" + - set(attributes["policy.evaluation.result"], "Unknown") where ParseJSON(body)["status"] == "unknown" or ParseJSON(body)["status"] == "error" or ParseJSON(body)["status"] == "timeout" + # Set default Unknown if status is not recognized + - set(attributes["policy.evaluation.result"], "Unknown") where ParseJSON(body)["status"] != nil and attributes["policy.evaluation.result"] == nil + # Extract severity from OCSF evidence + - set(attributes["severity"], ParseJSON(body)["severity"]) where ParseJSON(body)["severity"] != nil + # Set policy.rule.id as resource attribute for S3 partitioning + - set(resource.attributes["policy.rule.id"], attributes["policy.rule.id"]) where attributes["policy.rule.id"] != nil + +connectors: + # SignalToMetrics: converts enriched logs to metrics (exporter from logs, receiver to metrics) + signaltometrics: + logs: + # Control evaluation count metric + - name: compliance.control.evaluations + description: "Total number of control evaluations" + unit: "1" + sum: + value: "1" # increment by 1 for each log record + # Control evaluation result metric + - name: compliance.control.evaluation.result + description: "Count of control evaluations by result (Passed/Failed/etc)" + unit: "1" + sum: + value: "1" + # Control compliance status metric + - name: compliance.control.status + description: "Count of control evaluations by compliance status" + unit: "1" + sum: + value: "1" + # Policy config include metric - associates config.include with control ID + - name: compliance.control.config.include + description: "Count of control evaluations by policy config include value" + unit: "1" + sum: + value: "1" + # Control evaluations by policy engine + - name: compliance.control.by.engine + description: "Count of control evaluations by policy engine" + unit: "1" + sum: + value: "1" + # Control evaluations by control category + - name: compliance.control.by.category + description: "Count of control evaluations by control category" + unit: "1" + sum: + value: "1" + # Control evaluations by enrichment status + - name: compliance.control.by.enrichment.status + description: "Count of control evaluations by enrichment status" + unit: "1" + sum: + value: "1" + # Control evaluations by severity + - name: compliance.control.by.severity + description: "Count of control evaluations by severity level" + unit: "1" + sum: + value: "1" + +exporters: + debug: + verbosity: detailed + otlphttp/logs: + endpoint: "http://loki:3100/otlp" + tls: + insecure: true + # File exporter: writes metrics as JSON for filelog receiver + file/metrics: + path: /data/metrics.jsonl + format: json + awss3/logs: + s3uploader: + region: ${AWS_REGION} + s3_bucket: ${S3_BUCKETNAME} + s3_prefix: ${S3_OBJ_DIR} + endpoint: ${S3_ENDPOINT} + s3_force_path_style: true + disable_ssl: false + insecure: true # Allow self-signed certificates for testing + unique_key_func_name: uuidv7 + s3_partition_format: "" + file_prefix: "evidence_" + resource_attrs_to_s3: + s3_prefix: "policy.rule.id" + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [debug] + # Metrics pipeline: SignalToMetrics -> file -> filelog -> Loki + metrics/control_health: + receivers: [signaltometrics] + processors: [batch/metrics] + exporters: [debug, file/metrics] + # Logs pipeline: reads metrics from file and exports to Loki + logs/metrics_from_file: + receivers: [filelog/metrics] + processors: [batch/metrics, transform/metrics_to_logs] + exporters: [debug, otlphttp/logs] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [debug] + logs/analysis_pipeline: + receivers: [webhookevent, otlp] + processors: [batch, transform/ocsf] + exporters: [debug, otlphttp/logs, awss3/logs, signaltometrics] diff --git a/configs/collector-storage.yaml b/configs/collector-storage.yaml new file mode 100644 index 00000000..a2f21c26 --- /dev/null +++ b/configs/collector-storage.yaml @@ -0,0 +1,190 @@ +receivers: + webhookevent: + endpoint: 0.0.0.0:8088 + read_timeout: "500ms" + path: "/eventsource/receiver" + health_path: "/eventreceiver/healthcheck" + split_logs_at_newline: false + + # Filelog receiver: reads metrics JSON and sends to Loki as logs + filelog/metrics: + include: + - /data/metrics.jsonl + start_at: end + poll_interval: 250ms + operators: + - type: json_parser + id: parser-metrics + output: extract_metric_data + - type: add + id: extract_metric_data + field: attributes.service_name + value: "signaltometrics" + + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + batch/metrics: + timeout: 1s + send_batch_size: 10 + # Transform processor: extracts metric data from OTLP JSON and converts to log attributes + transform/metrics_to_logs: + error_mode: ignore + log_statements: + - context: log + conditions: + - attributes["service_name"] == "signaltometrics" + statements: + # Parse OTLP JSON: resourceMetrics[0].scopeMetrics[0].metrics[0] + - set(attributes["_metric_parsed"], true) where IsMap(ParseJSON(body)) + - set(attributes["metric_name"], ParseJSON(body)["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]["name"]) where IsMap(ParseJSON(body)) and ParseJSON(body)["resourceMetrics"] != nil + # Extract metric value (asDouble or asInt) + - set(attributes["metric_value"], ParseJSON(body)["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]["sum"]["dataPoints"][0]["asDouble"]) where ParseJSON(body)["resourceMetrics"] != nil and ParseJSON(body)["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]["sum"] != nil and ParseJSON(body)["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]["sum"]["dataPoints"][0]["asDouble"] != nil + - set(attributes["metric_value"], ParseJSON(body)["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]["sum"]["dataPoints"][0]["asInt"]) where ParseJSON(body)["resourceMetrics"] != nil and ParseJSON(body)["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]["sum"] != nil and ParseJSON(body)["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]["sum"]["dataPoints"][0]["asInt"] != nil + # Extract metric labels from dataPoints and resource attributes + - merge_maps(attributes, ParseJSON(body)["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]["sum"]["dataPoints"][0]["attributes"], "upsert") where ParseJSON(body)["resourceMetrics"] != nil and ParseJSON(body)["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]["sum"]["dataPoints"][0]["attributes"] != nil + - merge_maps(attributes, ParseJSON(body)["resourceMetrics"][0]["resource"]["attributes"], "upsert") where ParseJSON(body)["resourceMetrics"] != nil and ParseJSON(body)["resourceMetrics"][0]["resource"]["attributes"] != nil + - set(body, attributes) where attributes["_metric_parsed"] == true + + # For logs that are received from or something similar filelog instead of OTLP. + # These are expected to be in OCSF format before entering the pipeline. + transform/ocsf: + error_mode: ignore + log_statements: + - context: log + conditions: + - body != nil + statements: + - set(observed_time, Now()) where observed_time_unix_nano == 0 + - set(time, observed_time) where time_unix_nano == 0 + # Extract policy.rule.id from OCSF policy.uid field (procedure ID for Compass mapper lookup) + - set(attributes["policy.rule.id"], ParseJSON(body)["policy"]["uid"]) where ParseJSON(body)["policy"]["uid"] != nil + # Extract control ID from policy.data.sources[0].name as a separate attribute (don't overwrite policy.rule.id) + # Note: policy.rule.id must match procedure.id in plan.yml for Compass mapper to work correctly + - set(attributes["policy.control.id"], ParseJSON(ParseJSON(body)["policy"]["data"])["sources"][0]["name"]) where ParseJSON(body)["policy"]["data"] != nil and ParseJSON(ParseJSON(body)["policy"]["data"])["sources"] != nil and ParseJSON(ParseJSON(body)["policy"]["data"])["sources"][0]["name"] != nil + # Extract policy.data.config.include (first element of array) as attribute + - set(attributes["policy.config.include"], ParseJSON(ParseJSON(body)["policy"]["data"])["sources"][0]["config"]["include"][0]) where ParseJSON(body)["policy"]["data"] != nil and ParseJSON(ParseJSON(body)["policy"]["data"])["sources"] != nil and ParseJSON(ParseJSON(body)["policy"]["data"])["sources"][0]["config"] != nil and ParseJSON(ParseJSON(body)["policy"]["data"])["sources"][0]["config"]["include"] != nil and Len(ParseJSON(ParseJSON(body)["policy"]["data"])["sources"][0]["config"]["include"]) > 0 + # Extract policy.engine.name from OCSF metadata.product.name field + - set(attributes["policy.engine.name"], ParseJSON(body)["metadata"]["product"]["name"]) where ParseJSON(body)["metadata"]["product"]["name"] != nil + # Extract policy.evaluation.result from OCSF status field (mapped to Compass enum values) + - set(attributes["policy.evaluation.result"], "Passed") where ParseJSON(body)["status"] == "success" + - set(attributes["policy.evaluation.result"], "Failed") where ParseJSON(body)["status"] == "failure" + - set(attributes["policy.evaluation.result"], "Not Run") where ParseJSON(body)["status"] == "not_run" + - set(attributes["policy.evaluation.result"], "Needs Review") where ParseJSON(body)["status"] == "needs_review" + - set(attributes["policy.evaluation.result"], "Not Applicable") where ParseJSON(body)["status"] == "not_applicable" + - set(attributes["policy.evaluation.result"], "Unknown") where ParseJSON(body)["status"] == "unknown" or ParseJSON(body)["status"] == "error" or ParseJSON(body)["status"] == "timeout" + # Set default Unknown if status is not recognized + - set(attributes["policy.evaluation.result"], "Unknown") where ParseJSON(body)["status"] != nil and attributes["policy.evaluation.result"] == nil + # Extract severity from OCSF evidence + - set(attributes["severity"], ParseJSON(body)["severity"]) where ParseJSON(body)["severity"] != nil + # Set policy.rule.id as resource attribute for S3 partitioning + - set(resource.attributes["policy.rule.id"], attributes["policy.rule.id"]) where attributes["policy.rule.id"] != nil + +connectors: + # SignalToMetrics: converts enriched logs to metrics (exporter from logs, receiver to metrics) + signaltometrics: + logs: + # Control evaluation count metric + - name: compliance.control.evaluations + description: "Total number of control evaluations" + unit: "1" + sum: + value: "1" # increment by 1 for each log record + # Control evaluation result metric + - name: compliance.control.evaluation.result + description: "Count of control evaluations by result (Passed/Failed/etc)" + unit: "1" + sum: + value: "1" + # Control compliance status metric + - name: compliance.control.status + description: "Count of control evaluations by compliance status" + unit: "1" + sum: + value: "1" + # Policy config include metric - associates config.include with control ID + - name: compliance.control.config.include + description: "Count of control evaluations by policy config include value" + unit: "1" + sum: + value: "1" + # Control evaluations by policy engine + - name: compliance.control.by.engine + description: "Count of control evaluations by policy engine" + unit: "1" + sum: + value: "1" + # Control evaluations by control category + - name: compliance.control.by.category + description: "Count of control evaluations by control category" + unit: "1" + sum: + value: "1" + # Control evaluations by enrichment status + - name: compliance.control.by.enrichment.status + description: "Count of control evaluations by enrichment status" + unit: "1" + sum: + value: "1" + # Control evaluations by severity + - name: compliance.control.by.severity + description: "Count of control evaluations by severity level" + unit: "1" + sum: + value: "1" + +exporters: + debug: + verbosity: detailed + otlphttp/logs: + endpoint: "http://loki:3100/otlp" + tls: + insecure: true + # File exporter: writes metrics as JSON for filelog receiver + file/metrics: + path: /data/metrics.jsonl + format: json + awss3/logs: + s3uploader: + region: ${AWS_REGION} + s3_bucket: ${S3_BUCKETNAME} + s3_prefix: ${S3_OBJ_DIR} + endpoint: ${S3_ENDPOINT} + s3_force_path_style: true + disable_ssl: ${S3_DISABLE_SSL} + unique_key_func_name: uuidv7 + s3_partition_format: "" + file_prefix: "evidence_" + resource_attrs_to_s3: + s3_prefix: "policy.rule.id" + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [debug] + # Metrics pipeline: SignalToMetrics -> file -> filelog -> Loki + metrics/control_health: + receivers: [signaltometrics] + processors: [batch/metrics] + exporters: [debug, file/metrics] + # Logs pipeline: reads metrics from file and exports to Loki + logs/metrics_from_file: + receivers: [filelog/metrics] + processors: [batch/metrics, transform/metrics_to_logs] + exporters: [debug, otlphttp/logs] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [debug] + logs/analysis_pipeline: + receivers: [webhookevent, otlp] + processors: [batch, transform/ocsf] + exporters: [debug, otlphttp/logs, awss3/logs, signaltometrics] diff --git a/hack/demo/loki-config.yaml b/configs/loki.yaml similarity index 100% rename from hack/demo/loki-config.yaml rename to configs/loki.yaml diff --git a/hack/self-signed-cert/openssl.cnf b/configs/openssl.cnf similarity index 91% rename from hack/self-signed-cert/openssl.cnf rename to configs/openssl.cnf index ef62ca4b..dfa20121 100644 --- a/hack/self-signed-cert/openssl.cnf +++ b/configs/openssl.cnf @@ -22,4 +22,4 @@ DNS.1 = compass [v3_ca] subjectKeyIdentifier = hash authorityKeyIdentifier = keyid:always,issuer -basicConstraints = critical, CA:true \ No newline at end of file +basicConstraints = critical, CA:true diff --git a/hack/demo/terraform/.gitignore b/deploy/terraform/.gitignore similarity index 100% rename from hack/demo/terraform/.gitignore rename to deploy/terraform/.gitignore diff --git a/hack/demo/terraform/README.md b/deploy/terraform/README.md similarity index 100% rename from hack/demo/terraform/README.md rename to deploy/terraform/README.md diff --git a/hack/demo/terraform/main.tf b/deploy/terraform/main.tf similarity index 100% rename from hack/demo/terraform/main.tf rename to deploy/terraform/main.tf diff --git a/hack/demo/terraform/outputs.tf b/deploy/terraform/outputs.tf similarity index 100% rename from hack/demo/terraform/outputs.tf rename to deploy/terraform/outputs.tf diff --git a/deploy/terraform/variables.tf b/deploy/terraform/variables.tf new file mode 100644 index 00000000..0a9ef5c4 --- /dev/null +++ b/deploy/terraform/variables.tf @@ -0,0 +1,16 @@ +variable "grafana_url" { + description = "Grafana server URL" + type = string + default = "http://localhost:3000" +} + +variable "grafana_auth" { + description = "Grafana authentication (e.g., 'admin:secretpassword' or use API key)" + type = string + sensitive = true + + validation { + condition = var.grafana_auth != "admin:admin" && length(var.grafana_auth) > 0 + error_message = "Grafana authentication must be explicitly set and cannot use default 'admin:admin' credentials." + } +} diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 8f0474e0..21800bca 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -143,3 +143,5 @@ err = pw.LogWithSeverity(ctx, evidence, olog.SeverityWarn) * Receiving an EnrichmentRequest from `truthbeam`. * Performing a lookup based on the policy details. * Returning an EnrichmentResponse with compliance attributes. + +The full evidence pipeline is validated by automated integration tests in `tests/integration/`. See `docs/DEVELOPMENT.md` for how to run them. diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 2d3c10b4..c28eea66 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -36,7 +36,7 @@ It complements the [DESIGN.md](./DESIGN.md) document by focusing on the practica ### Required Software -- **Go 1.25+**: The project uses Go 1.25.8 with toolchain 1.25.9 +- **Go 1.26+**: The project uses Go 1.26.3 - **Podman**: For containerized development and deployment (Docker is not supported) - **Task**: For build automation ([installation guide](https://taskfile.dev/installation/)) - **Git**: For version control @@ -129,9 +129,15 @@ complybeacon/ ├── beacon-distro/ # OpenTelemetry Collector distribution │ ├── config.yaml # Collector configuration │ └── Containerfile.collector # Container definition -├── hack/ # Development utilities -│ ├── demo/ # Demo configurations -│ ├── sampledata/ # Sample data for testing +├── configs/ # Deployment configs (collector, Loki) +│ ├── collector-base.yaml # Base layer: OCSF transform + Loki +│ ├── collector-storage.yaml # Storage layer: adds S3 export +│ ├── collector-enrichment.yaml # Enrichment layer: adds TruthBeam +│ └── loki.yaml # Loki configuration +├── certs/ # TLS certificate generation +├── deploy/ # Deployment infrastructure (Terraform) +├── tests/ # Test infrastructure +│ └── integration/ # E2E Ginkgo tests, mock Compass, fixtures └── bin/ # Built binaries (created by task infra:deploy) ``` @@ -156,29 +162,36 @@ cd truthbeam && go test -v ./... ### Integration Testing -The project includes integration tests using the demo environment: +The project includes automated integration tests using [Ginkgo](https://onsi.github.io/ginkgo/) that validate the evidence pipeline at three deployment layers: -```bash -# Start the demo environment (builds images and starts services) -task deploy +| Layer | Profile | What it tests | +|-------|---------|--------------| +| Base | *(default)* | OCSF transform + Loki export | +| Storage | `storage` | S3 evidence export + partitioning | +| Enrichment | `enrichment` | TruthBeam enrichment via mock Compass | -# Or run in background -podman-compose -f compose.yaml up -d +**Prerequisites:** +- Podman and podman-compose +- Go 1.26+ (Ginkgo CLI is managed via `tool` directive in root `go.mod`) -# Test the pipeline -curl -X POST http://localhost:8088/eventsource/receiver \ - -H "Content-Type: application/json" \ - -d @hack/sampledata/evidence.json +**Run all layers:** -# View logs -podman-compose -f compose.yaml logs -f +```bash +task integration:test +``` -# Stop the environment -task infra:undeploy +**Run a single layer:** -# Check logs in Grafana at http://localhost:3000 +```bash +task integration:test-profile PROFILE=base +task integration:test-profile PROFILE=storage +task integration:test-profile PROFILE=enrichment ``` +Each run builds the collector image, starts the appropriate services, runs the matching Ginkgo test suite (filtered by label), and tears down. Certificates are generated automatically if missing. Test output is written to `.test-output/integration/`. + +For details on test cases, fixtures, and mock Compass configuration, see [tests/integration/README.md](../tests/integration/README.md). + ## Component Development ### 1. ProofWatch Development @@ -268,7 +281,7 @@ podman build --no-cache -f beacon-distro/Containerfile.collector -t complybeacon podman run --rm complybeacon-collector --config /etc/otelcol-beacon/config.yaml # Full stack deployment for integration testing -task deploy +task infra:deploy ``` ## Debugging and Troubleshooting @@ -330,10 +343,18 @@ task codegen:weaver-codegen The demo environment orchestrates multiple containers (Grafana, Loki, Beacon Collector, Compass). -1. **Start the full stack:** +1. **Generate self-signed certificate** + +Since compass and truthbeam enable TLS by default, first generate self-signed certificates for testing/development: + +```bash +task infra:generate-self-signed-cert +``` + +2. **Start the full stack:** ```bash # Interactive mode (shows logs in terminal) -task infra:deploy +task deploy # Or background/detached mode podman-compose -f compose.yaml up -d @@ -344,18 +365,18 @@ This automatically: - Builds the beacon collector image - Starts all services (Grafana, Loki, Collector) -2. **Test the pipeline:** +3. **Test the pipeline:** ```bash curl -X POST http://localhost:8088/eventsource/receiver \ -H "Content-Type: application/json" \ - -d @hack/sampledata/evidence.json + -d @tests/integration/fixtures/evidence-fail.json ``` -3. **View results:** +4. **View results:** - Grafana: - View logs: `podman-compose -f compose.yaml logs -f` -4. **Stop the stack:** +5. **Stop the stack:** ```bash task infra:undeploy ``` diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..c5ec9fc6 --- /dev/null +++ b/go.mod @@ -0,0 +1,56 @@ +module github.com/complytime/complybeacon + +go 1.26.3 + +tool github.com/unbound-force/gaze/cmd/gaze + +tool github.com/onsi/ginkgo/v2/ginkgo + +require ( + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/charmbracelet/bubbles v1.0.0 // indirect + github.com/charmbracelet/bubbletea v1.3.10 // indirect + github.com/charmbracelet/colorprofile v0.4.1 // indirect + github.com/charmbracelet/lipgloss v1.1.0 // indirect + github.com/charmbracelet/log v0.4.2 // indirect + github.com/charmbracelet/x/ansi v0.11.6 // indirect + github.com/charmbracelet/x/cellbuf v0.0.15 // indirect + github.com/charmbracelet/x/term v0.2.2 // indirect + github.com/clipperhouse/displaywidth v0.9.0 // indirect + github.com/clipperhouse/stringish v0.1.1 // indirect + github.com/clipperhouse/uax29/v2 v2.5.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/fzipp/gocyclo v0.6.0 // indirect + github.com/go-logfmt/logfmt v0.6.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/goccy/go-yaml v1.19.2 // indirect + github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/lucasb-eyer/go-colorful v1.3.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-localereader v0.0.1 // indirect + github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/muesli/termenv v0.16.0 // indirect + github.com/onsi/ginkgo/v2 v2.29.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/spf13/cobra v1.10.2 // indirect + github.com/spf13/pflag v1.0.9 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/unbound-force/gaze v1.5.0 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 // indirect + golang.org/x/mod v0.35.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/text v0.36.0 // indirect + golang.org/x/tools v0.44.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..6fa89098 --- /dev/null +++ b/go.sum @@ -0,0 +1,140 @@ +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3vj1nolY= +github.com/aymanbagabas/go-udiff v0.3.1/go.mod h1:G0fsKmG+P6ylD0r6N/KgQD/nWzgfnl8ZBcNLgcbrw8E= +github.com/charmbracelet/bubbles v1.0.0 h1:12J8/ak/uCZEMQ6KU7pcfwceyjLlWsDLAxB5fXonfvc= +github.com/charmbracelet/bubbles v1.0.0/go.mod h1:9d/Zd5GdnauMI5ivUIVisuEm3ave1XwXtD1ckyV6r3E= +github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw= +github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4= +github.com/charmbracelet/colorprofile v0.4.1 h1:a1lO03qTrSIRaK8c3JRxJDZOvhvIeSco3ej+ngLk1kk= +github.com/charmbracelet/colorprofile v0.4.1/go.mod h1:U1d9Dljmdf9DLegaJ0nGZNJvoXAhayhmidOdcBwAvKk= +github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= +github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= +github.com/charmbracelet/log v0.4.2 h1:hYt8Qj6a8yLnvR+h7MwsJv/XvmBJXiueUcI3cIxsyig= +github.com/charmbracelet/log v0.4.2/go.mod h1:qifHGX/tc7eluv2R6pWIpyHDDrrb/AG71Pf2ysQu5nw= +github.com/charmbracelet/x/ansi v0.11.6 h1:GhV21SiDz/45W9AnV2R61xZMRri5NlLnl6CVF7ihZW8= +github.com/charmbracelet/x/ansi v0.11.6/go.mod h1:2JNYLgQUsyqaiLovhU2Rv/pb8r6ydXKS3NIttu3VGZQ= +github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI= +github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q= +github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payRxjMjKgx2PaCWLZ4p3ro9y97+TVLZNaRZgJwSVDQ= +github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= +github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= +github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= +github.com/clipperhouse/displaywidth v0.9.0 h1:Qb4KOhYwRiN3viMv1v/3cTBlz3AcAZX3+y9OLhMtAtA= +github.com/clipperhouse/displaywidth v0.9.0/go.mod h1:aCAAqTlh4GIVkhQnJpbL0T/WfcrJXHcj8C0yjYcjOZA= +github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= +github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= +github.com/clipperhouse/uax29/v2 v2.5.0 h1:x7T0T4eTHDONxFJsL94uKNKPHrclyFI0lm7+w94cO8U= +github.com/clipperhouse/uax29/v2 v2.5.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= +github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= +github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= +github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= +github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= +github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= +github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/onsi/ginkgo/v2 v2.29.0 h1:rfh+ZFjgJhYWRoIqVf3Uwx/W20yLrcrE2h2GmYVRaag= +github.com/onsi/ginkgo/v2 v2.29.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= +github.com/onsi/gomega v1.40.0 h1:Vtol0e1MghCD2ZVIilPDIg44XSL9l2QAn8ZNaljWcJc= +github.com/onsi/gomega v1.40.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/unbound-force/gaze v1.5.0 h1:nlW2sBgpmZ3cn/LzA25WYx6R9UMjynLG2vqHWasQI/k= +github.com/unbound-force/gaze v1.5.0/go.mod h1:1y5Cgk7jPFuwe94qgXp5cSqH992IelzEpigX/AB+0xY= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= +golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/hack/demo/terraform/variables.tf b/hack/demo/terraform/variables.tf deleted file mode 100644 index 9f5357f6..00000000 --- a/hack/demo/terraform/variables.tf +++ /dev/null @@ -1,12 +0,0 @@ -variable "grafana_url" { - description = "Grafana server URL" - type = string - default = "http://localhost:3000" -} - -variable "grafana_auth" { - description = "Grafana authentication (e.g., 'admin:admin' or use API key)" - type = string - default = "admin:admin" - sensitive = true -} diff --git a/proofwatch/cmd/validate-logs/main.go b/proofwatch/cmd/validate-logs/main.go index aa950a09..53f63aa8 100644 --- a/proofwatch/cmd/validate-logs/main.go +++ b/proofwatch/cmd/validate-logs/main.go @@ -1,10 +1,11 @@ package main import ( - "context" "encoding/json" "fmt" "os" + "path/filepath" + "strings" "time" ocsf "github.com/Santiago-Labs/go-ocsf/ocsf/v1_5_0" @@ -245,11 +246,26 @@ func main() { format := os.Args[1] outputFile := "stdout" if len(os.Args) > 2 { - outputFile = os.Args[2] + outputFile = filepath.Clean(os.Args[2]) + + // Security: Validate output path to prevent directory traversal + if strings.Contains(outputFile, "..") { + fmt.Fprintf(os.Stderr, "Error: Path traversal detected in output file path\n") + os.Exit(1) + } + + // Ensure the output file is not an absolute path to sensitive locations + if filepath.IsAbs(outputFile) { + if strings.HasPrefix(outputFile, "/etc/") || + strings.HasPrefix(outputFile, "/sys/") || + strings.HasPrefix(outputFile, "/proc/") || + strings.HasPrefix(outputFile, "/dev/") { + fmt.Fprintf(os.Stderr, "Error: Cannot write to system directories\n") + os.Exit(1) + } + } } - ctx := context.Background() - _ = ctx // Suppress unused variable warning var allLogs []plog.Logs if format == "gemara" || format == "both" { diff --git a/proofwatch/go.mod b/proofwatch/go.mod index 4b3cd937..e5dd71b3 100644 --- a/proofwatch/go.mod +++ b/proofwatch/go.mod @@ -1,8 +1,6 @@ module github.com/complytime/complybeacon/proofwatch -go 1.25.8 - -toolchain go1.25.9 +go 1.26.3 require ( github.com/Santiago-Labs/go-ocsf v0.1.1-0.20250729170529-8b19b43949a6 @@ -41,8 +39,8 @@ require ( golang.org/x/mod v0.35.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.43.0 // indirect - golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c // indirect - golang.org/x/tools v0.43.0 // indirect + golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa // indirect + golang.org/x/tools v0.44.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/proofwatch/go.sum b/proofwatch/go.sum index ae3e62cf..d6a16e01 100644 --- a/proofwatch/go.sum +++ b/proofwatch/go.sum @@ -107,10 +107,10 @@ golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c h1:6a8FdnNk6bTXBjR4AGKFgUKuo+7GnR3FX5L7CbveeZc= -golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c/go.mod h1:TpUTTEp9frx7rTdLpC9gFG9kdI7zVLFTFFlqaH2Cncw= -golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= -golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= +golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa h1:efT73AJZfAAUV7SOip6pWGkwJDzIGiKBZGVzHYa+ve4= +golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa/go.mod h1:kHjTxDEnAu6/Nl9lDkzjWpR+bmKfxeiRuSDlsMb70gE= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= diff --git a/tests/integration/README.md b/tests/integration/README.md new file mode 100644 index 00000000..f47c2816 --- /dev/null +++ b/tests/integration/README.md @@ -0,0 +1,112 @@ +# Integration Tests + +End-to-end tests for the ComplyBeacon evidence pipeline. Tests use [Ginkgo](https://onsi.github.io/ginkgo/) v2 with [Gomega](https://onsi.github.io/gomega/) matchers to drive the compose stack at multiple deployment layers and validate that evidence flows correctly. + +## Prerequisites + +- [Task](https://taskfile.dev/installation/) v3+ +- [Podman](https://docs.podman.io/) and podman-compose (`pip install podman-compose`) +- Go 1.25+ (Ginkgo CLI is managed via `tool` directive in root `go.mod`) + +### Installing Task + +```bash +# macOS +brew install go-task/tap/go-task + +# Linux +sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b ~/.local/bin +``` + +Certificates are generated automatically if missing. + +## Running Tests + +```bash +# Run all layers sequentially +task integration:test + +# Run a single layer +task integration:test-profile PROFILE=base +task integration:test-profile PROFILE=storage +task integration:test-profile PROFILE=enrichment + +# Top-level alias (runs all layers) +task test:integration +``` + +### IDE / Manual Debugging + +Start the stack without running tests, then run Ginkgo directly: + +```bash +# Start the stack for a specific layer +task integration:up PROFILE=base + +# Run tests from repo root +go tool ginkgo run -vv --label-filter="base" ./tests/integration/ + +# Tear down when done +task integration:down +``` + +Test output is written to `.test-output/integration/`. + +## Deployment Layers + +| Layer | Compose Profile | Collector Config | Services | +|-------|----------------|------------------|----------| +| Base | *(none)* | `configs/collector-base.yaml` | collector, Loki | +| Storage | `storage` | `configs/collector-storage.yaml` | collector, Loki, RustFS | +| Storage TLS | `storage-tls` | `configs/collector-storage-tls.yaml` | collector-tls, Loki, RustFS (TLS) | +| Enrichment | `enrichment` | `configs/collector-enrichment.yaml` | collector, Loki, RustFS, mock Compass | + +## Test Suites + +| File | Label | Test Cases | +|------|-------|------------| +| `base_test.go` | `base` | Healthcheck, OCSF transform to Loki, success evidence, malformed evidence resilience | +| `storage_test.go` | `storage` | S3 export, S3 partitioning by policy ID | +| `storage_tls_test.go` | `storage-tls` | TLS S3 export, TLS S3 partitioning (via `rc` client) | +| `enrichment_test.go` | `enrichment` | Enrichment applied, unknown policy graceful handling | + +## Mock Compass + +The `mock-compass/` directory contains a lightweight Go HTTP server that simulates the Compass enrichment API. It: + +1. Loads `fixtures/compass-responses.json` at startup +2. Serves `POST /v1/enrich` — looks up `policyRuleId` in fixtures, returns matching response or `Unmapped` +3. Serves `GET /healthz` — returns 200 + +### Adding a New Policy Response + +Add an entry to `fixtures/compass-responses.json` keyed by the policy rule ID: + +```json +{ + "my_new_policy": { + "compliance": { + "control": { + "id": "MY-CTRL-01", + "catalogId": "MY-CATALOG", + "category": "My Category" + }, + "enrichmentStatus": "Success", + "frameworks": { + "frameworks": ["My Framework v1"], + "requirements": ["MY-CTRL-01"] + } + } + } +} +``` + +Then create a matching evidence fixture in `fixtures/` with `policy.uid` set to `my_new_policy`. + +## Adding a New Test Case + +1. Create evidence fixture(s) in `fixtures/` following the OCSF format from existing fixtures +2. If the test needs a new Compass response, add it to `compass-responses.json` +3. Add the test spec to the appropriate layer file (`base_test.go`, `storage_test.go`, `storage_tls_test.go`, or `enrichment_test.go`) +4. Use the `Label()` decorator matching the layer so `--label-filter` selects it correctly +5. Follow the pattern: `postEvidence()` → `Eventually` poll via `queryLoki()`/`listS3Objects()` → verify pipeline health diff --git a/tests/integration/base/base_suite_test.go b/tests/integration/base/base_suite_test.go new file mode 100644 index 00000000..3c2f983e --- /dev/null +++ b/tests/integration/base/base_suite_test.go @@ -0,0 +1,43 @@ +package base_test + +import ( + "fmt" + "net/http" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/complytime/complybeacon/tests/integration" +) + +var ( + webhookURL string + lokiURL string +) + +func TestBase(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Base Layer Suite") +} + +var _ = BeforeSuite(func() { + webhookURL = integration.EnvOrDefault("WEBHOOK_URL", "http://localhost:8088") + lokiURL = integration.EnvOrDefault("LOKI_URL", "http://localhost:3100") + + client := &http.Client{Timeout: 2 * time.Second} + resp, err := client.Get(webhookURL + "/eventreceiver/healthcheck") + if err != nil || resp.StatusCode != http.StatusOK { + msg := fmt.Sprintf( + "Stack not running — webhook healthcheck at %s failed.\n"+ + "Start it with: task integration:up PROFILE=base\n", + webhookURL+"/eventreceiver/healthcheck", + ) + if err != nil { + msg += fmt.Sprintf("Error: %v\n", err) + } + Fail(msg) + } + resp.Body.Close() +}) diff --git a/tests/integration/base/base_test.go b/tests/integration/base/base_test.go new file mode 100644 index 00000000..b9906d84 --- /dev/null +++ b/tests/integration/base/base_test.go @@ -0,0 +1,76 @@ +package base_test + +import ( + "net/http" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/complytime/complybeacon/tests/integration" +) + +var _ = Describe("Base Layer", func() { + Describe("Webhook Receiver", func() { + It("responds to healthcheck", func() { + resp, err := integration.HTTPClient.Get(webhookURL + "/eventreceiver/healthcheck") + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + }) + }) + + Describe("OCSF Transform Pipeline", func() { + It("transforms failure evidence to Loki", func() { + resp, err := integration.PostEvidence(webhookURL, "../fixtures/evidence-fail.json") + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + + Eventually(func() ([]string, error) { + return integration.QueryLoki(lokiURL, `{policy_rule_id="github_branch_protection"}`) + }, 30*time.Second, 3*time.Second).ShouldNot(BeEmpty()) + }) + + It("transforms success evidence to Loki", func() { + resp, err := integration.PostEvidence(webhookURL, "../fixtures/evidence-pass.json") + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + + Eventually(func() ([]string, error) { + return integration.QueryLoki(lokiURL, `{policy_rule_id="code_review"}`) + }, 30*time.Second, 3*time.Second).ShouldNot(BeEmpty()) + }) + }) + + Describe("Resilience", func() { + It("survives malformed evidence without disruption", func() { + resp, err := integration.PostEvidence(webhookURL, "../fixtures/evidence-malformed.json") + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(SatisfyAny( + Equal(http.StatusOK), + Equal(http.StatusBadRequest), + )) + + Eventually(func() (int, error) { + r, err := integration.HTTPClient.Get(webhookURL + "/eventreceiver/healthcheck") + if err != nil { + return 0, err + } + defer r.Body.Close() + return r.StatusCode, nil + }, 6*time.Second, 2*time.Second).Should(Equal(http.StatusOK)) + + resp2, err := integration.PostEvidence(webhookURL, "../fixtures/evidence-fail.json") + Expect(err).NotTo(HaveOccurred()) + defer resp2.Body.Close() + Expect(resp2.StatusCode).To(Equal(http.StatusOK)) + + Eventually(func() ([]string, error) { + return integration.QueryLoki(lokiURL, `{policy_rule_id="github_branch_protection"}`) + }, 30*time.Second, 3*time.Second).ShouldNot(BeEmpty()) + }) + }) +}) diff --git a/tests/integration/enrichment/enrichment_suite_test.go b/tests/integration/enrichment/enrichment_suite_test.go new file mode 100644 index 00000000..4bd569a6 --- /dev/null +++ b/tests/integration/enrichment/enrichment_suite_test.go @@ -0,0 +1,43 @@ +package enrichment_test + +import ( + "fmt" + "net/http" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/complytime/complybeacon/tests/integration" +) + +var ( + webhookURL string + lokiURL string +) + +func TestEnrichment(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Enrichment Layer Suite") +} + +var _ = BeforeSuite(func() { + webhookURL = integration.EnvOrDefault("WEBHOOK_URL", "http://localhost:8088") + lokiURL = integration.EnvOrDefault("LOKI_URL", "http://localhost:3100") + + client := &http.Client{Timeout: 2 * time.Second} + resp, err := client.Get(webhookURL + "/eventreceiver/healthcheck") + if err != nil || resp.StatusCode != http.StatusOK { + msg := fmt.Sprintf( + "Stack not running — webhook healthcheck at %s failed.\n"+ + "Start it with: task integration:up PROFILE=enrichment\n", + webhookURL+"/eventreceiver/healthcheck", + ) + if err != nil { + msg += fmt.Sprintf("Error: %v\n", err) + } + Fail(msg) + } + resp.Body.Close() +}) diff --git a/tests/integration/enrichment/enrichment_test.go b/tests/integration/enrichment/enrichment_test.go new file mode 100644 index 00000000..a478813b --- /dev/null +++ b/tests/integration/enrichment/enrichment_test.go @@ -0,0 +1,47 @@ +package enrichment_test + +import ( + "net/http" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/complytime/complybeacon/tests/integration" +) + +var _ = Describe("Enrichment Layer", func() { + Describe("Enrichment Pipeline", func() { + It("enriches evidence with compliance metadata", func() { + resp, err := integration.PostEvidence(webhookURL, "../fixtures/evidence-fail.json") + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + + Eventually(func() ([]string, error) { + return integration.QueryLoki(lokiURL, + `{policy_rule_id="github_branch_protection"} | compliance_enrichment_status="Success"`, + ) + }, 30*time.Second, 3*time.Second).ShouldNot(BeEmpty()) + }) + + It("marks unknown policies as Unmapped", func() { + resp, err := integration.PostEvidence(webhookURL, "../fixtures/evidence-unknown.json") + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + + Eventually(func() ([]string, error) { + return integration.QueryLoki(lokiURL, + `{policy_rule_id="unknown_policy_xyz"} | compliance_enrichment_status="Unmapped"`, + ) + }, 30*time.Second, 3*time.Second).ShouldNot(BeEmpty()) + + // Verify pipeline is still healthy after processing unknown policy. + resp2, err := integration.HTTPClient.Get(webhookURL + "/eventreceiver/healthcheck") + Expect(err).NotTo(HaveOccurred()) + defer resp2.Body.Close() + Expect(resp2.StatusCode).To(Equal(http.StatusOK)) + }) + }) +}) diff --git a/tests/integration/fixtures/compass-responses.json b/tests/integration/fixtures/compass-responses.json new file mode 100644 index 00000000..34769e6f --- /dev/null +++ b/tests/integration/fixtures/compass-responses.json @@ -0,0 +1,30 @@ +{ + "github_branch_protection": { + "compliance": { + "control": { + "id": "OSPS-QA-07.01", + "catalogId": "OSPS", + "category": "Quality Assurance" + }, + "enrichmentStatus": "Success", + "frameworks": { + "frameworks": ["OSPS Baseline 2025-02"], + "requirements": ["OSPS-QA-07.01"] + } + } + }, + "code_review": { + "compliance": { + "control": { + "id": "OSPS-QA-07.02", + "catalogId": "OSPS", + "category": "Quality Assurance" + }, + "enrichmentStatus": "Success", + "frameworks": { + "frameworks": ["OSPS Baseline 2025-02"], + "requirements": ["OSPS-QA-07.02"] + } + } + } +} diff --git a/hack/sampledata/evidence.json b/tests/integration/fixtures/evidence-fail.json similarity index 76% rename from hack/sampledata/evidence.json rename to tests/integration/fixtures/evidence-fail.json index 28d60d1d..8daec956 100644 --- a/hack/sampledata/evidence.json +++ b/tests/integration/fixtures/evidence-fail.json @@ -5,32 +5,18 @@ "category_uid": 6, "class_name": "Scan Activity", "class_uid": 6007, - "cloud": { - "provider": "" - }, + "cloud": {"provider": ""}, "event_day": 0, "metadata": { "log_provider": "conforma", - "product": { - "name": "conforma", - "vendor_name": "conforma", - "version": "v0.8.6" - }, + "product": {"name": "conforma", "vendor_name": "conforma", "version": "v0.8.6"}, "uid": "c2p-conforma-C2PPolicy", "version": "v0.8.6" }, "num_files": 1, - "observables": [ - { - "name": "evidence.json", - "type": "File Name", - "type_id": 7 - } - ], + "observables": [{"name": "evidence.json", "type": "File Name", "type_id": 7}], "osint": null, - "scan": { - "type_id": 0 - }, + "scan": {"type_id": 0}, "severity": "unknown", "severity_id": 0, "status": "failure", @@ -46,4 +32,4 @@ }, "action": "observed", "action_id": 3 -} \ No newline at end of file +} diff --git a/tests/integration/fixtures/evidence-malformed.json b/tests/integration/fixtures/evidence-malformed.json new file mode 100644 index 00000000..e23239cf --- /dev/null +++ b/tests/integration/fixtures/evidence-malformed.json @@ -0,0 +1 @@ +{"not_ocsf": true, "garbage_field": 12345, "nested": {"but": "no_policy_key"}} diff --git a/hack/sampledata/evidence-code-review.json b/tests/integration/fixtures/evidence-pass.json similarity index 76% rename from hack/sampledata/evidence-code-review.json rename to tests/integration/fixtures/evidence-pass.json index 81c27d48..6f0a67b7 100644 --- a/hack/sampledata/evidence-code-review.json +++ b/tests/integration/fixtures/evidence-pass.json @@ -5,32 +5,18 @@ "category_uid": 6, "class_name": "Scan Activity", "class_uid": 6007, - "cloud": { - "provider": "" - }, + "cloud": {"provider": ""}, "event_day": 0, "metadata": { "log_provider": "conforma", - "product": { - "name": "conforma", - "vendor_name": "conforma", - "version": "v0.8.6" - }, + "product": {"name": "conforma", "vendor_name": "conforma", "version": "v0.8.6"}, "uid": "c2p-conforma-C2PPolicy", "version": "v0.8.6" }, "num_files": 1, - "observables": [ - { - "name": "evidence-code-review.json", - "type": "File Name", - "type_id": 7 - } - ], + "observables": [{"name": "evidence-code-review.json", "type": "File Name", "type_id": 7}], "osint": null, - "scan": { - "type_id": 0 - }, + "scan": {"type_id": 0}, "severity": "unknown", "severity_id": 0, "status": "success", diff --git a/tests/integration/fixtures/evidence-unknown.json b/tests/integration/fixtures/evidence-unknown.json new file mode 100644 index 00000000..f57ba805 --- /dev/null +++ b/tests/integration/fixtures/evidence-unknown.json @@ -0,0 +1,35 @@ +{ + "activity_id": 0, + "activity_name": "", + "category_name": "Application Activity", + "category_uid": 6, + "class_name": "Scan Activity", + "class_uid": 6007, + "cloud": {"provider": ""}, + "event_day": 0, + "metadata": { + "log_provider": "conforma", + "product": {"name": "conforma", "vendor_name": "conforma", "version": "v0.8.6"}, + "uid": "c2p-conforma-C2PPolicy", + "version": "v0.8.6" + }, + "num_files": 1, + "observables": [{"name": "evidence-unknown.json", "type": "File Name", "type_id": 7}], + "osint": null, + "scan": {"type_id": 0}, + "severity": "unknown", + "severity_id": 0, + "status": "failure", + "status_id": 2, + "time": 1757088191576, + "type_name": "", + "type_uid": 60070, + "policy": { + "data": "{\"name\":\"Unknown Policy\",\"description\":\"Policy for unknown testing\",\"sources\":[{\"name\":\"UNKNOWN-01\",\"policy\":[],\"ruleData\":{},\"config\":{\"include\":[\"unknown_policy_xyz\"]}}]}", + "desc": "Policy for unknown testing", + "name": "Unknown Policy", + "uid": "unknown_policy_xyz" + }, + "action": "observed", + "action_id": 3 +} diff --git a/tests/integration/go.mod b/tests/integration/go.mod new file mode 100644 index 00000000..ecebcdfa --- /dev/null +++ b/tests/integration/go.mod @@ -0,0 +1,23 @@ +module github.com/complytime/complybeacon/tests/integration + +go 1.26.3 + +require ( + github.com/onsi/ginkgo/v2 v2.29.0 + github.com/onsi/gomega v1.41.0 +) + +require ( + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/mod v0.35.0 // indirect + golang.org/x/net v0.53.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/text v0.36.0 // indirect + golang.org/x/tools v0.44.0 // indirect +) diff --git a/tests/integration/go.sum b/tests/integration/go.sum new file mode 100644 index 00000000..8e1e80b9 --- /dev/null +++ b/tests/integration/go.sum @@ -0,0 +1,69 @@ +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= +github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/onsi/ginkgo/v2 v2.29.0 h1:rfh+ZFjgJhYWRoIqVf3Uwx/W20yLrcrE2h2GmYVRaag= +github.com/onsi/ginkgo/v2 v2.29.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= +github.com/onsi/gomega v1.41.0 h1:OwKp4pXNgVxf6sCplzYo794OFNuoL2q2SBMU5NSWOjA= +github.com/onsi/gomega v1.41.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= +google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tests/integration/helpers.go b/tests/integration/helpers.go new file mode 100644 index 00000000..493059d5 --- /dev/null +++ b/tests/integration/helpers.go @@ -0,0 +1,189 @@ +// Package integration provides shared helpers for integration test suites. +package integration + +import ( + "bytes" + "encoding/json" + "encoding/xml" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "strings" + "time" +) + +// HTTPClient is a shared client with a generous timeout for polling operations. +var HTTPClient = &http.Client{Timeout: 10 * time.Second} + +// PostEvidence reads a fixture file and POSTs it to the webhook endpoint. +// Returns the HTTP response for status code assertions. +func PostEvidence(webhookURL, fixturePath string) (*http.Response, error) { + body, err := os.ReadFile(fixturePath) + if err != nil { + return nil, fmt.Errorf("reading fixture %s: %w", fixturePath, err) + } + + req, err := http.NewRequest(http.MethodPost, webhookURL+"/eventsource/receiver", bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("creating request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + return HTTPClient.Do(req) +} + +// LokiQueryResponse is the envelope returned by Loki's query_range API. +type LokiQueryResponse struct { + Status string `json:"status"` + Data struct { + Result []struct { + Values [][]string `json:"values"` + } `json:"result"` + } `json:"data"` +} + +// QueryLoki queries Loki's query_range endpoint and returns the log line strings. +// The start time is set to 1 hour ago to capture recent logs. Returns an empty +// slice (not an error) when no results match — this works with Eventually/ShouldNot(BeEmpty()). +func QueryLoki(lokiURL, query string) ([]string, error) { + startTime := time.Now().Add(-1 * time.Hour).Format(time.RFC3339Nano) + + req, err := http.NewRequest(http.MethodGet, lokiURL+"/loki/api/v1/query_range", nil) + if err != nil { + return nil, fmt.Errorf("creating loki request: %w", err) + } + q := req.URL.Query() + q.Set("query", query) + q.Set("start", startTime) + q.Set("limit", "10") + req.URL.RawQuery = q.Encode() + + resp, err := HTTPClient.Do(req) + if err != nil { + return nil, fmt.Errorf("querying loki: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("loki returned status %d: %s", resp.StatusCode, string(body)) + } + + var result LokiQueryResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decoding loki response: %w", err) + } + + if result.Status != "success" { + return nil, fmt.Errorf("loki query status: %s", result.Status) + } + + var lines []string + for _, stream := range result.Data.Result { + for _, entry := range stream.Values { + if len(entry) >= 2 { + lines = append(lines, entry[1]) + } + } + } + return lines, nil +} + +// S3ListBucketResult is the XML envelope for S3 ListObjectsV2. +type S3ListBucketResult struct { + XMLName xml.Name `xml:"ListBucketResult"` + Contents []S3Object `xml:"Contents"` +} + +// S3Object represents a single object in an S3 listing. +type S3Object struct { + Key string `xml:"Key"` +} + +// ListS3Objects queries the S3 ListObjectsV2 API via plain HTTP (anonymous access) +// and returns the object keys matching the given prefix. +func ListS3Objects(s3URL, bucket, prefix string) ([]string, error) { + url := fmt.Sprintf("%s/%s?list-type=2&prefix=%s", s3URL, bucket, prefix) + + resp, err := HTTPClient.Get(url) + if err != nil { + return nil, fmt.Errorf("querying S3: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("S3 returned status %d: %s", resp.StatusCode, string(body)) + } + + var result S3ListBucketResult + if err := xml.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decoding S3 XML: %w", err) + } + + var keys []string + for _, obj := range result.Contents { + keys = append(keys, obj.Key) + } + return keys, nil +} + +// ExecInContainer runs a command inside a running compose service container using +// podman-compose exec. Returns combined stdout+stderr output. The compose project +// is determined by the working directory (repo root is expected). +func ExecInContainer(service string, command ...string) (string, error) { + composeBin, composeArgs := DetectComposeTool() + + args := append(composeArgs, "exec", "-T", service) + args = append(args, command...) + + cmd := exec.Command(composeBin, args...) + cmd.Dir = RepoRoot() + + out, err := cmd.CombinedOutput() + if err != nil { + return string(out), fmt.Errorf("exec in %s failed: %w\noutput: %s", service, err, string(out)) + } + return string(out), nil +} + +// DetectComposeTool returns the binary name and base args for the compose tool. +func DetectComposeTool() (string, []string) { + if _, err := exec.LookPath("podman-compose"); err == nil { + return "podman-compose", []string{"-f", "compose.yaml"} + } + return "docker", []string{"compose", "-f", "compose.yaml"} +} + +// RepoRoot walks up from the test directory to find the repo root (where compose.yaml lives). +func RepoRoot() string { + wd, err := os.Getwd() + if err != nil { + return "../../.." + } + + if _, err := os.Stat(wd + "/compose.yaml"); err == nil { + return wd + } + + parts := strings.Split(wd, string(os.PathSeparator)) + for i := len(parts); i > 0; i-- { + candidate := string(os.PathSeparator) + strings.Join(parts[1:i], string(os.PathSeparator)) + if _, err := os.Stat(candidate + "/compose.yaml"); err == nil { + return candidate + } + } + + return "../../.." +} + +// EnvOrDefault returns the value of the environment variable named by key, +// or fallback if the variable is empty or unset. +func EnvOrDefault(key, fallback string) string { + if v := os.Getenv(key); v != "" { + return v + } + return fallback +} diff --git a/tests/integration/mock-compass/Containerfile b/tests/integration/mock-compass/Containerfile new file mode 100644 index 00000000..efd2b8ae --- /dev/null +++ b/tests/integration/mock-compass/Containerfile @@ -0,0 +1,13 @@ +FROM golang:1.26.3@sha256:e3665e241a474aba30bbfaf177cfa88e1913e970c83bd86889cacfb67d6e7e51 AS build +WORKDIR /build +COPY go.mod ./ +RUN go mod download +COPY main.go ./ +RUN CGO_ENABLED=0 go build -o mock-compass . + +FROM registry.access.redhat.com/ubi10/ubi-micro@sha256:1983e3d2fa54a1afedf015ad4d8f9cfaeed15182d9cf532e5cb35de2c1bda580 +COPY --from=build /build/mock-compass /mock-compass +HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \ + CMD ["/mock-compass", "-healthcheck"] +USER 65534 +ENTRYPOINT ["/mock-compass"] diff --git a/tests/integration/mock-compass/go.mod b/tests/integration/mock-compass/go.mod new file mode 100644 index 00000000..7df4846d --- /dev/null +++ b/tests/integration/mock-compass/go.mod @@ -0,0 +1,3 @@ +module github.com/complytime/complybeacon/tests/integration/mock-compass + +go 1.26.3 diff --git a/tests/integration/mock-compass/main.go b/tests/integration/mock-compass/main.go new file mode 100644 index 00000000..339059fc --- /dev/null +++ b/tests/integration/mock-compass/main.go @@ -0,0 +1,148 @@ +package main + +import ( + "crypto/tls" + "encoding/json" + "flag" + "fmt" + "log" + "net" + "net/http" + "os" + "time" +) + +type Policy struct { + PolicyEngineName string `json:"policyEngineName"` + PolicyRuleId string `json:"policyRuleId"` +} + +type EnrichmentRequest struct { + Policy Policy `json:"policy"` +} + +type ComplianceControl struct { + Id string `json:"id"` + CatalogId string `json:"catalogId"` + Category string `json:"category"` + Applicability []string `json:"applicability,omitempty"` + RemediationDescription *string `json:"remediationDescription,omitempty"` +} + +type ComplianceFrameworks struct { + Frameworks []string `json:"frameworks"` + Requirements []string `json:"requirements"` +} + +type ComplianceRisk struct { + Level string `json:"level"` +} + +type Compliance struct { + Control ComplianceControl `json:"control"` + EnrichmentStatus string `json:"enrichmentStatus"` + Frameworks ComplianceFrameworks `json:"frameworks"` + Risk *ComplianceRisk `json:"risk,omitempty"` +} + +type EnrichmentResponse struct { + Compliance Compliance `json:"compliance"` +} + +func main() { + healthcheck := flag.Bool("healthcheck", false, "Run TCP healthcheck against addr and exit") + fixturesPath := flag.String("fixtures", "/fixtures/compass-responses.json", "Path to fixtures JSON file") + certPath := flag.String("cert", "/certs/compass.crt", "Path to TLS certificate") + keyPath := flag.String("key", "/certs/compass.key", "Path to TLS private key") + addr := flag.String("addr", ":8081", "Address to listen on") + flag.Parse() + + if *healthcheck { + conn, err := net.DialTimeout("tcp", *addr, 3*time.Second) + if err != nil { + os.Exit(1) + } + conn.Close() + os.Exit(0) + } + + fixtures, err := loadFixtures(*fixturesPath) + if err != nil { + log.Fatalf("Failed to load fixtures: %v", err) + } + + mux := http.NewServeMux() + mux.HandleFunc("POST /v1/enrich", handleEnrich(fixtures)) + mux.HandleFunc("GET /healthz", handleHealth) + + cert, err := tls.LoadX509KeyPair(*certPath, *keyPath) + if err != nil { + log.Fatalf("Failed to load TLS certificate: %v", err) + } + + server := &http.Server{ + Addr: *addr, + Handler: mux, + TLSConfig: &tls.Config{ + Certificates: []tls.Certificate{cert}, + }, + } + + log.Printf("Starting mock Compass server on %s", *addr) + if err := server.ListenAndServeTLS("", ""); err != nil { + log.Fatalf("Server failed: %v", err) + } +} + +func loadFixtures(path string) (map[string]EnrichmentResponse, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading fixtures file: %w", err) + } + + var fixtures map[string]EnrichmentResponse + if err := json.Unmarshal(data, &fixtures); err != nil { + return nil, fmt.Errorf("parsing fixtures JSON: %w", err) + } + + return fixtures, nil +} + +func handleEnrich(fixtures map[string]EnrichmentResponse) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req EnrichmentRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "Invalid request body", http.StatusBadRequest) + return + } + + response, ok := fixtures[req.Policy.PolicyRuleId] + if !ok { + response = EnrichmentResponse{ + Compliance: Compliance{ + Control: ComplianceControl{ + Id: "", + CatalogId: "", + Category: "", + }, + EnrichmentStatus: "Unmapped", + Frameworks: ComplianceFrameworks{ + Frameworks: []string{}, + Requirements: []string{}, + }, + }, + } + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(response); err != nil { + log.Printf("Error encoding response: %v", err) + } + } +} + +func handleHealth(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + fmt.Fprintf(w, `{"status":"ok"}`) +} diff --git a/tests/integration/storage/storage_suite_test.go b/tests/integration/storage/storage_suite_test.go new file mode 100644 index 00000000..82c00756 --- /dev/null +++ b/tests/integration/storage/storage_suite_test.go @@ -0,0 +1,47 @@ +package storage_test + +import ( + "fmt" + "net/http" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/complytime/complybeacon/tests/integration" +) + +var ( + webhookURL string + lokiURL string + s3URL string + s3Bucket string +) + +func TestStorage(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Storage Layer Suite") +} + +var _ = BeforeSuite(func() { + webhookURL = integration.EnvOrDefault("WEBHOOK_URL", "http://localhost:8088") + lokiURL = integration.EnvOrDefault("LOKI_URL", "http://localhost:3100") + s3URL = integration.EnvOrDefault("S3_URL", "http://localhost:9000") + s3Bucket = integration.EnvOrDefault("S3_BUCKET", "complybeacon-evidence") + + client := &http.Client{Timeout: 2 * time.Second} + resp, err := client.Get(webhookURL + "/eventreceiver/healthcheck") + if err != nil || resp.StatusCode != http.StatusOK { + msg := fmt.Sprintf( + "Stack not running — webhook healthcheck at %s failed.\n"+ + "Start it with: task integration:up PROFILE=storage\n", + webhookURL+"/eventreceiver/healthcheck", + ) + if err != nil { + msg += fmt.Sprintf("Error: %v\n", err) + } + Fail(msg) + } + resp.Body.Close() +}) diff --git a/tests/integration/storage/storage_test.go b/tests/integration/storage/storage_test.go new file mode 100644 index 00000000..26f6c453 --- /dev/null +++ b/tests/integration/storage/storage_test.go @@ -0,0 +1,52 @@ +package storage_test + +import ( + "net/http" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/complytime/complybeacon/tests/integration" +) + +var _ = Describe("Storage Layer", func() { + Describe("S3 Evidence Export", func() { + It("exports evidence objects to S3", func() { + resp, err := integration.PostEvidence(webhookURL, "../fixtures/evidence-fail.json") + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + + Eventually(func() ([]string, error) { + return integration.ListS3Objects(s3URL, s3Bucket, "github_branch_protection/") + }, 30*time.Second, 3*time.Second).Should( + ContainElement(ContainSubstring("evidence_")), + ) + }) + + It("partitions by policy_rule_id", func() { + resp1, err := integration.PostEvidence(webhookURL, "../fixtures/evidence-fail.json") + Expect(err).NotTo(HaveOccurred()) + defer resp1.Body.Close() + Expect(resp1.StatusCode).To(Equal(http.StatusOK)) + + resp2, err := integration.PostEvidence(webhookURL, "../fixtures/evidence-pass.json") + Expect(err).NotTo(HaveOccurred()) + defer resp2.Body.Close() + Expect(resp2.StatusCode).To(Equal(http.StatusOK)) + + Eventually(func() ([]string, error) { + return integration.ListS3Objects(s3URL, s3Bucket, "github_branch_protection/") + }, 30*time.Second, 3*time.Second).Should( + ContainElement(ContainSubstring("evidence_")), + ) + + Eventually(func() ([]string, error) { + return integration.ListS3Objects(s3URL, s3Bucket, "code_review/") + }, 30*time.Second, 3*time.Second).Should( + ContainElement(ContainSubstring("evidence_")), + ) + }) + }) +}) diff --git a/tests/integration/storage_tls/storage_tls_suite_test.go b/tests/integration/storage_tls/storage_tls_suite_test.go new file mode 100644 index 00000000..d7e91a37 --- /dev/null +++ b/tests/integration/storage_tls/storage_tls_suite_test.go @@ -0,0 +1,43 @@ +package storage_tls_test + +import ( + "fmt" + "net/http" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/complytime/complybeacon/tests/integration" +) + +var ( + webhookURL string + s3Bucket string +) + +func TestStorageTLS(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Storage TLS Layer Suite") +} + +var _ = BeforeSuite(func() { + webhookURL = integration.EnvOrDefault("WEBHOOK_URL", "http://localhost:8088") + s3Bucket = integration.EnvOrDefault("S3_BUCKET", "complybeacon-evidence") + + client := &http.Client{Timeout: 2 * time.Second} + resp, err := client.Get(webhookURL + "/eventreceiver/healthcheck") + if err != nil || resp.StatusCode != http.StatusOK { + msg := fmt.Sprintf( + "Stack not running — webhook healthcheck at %s failed.\n"+ + "Start it with: task integration:up PROFILE=storage-tls\n", + webhookURL+"/eventreceiver/healthcheck", + ) + if err != nil { + msg += fmt.Sprintf("Error: %v\n", err) + } + Fail(msg) + } + resp.Body.Close() +}) diff --git a/tests/integration/storage_tls/storage_tls_test.go b/tests/integration/storage_tls/storage_tls_test.go new file mode 100644 index 00000000..0687edef --- /dev/null +++ b/tests/integration/storage_tls/storage_tls_test.go @@ -0,0 +1,55 @@ +package storage_tls_test + +import ( + "net/http" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/complytime/complybeacon/tests/integration" +) + +var _ = Describe("Storage TLS Layer", func() { + Describe("TLS S3 Export", func() { + It("exports evidence to S3 over TLS", func() { + resp, err := integration.PostEvidence(webhookURL, "../fixtures/evidence-fail.json") + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + + Eventually(func() (string, error) { + return integration.ExecInContainer("rustfs-init-tls", + "rc", "ls", "--insecure", + "local/"+s3Bucket+"/github_branch_protection/", + ) + }, 30*time.Second, 3*time.Second).Should(ContainSubstring("evidence_")) + }) + + It("partitions by policy_rule_id over TLS", func() { + resp1, err := integration.PostEvidence(webhookURL, "../fixtures/evidence-fail.json") + Expect(err).NotTo(HaveOccurred()) + defer resp1.Body.Close() + Expect(resp1.StatusCode).To(Equal(http.StatusOK)) + + resp2, err := integration.PostEvidence(webhookURL, "../fixtures/evidence-pass.json") + Expect(err).NotTo(HaveOccurred()) + defer resp2.Body.Close() + Expect(resp2.StatusCode).To(Equal(http.StatusOK)) + + Eventually(func() (string, error) { + return integration.ExecInContainer("rustfs-init-tls", + "rc", "ls", "--insecure", + "local/"+s3Bucket+"/github_branch_protection/", + ) + }, 30*time.Second, 3*time.Second).Should(ContainSubstring("evidence_")) + + Eventually(func() (string, error) { + return integration.ExecInContainer("rustfs-init-tls", + "rc", "ls", "--insecure", + "local/"+s3Bucket+"/code_review/", + ) + }, 30*time.Second, 3*time.Second).Should(ContainSubstring("evidence_")) + }) + }) +}) diff --git a/truthbeam/go.mod b/truthbeam/go.mod index 21c4f2a0..ba82c531 100644 --- a/truthbeam/go.mod +++ b/truthbeam/go.mod @@ -1,8 +1,6 @@ module github.com/complytime/complybeacon/truthbeam -go 1.25.8 - -toolchain go1.25.9 +go 1.26.3 tool github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen @@ -59,6 +57,7 @@ require ( github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect github.com/onsi/ginkgo v1.16.5 // indirect + github.com/onsi/gomega v1.40.0 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pierrec/lz4/v4 v4.1.26 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -102,7 +101,7 @@ require ( golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.43.0 // indirect golang.org/x/text v0.36.0 // indirect - golang.org/x/tools v0.43.0 // indirect + golang.org/x/tools v0.44.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 // indirect google.golang.org/grpc v1.80.0 // indirect google.golang.org/protobuf v1.36.11 // indirect diff --git a/truthbeam/go.sum b/truthbeam/go.sum index f2160343..edb55e52 100644 --- a/truthbeam/go.sum +++ b/truthbeam/go.sum @@ -135,8 +135,9 @@ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.40.0 h1:Vtol0e1MghCD2ZVIilPDIg44XSL9l2QAn8ZNaljWcJc= +github.com/onsi/gomega v1.40.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= @@ -309,8 +310,8 @@ golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= -golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 295a2070fffb502812acbe7f2af0ceea09bbd684 Mon Sep 17 00:00:00 2001 From: sonupreetam Date: Mon, 25 May 2026 12:36:24 +0200 Subject: [PATCH 2/2] ci: add workflow_dispatch for UBI build verification --- .github/workflows/verify-ubi-build.yml | 29 ++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/verify-ubi-build.yml diff --git a/.github/workflows/verify-ubi-build.yml b/.github/workflows/verify-ubi-build.yml new file mode 100644 index 00000000..17e2b729 --- /dev/null +++ b/.github/workflows/verify-ubi-build.yml @@ -0,0 +1,29 @@ +name: Verify UBI Build +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + build-collector: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Build collector image with UBI Containerfile + run: | + echo "=== Building collector with UBI10 Containerfile ===" + podman build -f beacon-distro/Containerfile.collector \ + -t complybeacon/collector:verify \ + beacon-distro/ 2>&1 + + - name: Verify image + run: | + echo "=== Image details ===" + podman inspect complybeacon/collector:verify --format '{{.Os}}/{{.Architecture}}' + echo "=== Check CA bundle exists ===" + podman run --rm complybeacon/collector:verify \ + ls -la /etc/pki/tls/certs/ca-bundle.crt + echo "=== Collector version ===" + podman run --rm complybeacon/collector:verify --version