Repository navigation
feat: add native NetFlow monitoring with Lucene search and Sankey #3787
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Docker Build | |
| # Verifies the release Docker images actually build, without pushing. | |
| # A source-level type check (`make ci-lint`) cannot catch build failures that | |
| # only surface inside the image — e.g. a declaration file that exists in the | |
| # repo but was never COPYed into the Docker build stage. | |
| # | |
| # On pull requests each image is built only when files that affect it change, | |
| # so unrelated PRs stay fast. | |
| # | |
| # On pushes to main every image is built regardless of what changed. The path | |
| # filter is the wrong instrument there, because these builds can break with no | |
| # repo change at all: the Dockerfile pins floating upstream tags | |
| # (`node:${NODE_VERSION}-alpine`, `clickhouse-server:${CLICKHOUSE_VERSION}-alpine`) | |
| # and the registry re-points them whenever upstream publishes a patch. That is | |
| # exactly how the 26.8-alpine break reached a release: the tag moved from | |
| # ClickHouse 26.8.6 to 26.8.7 mid-afternoon, upgrading apk-tools 3.0.6 -> 3.0.8 | |
| # underneath us. The PR check and the nightly had both already passed against | |
| # the older digest, two commits then merged to main with no image build between | |
| # them, and the failure only appeared in the release run hours later. Building | |
| # unconditionally on main turns that into a red build on the next merge. | |
| on: | |
| pull_request: | |
| branches: [main] | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| concurrency: | |
| # Keyed on the event as well as the ref to separate a manual run on main from | |
| # a merge: both carry `refs/heads/main`, so they would otherwise contend for | |
| # one group. (A pull request run never collides with either — its ref is | |
| # `refs/pull/<n>/merge`.) | |
| # | |
| # Pushes to main do not cancel in progress, so a merge cannot kill the build | |
| # verifying the commit before it. This guarantees the tip of main gets built, | |
| # not that every individual commit does: GitHub holds only one pending run per | |
| # group, so a third merge arriving during a long build supersedes the queued | |
| # one. That is fine for what this job is for — an upstream base-image move | |
| # affects every commit equally, and a Dockerfile break in a superseded commit | |
| # is still present at the tip. Per-commit attribution comes from the pull | |
| # request run, not from here. | |
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| permissions: | |
| contents: read | |
| jobs: | |
| # OTel Collector image (Go build via OCB) — docker/otel-collector/Dockerfile. | |
| otel-collector-image: | |
| name: Build OTel Collector Image | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 25 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Get changed files | |
| id: changed-files | |
| if: github.event_name == 'pull_request' | |
| uses: tj-actions/changed-files@v47.0.6 | |
| with: | |
| files: | | |
| docker/otel-collector/** | |
| packages/otel-collector/** | |
| - name: Setup Docker Buildx | |
| if: | |
| github.event_name != 'pull_request' || | |
| steps.changed-files.outputs.any_changed == 'true' | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build (no push) | |
| if: | |
| github.event_name != 'pull_request' || | |
| steps.changed-files.outputs.any_changed == 'true' | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: ./docker/otel-collector/Dockerfile | |
| target: prod | |
| platforms: linux/amd64 | |
| push: false | |
| cache-from: | | |
| type=gha,scope=otel-collector-nightly-amd64 | |
| type=gha,scope=docker-build-pr-otel-collector | |
| cache-to: type=gha,mode=max,scope=docker-build-pr-otel-collector | |
| # App/prod image (API + App + common-utils Node build) — docker/hyperdx target prod. | |
| # This is the image the nightly css.d.ts failure surfaced in; the same builder | |
| # stage backs the all-in-one targets below. | |
| app-image: | |
| name: Build App Image | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 25 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Get changed files | |
| id: changed-files | |
| if: github.event_name == 'pull_request' | |
| uses: tj-actions/changed-files@v47.0.6 | |
| with: | |
| files: | | |
| packages/api/** | |
| packages/app/** | |
| packages/common-utils/** | |
| docker/hyperdx/** | |
| package.json | |
| yarn.lock | |
| .yarn/** | |
| .yarnrc.yml | |
| .prettierrc | |
| .prettierignore | |
| tsconfig.base.json | |
| nx.json | |
| - name: Setup Docker Buildx | |
| if: | |
| github.event_name != 'pull_request' || | |
| steps.changed-files.outputs.any_changed == 'true' | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build (no push) | |
| if: | |
| github.event_name != 'pull_request' || | |
| steps.changed-files.outputs.any_changed == 'true' | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: ./docker/hyperdx/Dockerfile | |
| target: prod | |
| platforms: linux/amd64 | |
| push: false | |
| build-contexts: | | |
| hyperdx=./docker/hyperdx | |
| api=./packages/api | |
| app=./packages/app | |
| build-args: | | |
| CODE_VERSION=pr-${{ github.event.pull_request.number || github.run_id }} | |
| cache-from: | | |
| type=gha,scope=app-nightly-amd64 | |
| type=gha,scope=docker-build-pr-app | |
| cache-to: type=gha,mode=max,scope=docker-build-pr-app | |
| # All-in-one image (adds ClickHouse, MongoDB, OTel Collector to the app build) | |
| # — docker/hyperdx targets all-in-one-auth AND all-in-one-noauth (the Local | |
| # image). The two are siblings off the shared all-in-one-base, each with its | |
| # own final COPY of an entry script (entry.local.auth.sh / .noauth.sh), so | |
| # building auth alone would NOT catch a break in the noauth-specific layer — | |
| # which is what surfaced in the nightly. Both are built here; the base is | |
| # built once and reused from buildx's local cache for the second target, so | |
| # the noauth build only adds its one tiny COPY layer. | |
| # | |
| # This target's base does `COPY --from=prod /app /app`, so it already | |
| # rebuilds the entire App/prod image. To avoid duplicating that ~8-minute | |
| # Node build on every app change, the pull-request path filter below is | |
| # scoped ONLY to the incremental bundling inputs (ClickHouse / OTel / the | |
| # shared Dockerfile) — NOT api/app/common-utils/deps. Those are covered by | |
| # the App Image job above, and they can't break the bundling layers, which | |
| # only COPY the prod output. Pushes to main ignore the filter entirely. | |
| all-in-one-image: | |
| name: Build All-in-One Image | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 40 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Get changed files | |
| id: changed-files | |
| if: github.event_name == 'pull_request' | |
| uses: tj-actions/changed-files@v47.0.6 | |
| with: | |
| files: | | |
| packages/otel-collector/** | |
| docker/hyperdx/** | |
| docker/clickhouse/** | |
| docker/otel-collector/** | |
| .vex/** | |
| - name: Setup Docker Buildx | |
| if: | |
| github.event_name != 'pull_request' || | |
| steps.changed-files.outputs.any_changed == 'true' | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build all-in-one-auth (no push) | |
| if: | |
| github.event_name != 'pull_request' || | |
| steps.changed-files.outputs.any_changed == 'true' | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: ./docker/hyperdx/Dockerfile | |
| target: all-in-one-auth | |
| platforms: linux/amd64 | |
| push: false | |
| build-contexts: | | |
| clickhouse=./docker/clickhouse | |
| otel-collector=./docker/otel-collector | |
| hyperdx=./docker/hyperdx | |
| api=./packages/api | |
| app=./packages/app | |
| build-args: | | |
| CODE_VERSION=pr-${{ github.event.pull_request.number || github.run_id }} | |
| cache-from: | | |
| type=gha,scope=all-in-one-nightly-amd64 | |
| type=gha,scope=docker-build-pr-all-in-one | |
| cache-to: type=gha,mode=max,scope=docker-build-pr-all-in-one | |
| # noauth is a sibling of auth off the shared all-in-one-base; the base is | |
| # already in buildx's local cache from the step above, so this only builds | |
| # the final noauth COPY layer. | |
| - name: Build all-in-one-noauth / Local (no push) | |
| if: | |
| github.event_name != 'pull_request' || | |
| steps.changed-files.outputs.any_changed == 'true' | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: ./docker/hyperdx/Dockerfile | |
| target: all-in-one-noauth | |
| platforms: linux/amd64 | |
| push: false | |
| load: true | |
| tags: hyperdx-aio-smoke:pr | |
| build-contexts: | | |
| clickhouse=./docker/clickhouse | |
| otel-collector=./docker/otel-collector | |
| hyperdx=./docker/hyperdx | |
| api=./packages/api | |
| app=./packages/app | |
| build-args: | | |
| CODE_VERSION=pr-${{ github.event.pull_request.number || github.run_id }} | |
| cache-from: | | |
| type=gha,scope=all-in-one-nightly-amd64 | |
| type=gha,scope=docker-build-pr-all-in-one | |
| # Boot the freshly built Local image and push one OTLP log end-to-end into | |
| # ClickHouse. Proves the artefact we ship actually runs, not just builds. | |
| - name: Smoke test the built image | |
| if: | |
| github.event_name != 'pull_request' || | |
| steps.changed-files.outputs.any_changed == 'true' | |
| timeout-minutes: 10 | |
| run: scripts/ci/smoke-all-in-one.sh hyperdx-aio-smoke:pr |