From 58bbd365eb0ea4ec7759bd8e5523c1b4bf09fe77 Mon Sep 17 00:00:00 2001 From: Thomas Powell <104777878+tpowell-progress@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:33:27 -0400 Subject: [PATCH 1/8] Add local_build_gems.ps1 to run build_gems.ps1 in a Windows Server 2022 Core container Adds docker/windows2022-core/Dockerfile (Windows Server 2022 Core + Chocolatey/Ruby/Git baseline) and .expeditor/local_build_gems.ps1, which builds that image and runs .expeditor/build_gems.ps1 inside a container with the repo mounted read-write at C:\workdir and a host output directory mounted at C:\output, copying resulting Habitat/gem/DLL artifacts to the output directory. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Thomas Powell <104777878+tpowell-progress@users.noreply.github.com> --- .expeditor/local_build_gems.ps1 | 198 +++++++++++++++++++++++++++++ README.md | 13 ++ docker/windows2022-core/Dockerfile | 40 ++++++ 3 files changed, 251 insertions(+) create mode 100644 .expeditor/local_build_gems.ps1 create mode 100644 docker/windows2022-core/Dockerfile diff --git a/.expeditor/local_build_gems.ps1 b/.expeditor/local_build_gems.ps1 new file mode 100644 index 00000000..cd3f4859 --- /dev/null +++ b/.expeditor/local_build_gems.ps1 @@ -0,0 +1,198 @@ +#!/usr/bin/env powershell + +#Requires -Version 5 + +<# +.SYNOPSIS + Reproduces the ".expeditor/build_gems.ps1" Buildkite build locally by + spinning up a disposable Windows Server 2022 Core container and running + the build inside it, instead of on your workstation directly. + +.DESCRIPTION + 1. Builds (or reuses) a Windows Server 2022 Core-based Docker image + (docker/windows2022-core/Dockerfile) containing just Ruby, Git, and + Chocolatey - the same baseline the Buildkite `rubydistros/windows-*` + agents provide. Everything else (Habitat, Chef-Client, Node.js, etc.) + is installed by build_gems.ps1 itself at build time, exactly like CI. + 2. Runs that image as a container with two volumes mounted: + - the repository root, read-write, at C:\workdir + - -OutputPath (created if needed), at C:\output + 3. Executes .expeditor\build_gems.ps1 inside the container. + 4. Copies the resulting Habitat artifacts and gem/DLL output out of the + container's C:\workdir into C:\output (i.e. -OutputPath on the host), + so build artifacts are easy to find without digging through the repo + working tree or a since-removed container. + +.PARAMETER OutputPath + Host directory to mount as the container's build output location. Created + if it doesn't already exist. Defaults to ".\build-output" under the repo. + +.PARAMETER RubyVersion + Ruby version to install in the image, matching one of the Ruby versions + build_gems.ps1 is verified against in .expeditor/verify.pipeline.yml. + +.PARAMETER ImageTag + Tag to build/reuse for the Windows Server 2022 Core build image. + +.PARAMETER NoCache + Forces a full rebuild of the Docker image instead of using layer cache. + +.PARAMETER Isolation + Container isolation mode to use ("process" or "hyperv"). "process" only + works when the container base image version matches the host OS build + exactly, so it defaults to "hyperv" for broader compatibility. + +.PARAMETER SkipImageBuild + Skips (re)building the Docker image and just runs -ImageTag as-is. Useful + once you already have an image built and are iterating on build_gems.ps1. + +.EXAMPLE + .\.expeditor\local_build_gems.ps1 + +.EXAMPLE + .\.expeditor\local_build_gems.ps1 -OutputPath C:\chef-powershell-shim-output -RubyVersion 3.1.7 +#> + +[CmdletBinding()] +param( + [Parameter()] [string] $OutputPath, + [Parameter()] [string] $RubyVersion = '3.4.8', + [Parameter()] [string] $ImageTag = 'chef-powershell-shim/windows2022-core:latest', + [Parameter()] [switch] $NoCache, + [Parameter()] [ValidateSet('process', 'hyperv')] [string] $Isolation = 'hyperv', + [Parameter()] [switch] $SkipImageBuild +) + +$ErrorActionPreference = 'Stop' + +function Write-Step { + param([string] $Message) + Write-Output "--- :whale: $Message" +} + +if (-not (Get-Command docker -ErrorAction SilentlyContinue)) { + throw 'Docker CLI not found. Install Docker Desktop (with Windows container support) and retry.' +} + +$dockerOsType = (docker info --format '{{.OSType}}' 2>$null) +if ($dockerOsType -ne 'windows') { + throw "Docker is currently configured for '$dockerOsType' containers. Switch Docker Desktop to Windows containers and retry." +} + +Write-Step 'Resolving project root' +$projectRoot = "$(git rev-parse --show-toplevel)" +if (-not $projectRoot) { + throw 'Unable to resolve the git project root. Run this script from within the chef-powershell-shim repo.' +} +$projectRoot = (Resolve-Path $projectRoot).Path +Write-Output "Project root: $projectRoot" +Write-Output "`r" + +if (-not $OutputPath) { + $OutputPath = Join-Path $projectRoot 'build-output' +} +New-Item -ItemType Directory -Path $OutputPath -Force | Out-Null +$OutputPath = (Resolve-Path $OutputPath).Path +Write-Output "Output directory (mounted at C:\output in the container): $OutputPath" +Write-Output "`r" + +$dockerfilePath = Join-Path $projectRoot 'docker\windows2022-core\Dockerfile' +if (-not (Test-Path -Path $dockerfilePath -PathType Leaf)) { + throw "Dockerfile not found at '$dockerfilePath'" +} + +if (-not $SkipImageBuild) { + Write-Step "Building the Windows Server 2022 Core build image ($ImageTag)" + $buildArgs = @( + 'build', + '--file', $dockerfilePath, + '--tag', $ImageTag, + '--build-arg', "RUBY_VERSION=$RubyVersion", + '--isolation', $Isolation + ) + if ($NoCache) { + $buildArgs += '--no-cache' + } + $buildArgs += (Split-Path -Path $dockerfilePath -Parent) + + & docker @buildArgs + if ($LASTEXITCODE -ne 0) { + throw "Docker image build failed with exit code $LASTEXITCODE" + } + Write-Output "`r" +} +else { + Write-Step "Skipping image build, reusing existing image ($ImageTag)" + Write-Output "`r" +} + +# The bulk of the actual build work (installing Habitat/Chef-Client/Node, +# building the Habitat package, building the gem) is delegated to +# build_gems.ps1 itself so this container reproduces CI exactly. After it +# completes (or fails), copy whatever build artifacts exist into C:\output +# so a failed run still leaves useful output/logs behind for inspection. +$containerCommand = @' +$ErrorActionPreference = "Continue" +Set-Location C:\workdir + +$buildFailed = $false +try { + powershell -ExecutionPolicy Bypass -File .expeditor\build_gems.ps1 + if ($LASTEXITCODE -ne 0) { throw "build_gems.ps1 exited with code $LASTEXITCODE" } +} catch { + Write-Warning "build_gems.ps1 failed: $_" + $buildFailed = $true +} + +Write-Output "--- :outbox_tray: Copying build artifacts to C:\output" +New-Item -ItemType Directory -Path C:\output -Force | Out-Null + +if (Test-Path C:\workdir\results) { + Copy-Item C:\workdir\results C:\output\results -Recurse -Force +} + +$gemFiles = Get-ChildItem -Path C:\workdir\chef-powershell\*.gem -ErrorAction SilentlyContinue +if ($gemFiles) { + New-Item -ItemType Directory -Path C:\output\gem -Force | Out-Null + Copy-Item $gemFiles.FullName -Destination C:\output\gem -Force +} + +$binFolder = "C:\workdir\chef-powershell\bin\ruby_bin_folder" +if (Test-Path $binFolder) { + Copy-Item $binFolder C:\output\ruby_bin_folder -Recurse -Force +} + +if ($buildFailed) { exit 1 } else { exit 0 } +'@ + +Write-Step 'Running build_gems.ps1 inside the container' +$dockerRunArgs = @( + 'run', + '--rm', + '--isolation', $Isolation, + '--volume', "${projectRoot}:C:\workdir", + '--volume', "${OutputPath}:C:\output", + '--workdir', 'C:\workdir' +) + +foreach ($envVar in @('HAB_AUTH_TOKEN', 'GEM_HOST_API_KEY', 'CHEF_POWERSHELL_VERSION_UPDATE')) { + $value = [Environment]::GetEnvironmentVariable($envVar, 'Process') + if (-not [string]::IsNullOrWhiteSpace($value)) { + $dockerRunArgs += '-e' + $dockerRunArgs += "$envVar=$value" + } +} + +$dockerRunArgs += $ImageTag +$dockerRunArgs += @('powershell', '-NoLogo', '-ExecutionPolicy', 'Bypass', '-Command', $containerCommand) + +& docker @dockerRunArgs +$buildExitCode = $LASTEXITCODE + +Write-Output "`r" +if ($buildExitCode -ne 0) { + Write-Output "--- :x: Build failed (exit code $buildExitCode). Partial artifacts, if any, are available under: $OutputPath" + throw "local build_gems.ps1 run failed with exit code $buildExitCode" +} + +Write-Output "--- :white_check_mark: Build complete. Artifacts available under: $OutputPath" diff --git a/README.md b/README.md index 55d7154e..340e01ff 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,19 @@ A .NET Assembly to facilitate communication between Chef and PowerShell on the W Binaries can be built with Habitat. See the PowerShell script `.\.expeditor\build_gems.ps1` to test your changes locally. +If you'd rather not install Habitat, Chef-Client, and the other build dependencies directly on your +workstation, `.\.expeditor\local_build_gems.ps1` runs the exact same `build_gems.ps1` build inside a +disposable Windows Server 2022 Core Docker container (Docker Desktop with Windows containers enabled +is required). It mounts your repo checkout read-write into the container and copies the resulting +Habitat artifacts, built gem, and compiled DLLs out to an output directory on your host (`.\build-output` +by default): + +```powershell +.\.expeditor\local_build_gems.ps1 +# or, to control the Ruby version / output location: +.\.expeditor\local_build_gems.ps1 -RubyVersion 3.1.7 -OutputPath C:\chef-powershell-shim-output +``` + You will need to have the following things installed: 1) .net framework 4.8.1 development pack 2) Windows 11 SDK build 26100 diff --git a/docker/windows2022-core/Dockerfile b/docker/windows2022-core/Dockerfile new file mode 100644 index 00000000..36f1b705 --- /dev/null +++ b/docker/windows2022-core/Dockerfile @@ -0,0 +1,40 @@ +# escape=` +# +# Windows Server 2022 Core build environment for chef-powershell-shim. +# +# This image intentionally stays minimal: `.expeditor\build_gems.ps1` itself +# installs Habitat, Chef-Client, Node.js, etc. at build time (exactly like the +# Buildkite CI agents do), so this Dockerfile only needs to provide the base +# OS plus the handful of things CI gets "for free" from the `rubydistros` +# image it runs on: Ruby, Git, and Chocolatey (used by build_gems.ps1 to +# install everything else). +ARG RUBY_VERSION=3.4.8 + +FROM mcr.microsoft.com/windows/servercore:ltsc2022 + +SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] + +# Chocolatey - used by build_gems.ps1 to install Habitat/Chef-Client/Node.js, +# and used here to install Ruby and Git. +RUN Set-ExecutionPolicy Bypass -Scope Process -Force ; ` + [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072 ; ` + iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) + +RUN choco feature enable -n allowGlobalConfirmation + +ARG RUBY_VERSION +RUN choco install git --no-progress ; ` + choco install ruby --version=$env:RUBY_VERSION --no-progress + +# Pick up choco/ruby/git on PATH for subsequent RUN/ENTRYPOINT steps. +RUN Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1 ; ` + Update-SessionEnvironment ; ` + ruby --version ; ` + git --version + +ENV HAB_LICENSE=accept-no-persist +ENV FORCE_FFI_YAJL=ext + +WORKDIR C:/workdir + +ENTRYPOINT ["powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass"] From 93377addf2544cbb24ed82324ee82e51ee852bc6 Mon Sep 17 00:00:00 2001 From: Thomas Powell <104777878+tpowell-progress@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:59:37 -0400 Subject: [PATCH 2/8] Fix Chocolatey ruby version resolution in windows2022-core image Chocolatey's 'ruby' package versions carry an extra packaging revision suffix (e.g. 3.4.10.1) that drifts over time, so an exact '3.4.8' isn't guaranteed to exist (and '3.1.7' never did). Resolve RUBY_VERSION as a major.minor prefix to the newest matching Chocolatey package version at image build time instead of hardcoding a full version string. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Thomas Powell <104777878+tpowell-progress@users.noreply.github.com> --- .expeditor/local_build_gems.ps1 | 10 ++++++---- README.md | 2 +- docker/windows2022-core/Dockerfile | 17 +++++++++++++++-- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/.expeditor/local_build_gems.ps1 b/.expeditor/local_build_gems.ps1 index cd3f4859..a49e6578 100644 --- a/.expeditor/local_build_gems.ps1 +++ b/.expeditor/local_build_gems.ps1 @@ -28,8 +28,10 @@ if it doesn't already exist. Defaults to ".\build-output" under the repo. .PARAMETER RubyVersion - Ruby version to install in the image, matching one of the Ruby versions - build_gems.ps1 is verified against in .expeditor/verify.pipeline.yml. + Major.minor Ruby version to install in the image (e.g. "3.4" or "3.1"), + matching one of the Ruby lines build_gems.ps1 is verified against in + .expeditor/verify.pipeline.yml. Resolved to the newest matching Chocolatey + "ruby" package version at image build time. .PARAMETER ImageTag Tag to build/reuse for the Windows Server 2022 Core build image. @@ -50,13 +52,13 @@ .\.expeditor\local_build_gems.ps1 .EXAMPLE - .\.expeditor\local_build_gems.ps1 -OutputPath C:\chef-powershell-shim-output -RubyVersion 3.1.7 + .\.expeditor\local_build_gems.ps1 -OutputPath C:\chef-powershell-shim-output -RubyVersion 3.1 #> [CmdletBinding()] param( [Parameter()] [string] $OutputPath, - [Parameter()] [string] $RubyVersion = '3.4.8', + [Parameter()] [string] $RubyVersion = '3.4', [Parameter()] [string] $ImageTag = 'chef-powershell-shim/windows2022-core:latest', [Parameter()] [switch] $NoCache, [Parameter()] [ValidateSet('process', 'hyperv')] [string] $Isolation = 'hyperv', diff --git a/README.md b/README.md index 340e01ff..e790aa0c 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ by default): ```powershell .\.expeditor\local_build_gems.ps1 # or, to control the Ruby version / output location: -.\.expeditor\local_build_gems.ps1 -RubyVersion 3.1.7 -OutputPath C:\chef-powershell-shim-output +.\.expeditor\local_build_gems.ps1 -RubyVersion 3.1 -OutputPath C:\chef-powershell-shim-output ``` You will need to have the following things installed: diff --git a/docker/windows2022-core/Dockerfile b/docker/windows2022-core/Dockerfile index 36f1b705..42054b99 100644 --- a/docker/windows2022-core/Dockerfile +++ b/docker/windows2022-core/Dockerfile @@ -8,7 +8,13 @@ # OS plus the handful of things CI gets "for free" from the `rubydistros` # image it runs on: Ruby, Git, and Chocolatey (used by build_gems.ps1 to # install everything else). -ARG RUBY_VERSION=3.4.8 +# +# RUBY_VERSION is a major.minor prefix (e.g. "3.4" or "3.1"), matching the +# Ruby lines .expeditor/verify.pipeline.yml is tested against. Chocolatey's +# "ruby" package versions are suffixed with an extra packaging revision +# (e.g. "3.4.10.1") that changes over time, so it's resolved to the newest +# matching package version at build time rather than hardcoded here. +ARG RUBY_VERSION=3.4 FROM mcr.microsoft.com/windows/servercore:ltsc2022 @@ -24,7 +30,14 @@ RUN choco feature enable -n allowGlobalConfirmation ARG RUBY_VERSION RUN choco install git --no-progress ; ` - choco install ruby --version=$env:RUBY_VERSION --no-progress + $feed = Invoke-RestMethod "https://community.chocolatey.org/api/v2/FindPackagesById()?id='ruby'" ; ` + $resolved = $feed | ForEach-Object { $_.properties.Version } | ` + Where-Object { $_ -eq $env:RUBY_VERSION -or $_ -like "$env:RUBY_VERSION.*" } | ` + Sort-Object { [version]($_ -replace '^(\d+\.\d+\.\d+)\.(\d+)$', '$1.$2') } | ` + Select-Object -Last 1 ; ` + if (-not $resolved) { throw "No Chocolatey 'ruby' package found matching version '$env:RUBY_VERSION'" } ; ` + Write-Output "Resolved Chocolatey ruby package version '$env:RUBY_VERSION' -> '$resolved'" ; ` + choco install ruby --version=$resolved --no-progress # Pick up choco/ruby/git on PATH for subsequent RUN/ENTRYPOINT steps. RUN Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1 ; ` From 8c2225849fb702395b5f7a20711858eac640c3e7 Mon Sep 17 00:00:00 2001 From: Thomas Powell <104777878+tpowell-progress@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:03:39 -0400 Subject: [PATCH 3/8] Move Chocolatey ruby-version resolution out of the inline RUN command Docker's Windows shell-form RUN concatenates the SHELL directive's prefix with the RUN command text into a single string before handing it to 'powershell -Command', which mangled the nested double/single quotes in the inline Chocolatey OData query one-liner (URL literal + regex + string interpolation). Move that logic into Install-Ruby.ps1, COPY'd into the image and invoked with 'powershell -File', which sidesteps the quoting collision entirely. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Thomas Powell <104777878+tpowell-progress@users.noreply.github.com> --- docker/windows2022-core/Dockerfile | 14 +++----- docker/windows2022-core/Install-Ruby.ps1 | 43 ++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 9 deletions(-) create mode 100644 docker/windows2022-core/Install-Ruby.ps1 diff --git a/docker/windows2022-core/Dockerfile b/docker/windows2022-core/Dockerfile index 42054b99..5b18cf25 100644 --- a/docker/windows2022-core/Dockerfile +++ b/docker/windows2022-core/Dockerfile @@ -28,16 +28,12 @@ RUN Set-ExecutionPolicy Bypass -Scope Process -Force ; ` RUN choco feature enable -n allowGlobalConfirmation +# Installs Git plus a Chocolatey "ruby" package resolved from RUBY_VERSION. +# Kept as an external script (rather than an inline RUN one-liner) to avoid +# Dockerfile/PowerShell quoting collisions - see Install-Ruby.ps1 for why. ARG RUBY_VERSION -RUN choco install git --no-progress ; ` - $feed = Invoke-RestMethod "https://community.chocolatey.org/api/v2/FindPackagesById()?id='ruby'" ; ` - $resolved = $feed | ForEach-Object { $_.properties.Version } | ` - Where-Object { $_ -eq $env:RUBY_VERSION -or $_ -like "$env:RUBY_VERSION.*" } | ` - Sort-Object { [version]($_ -replace '^(\d+\.\d+\.\d+)\.(\d+)$', '$1.$2') } | ` - Select-Object -Last 1 ; ` - if (-not $resolved) { throw "No Chocolatey 'ruby' package found matching version '$env:RUBY_VERSION'" } ; ` - Write-Output "Resolved Chocolatey ruby package version '$env:RUBY_VERSION' -> '$resolved'" ; ` - choco install ruby --version=$resolved --no-progress +COPY Install-Ruby.ps1 C:\Install-Ruby.ps1 +RUN powershell -ExecutionPolicy Bypass -File C:\Install-Ruby.ps1 -RubyVersion $env:RUBY_VERSION # Pick up choco/ruby/git on PATH for subsequent RUN/ENTRYPOINT steps. RUN Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1 ; ` diff --git a/docker/windows2022-core/Install-Ruby.ps1 b/docker/windows2022-core/Install-Ruby.ps1 new file mode 100644 index 00000000..3a18f2e2 --- /dev/null +++ b/docker/windows2022-core/Install-Ruby.ps1 @@ -0,0 +1,43 @@ +#!/usr/bin/env powershell + +<# +.SYNOPSIS + Installs Git and a Chocolatey "ruby" package matching a major.minor prefix. + +.DESCRIPTION + Chocolatey's "ruby" package versions carry an extra packaging-revision + suffix (e.g. "3.4.10.1") that changes over time, so an exact patch version + like "3.4.8" isn't guaranteed to exist. This resolves -RubyVersion (a + major.minor prefix such as "3.4" or "3.1") to the newest matching + Chocolatey package version at image build time instead of hardcoding one. + + Kept as a standalone script (rather than an inline `RUN` one-liner) so the + quoting needed for the Chocolatey OData query and PowerShell string/regex + literals doesn't have to survive being re-flattened through Docker's + Windows shell-form command concatenation. +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] [string] $RubyVersion +) + +$ErrorActionPreference = 'Stop' + +choco install git --no-progress +if ($LASTEXITCODE -ne 0) { throw "choco install git failed with exit code $LASTEXITCODE" } + +$feed = Invoke-RestMethod "https://community.chocolatey.org/api/v2/FindPackagesById()?id='ruby'" +$resolved = $feed | + ForEach-Object { $_.properties.Version } | + Where-Object { $_ -eq $RubyVersion -or $_ -like "$RubyVersion.*" } | + Sort-Object { [version]($_ -replace '^(\d+\.\d+\.\d+)\.(\d+)$', '$1.$2') } | + Select-Object -Last 1 + +if (-not $resolved) { + throw "No Chocolatey 'ruby' package found matching version '$RubyVersion'" +} +Write-Output "Resolved Chocolatey ruby package version '$RubyVersion' -> '$resolved'" + +choco install ruby --version=$resolved --no-progress +if ($LASTEXITCODE -ne 0) { throw "choco install ruby failed with exit code $LASTEXITCODE" } From ec3a09fddf0a7aeae06df9e3f627345acd4a695b Mon Sep 17 00:00:00 2001 From: Thomas Powell <104777878+tpowell-progress@users.noreply.github.com> Date: Fri, 11 Sep 2026 10:31:00 -0400 Subject: [PATCH 4/8] Add HAB_AUTH_TOKEN preflight check/param to local_build_gems.ps1 'hab pkg build' needs a Habitat Builder token to install chef/hab-studio and this project's Habitat build dependencies, failing with '401 Unauthorized' without one. Add a -HabAuthToken parameter (defaulting to \_Qk9YLTEKYmxkci0yMDE3MDkyNzAyMzcxNApibGRyLTIwMTcwOTI3MDIzNzE0Cnh4T1Y3WlhrSW9jWUkxdis1SGx1UnVWcXQyQkRKSU0rCkJTQUVQbU1OR3daejNEamhsM2k4SzQxTG05emRIV3c2Y0pDRmFwb1FZR05yQlhHcg==) that's forwarded into the container, plus an early warning when it's missing so the failure is obvious before the build proceeds. Documented in the README. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Thomas Powell <104777878+tpowell-progress@users.noreply.github.com> --- .expeditor/local_build_gems.ps1 | 27 +++++++++++++++++++++++++-- README.md | 5 +++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.expeditor/local_build_gems.ps1 b/.expeditor/local_build_gems.ps1 index a49e6578..842dec7b 100644 --- a/.expeditor/local_build_gems.ps1 +++ b/.expeditor/local_build_gems.ps1 @@ -48,6 +48,14 @@ Skips (re)building the Docker image and just runs -ImageTag as-is. Useful once you already have an image built and are iterating on build_gems.ps1. +.PARAMETER HabAuthToken + Habitat Builder personal access token, forwarded into the container as + HAB_AUTH_TOKEN. Required because `hab pkg build` needs to install + chef/hab-studio (and this project's Habitat build dependencies) from + Habitat Builder, which returns 401 Unauthorized for the 'chef' origin's + packages without a valid token. Defaults to $env:HAB_AUTH_TOKEN if set; + generate one at https://bldr.habitat.sh/#/profile if you don't have one. + .EXAMPLE .\.expeditor\local_build_gems.ps1 @@ -62,7 +70,8 @@ param( [Parameter()] [string] $ImageTag = 'chef-powershell-shim/windows2022-core:latest', [Parameter()] [switch] $NoCache, [Parameter()] [ValidateSet('process', 'hyperv')] [string] $Isolation = 'hyperv', - [Parameter()] [switch] $SkipImageBuild + [Parameter()] [switch] $SkipImageBuild, + [Parameter()] [string] $HabAuthToken = $env:HAB_AUTH_TOKEN ) $ErrorActionPreference = 'Stop' @@ -81,6 +90,15 @@ if ($dockerOsType -ne 'windows') { throw "Docker is currently configured for '$dockerOsType' containers. Switch Docker Desktop to Windows containers and retry." } +if ([string]::IsNullOrWhiteSpace($HabAuthToken)) { + Write-Warning ( + "HAB_AUTH_TOKEN is not set. 'hab pkg build' needs it to install chef/hab-studio and " + + "this project's Habitat build dependencies, and will fail with '401 Unauthorized' " + + "without it. Set `$env:HAB_AUTH_TOKEN or pass -HabAuthToken before retrying " + + '(see https://bldr.habitat.sh/#/profile to generate a token).' + ) +} + Write-Step 'Resolving project root' $projectRoot = "$(git rev-parse --show-toplevel)" if (-not $projectRoot) { @@ -177,7 +195,12 @@ $dockerRunArgs = @( '--workdir', 'C:\workdir' ) -foreach ($envVar in @('HAB_AUTH_TOKEN', 'GEM_HOST_API_KEY', 'CHEF_POWERSHELL_VERSION_UPDATE')) { +if (-not [string]::IsNullOrWhiteSpace($HabAuthToken)) { + $dockerRunArgs += '-e' + $dockerRunArgs += "HAB_AUTH_TOKEN=$HabAuthToken" +} + +foreach ($envVar in @('GEM_HOST_API_KEY', 'CHEF_POWERSHELL_VERSION_UPDATE')) { $value = [Environment]::GetEnvironmentVariable($envVar, 'Process') if (-not [string]::IsNullOrWhiteSpace($value)) { $dockerRunArgs += '-e' diff --git a/README.md b/README.md index e790aa0c..1c8d4dc8 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,11 @@ by default): .\.expeditor\local_build_gems.ps1 -RubyVersion 3.1 -OutputPath C:\chef-powershell-shim-output ``` +You'll need a Habitat Builder personal access token for this too - `hab pkg build` installs +chef/hab-studio and this project's Habitat build dependencies from Habitat Builder, which fails +with `401 Unauthorized` without one. Generate one at https://bldr.habitat.sh/#/profile, then either +set `$env:HAB_AUTH_TOKEN` before running the script or pass `-HabAuthToken`. + You will need to have the following things installed: 1) .net framework 4.8.1 development pack 2) Windows 11 SDK build 26100 From 2e0bfaf188122526fcae9edc54ca7673674bf0a1 Mon Sep 17 00:00:00 2001 From: Thomas Powell <104777878+tpowell-progress@users.noreply.github.com> Date: Fri, 11 Sep 2026 13:12:33 -0400 Subject: [PATCH 5/8] Read HAB_AUTH_TOKEN from ~/.env as a fallback in local_build_gems.ps1 Resolution order is now: -HabAuthToken parameter, then $env:HAB_AUTH_TOKEN, then a HAB_AUTH_TOKEN=... entry in ~/.env (a common place devs stash tokens for local tooling). Documented in the parameter help and README. Signed-off-by: Thomas Powell <104777878+tpowell-progress@users.noreply.github.com> --- .expeditor/local_build_gems.ps1 | 47 ++++++++++++++++++++++++++++++--- README.md | 2 +- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/.expeditor/local_build_gems.ps1 b/.expeditor/local_build_gems.ps1 index 842dec7b..9788241b 100644 --- a/.expeditor/local_build_gems.ps1 +++ b/.expeditor/local_build_gems.ps1 @@ -53,8 +53,9 @@ HAB_AUTH_TOKEN. Required because `hab pkg build` needs to install chef/hab-studio (and this project's Habitat build dependencies) from Habitat Builder, which returns 401 Unauthorized for the 'chef' origin's - packages without a valid token. Defaults to $env:HAB_AUTH_TOKEN if set; - generate one at https://bldr.habitat.sh/#/profile if you don't have one. + packages without a valid token. Resolved from, in order: this parameter, + $env:HAB_AUTH_TOKEN, then a HAB_AUTH_TOKEN=... entry in ~/.env. Generate + one at https://bldr.habitat.sh/#/profile if you don't have one. .EXAMPLE .\.expeditor\local_build_gems.ps1 @@ -81,6 +82,35 @@ function Write-Step { Write-Output "--- :whale: $Message" } +function Get-DotEnvValue { + # Minimal ~/.env reader: looks for a KEY=value (optionally quoted) line, + # ignoring blank lines and '#' comments. Does not mutate the process + # environment - just returns the value for this script to use directly. + param( + [Parameter(Mandatory = $true)] [string] $Path, + [Parameter(Mandatory = $true)] [string] $Key + ) + + if (-not (Test-Path -Path $Path -PathType Leaf)) { + return $null + } + + foreach ($line in Get-Content -Path $Path) { + $trimmed = $line.Trim() + if (-not $trimmed -or $trimmed.StartsWith('#')) { + continue + } + if ($trimmed -match "^(?:export\s+)?$([regex]::Escape($Key))\s*=\s*(.*)$") { + $value = $Matches[1].Trim() + if (($value.StartsWith('"') -and $value.EndsWith('"')) -or ($value.StartsWith("'") -and $value.EndsWith("'"))) { + $value = $value.Substring(1, $value.Length - 2) + } + return $value + } + } + return $null +} + if (-not (Get-Command docker -ErrorAction SilentlyContinue)) { throw 'Docker CLI not found. Install Docker Desktop (with Windows container support) and retry.' } @@ -90,12 +120,21 @@ if ($dockerOsType -ne 'windows') { throw "Docker is currently configured for '$dockerOsType' containers. Switch Docker Desktop to Windows containers and retry." } +if ([string]::IsNullOrWhiteSpace($HabAuthToken)) { + $dotEnvPath = Join-Path $HOME '.env' + $dotEnvToken = Get-DotEnvValue -Path $dotEnvPath -Key 'HAB_AUTH_TOKEN' + if (-not [string]::IsNullOrWhiteSpace($dotEnvToken)) { + Write-Output "Using HAB_AUTH_TOKEN from $dotEnvPath" + $HabAuthToken = $dotEnvToken + } +} + if ([string]::IsNullOrWhiteSpace($HabAuthToken)) { Write-Warning ( "HAB_AUTH_TOKEN is not set. 'hab pkg build' needs it to install chef/hab-studio and " + "this project's Habitat build dependencies, and will fail with '401 Unauthorized' " + - "without it. Set `$env:HAB_AUTH_TOKEN or pass -HabAuthToken before retrying " + - '(see https://bldr.habitat.sh/#/profile to generate a token).' + "without it. Set `$env:HAB_AUTH_TOKEN, add HAB_AUTH_TOKEN=... to ~/.env, or pass " + + '-HabAuthToken before retrying (see https://bldr.habitat.sh/#/profile to generate a token).' ) } diff --git a/README.md b/README.md index 1c8d4dc8..c7507224 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ by default): You'll need a Habitat Builder personal access token for this too - `hab pkg build` installs chef/hab-studio and this project's Habitat build dependencies from Habitat Builder, which fails with `401 Unauthorized` without one. Generate one at https://bldr.habitat.sh/#/profile, then either -set `$env:HAB_AUTH_TOKEN` before running the script or pass `-HabAuthToken`. +set `$env:HAB_AUTH_TOKEN`, add a `HAB_AUTH_TOKEN=...` line to `~/.env`, or pass `-HabAuthToken`. You will need to have the following things installed: 1) .net framework 4.8.1 development pack From 564009fa7caed71f0126d64f1b4a9b26a38280a1 Mon Sep 17 00:00:00 2001 From: Thomas Powell <104777878+tpowell-progress@users.noreply.github.com> Date: Mon, 14 Sep 2026 10:55:38 -0400 Subject: [PATCH 6/8] Install VC++ Redistributable in the local Docker build image The .NET Framework / Windows PowerShell side of the gem (Chef.PowerShell.Wrapper.dll) is a mixed-mode C++/CLI assembly built with the VS2022 (v145) toolset, which needs the matching Visual C++ Redistributable at load time. The minimal windows/servercore:ltsc2022 base image doesn't ship it, so ffi_lib failed with Win32 error 126 ('The specified module could not be found') even though the DLL itself was present on disk. Also surface bundle update failures immediately instead of silently continuing into gem_check. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Thomas Powell <104777878+tpowell-progress@users.noreply.github.com> --- .expeditor/build_gems.ps1 | 1 + docker/windows2022-core/Dockerfile | 9 +++++++++ docker/windows2022-core/Install-Ruby.ps1 | 21 +++++++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/.expeditor/build_gems.ps1 b/.expeditor/build_gems.ps1 index bcb9a060..00ee5876 100644 --- a/.expeditor/build_gems.ps1 +++ b/.expeditor/build_gems.ps1 @@ -183,6 +183,7 @@ Write-Output "`r" Write-Output "--- :point_right: finally verifying the gem code (chefstyle, spellcheck, spec)" bundle update +if (-not $?) { throw "Bundle update failed"} bundle exec rake gem_check if (-not $?) { throw "Bundle Gem failed"} diff --git a/docker/windows2022-core/Dockerfile b/docker/windows2022-core/Dockerfile index 5b18cf25..26ab729b 100644 --- a/docker/windows2022-core/Dockerfile +++ b/docker/windows2022-core/Dockerfile @@ -28,6 +28,15 @@ RUN Set-ExecutionPolicy Bypass -Scope Process -Force ; ` RUN choco feature enable -n allowGlobalConfirmation +# Chef.PowerShell.Wrapper.dll (the Windows PowerShell / .NET Framework side of +# this gem, built as a mixed-mode C++/CLI assembly with the VS2022 (v145) +# toolset) depends on the matching Visual C++ Redistributable at load time. +# The minimal servercore image doesn't ship it, so ffi_lib fails with Win32 +# error 126 ("The specified module could not be found") even though the DLL +# itself is present on disk - it's a transitive native dependency of the DLL, +# not the DLL itself, that's missing. +RUN choco install vcredist140 -y + # Installs Git plus a Chocolatey "ruby" package resolved from RUBY_VERSION. # Kept as an external script (rather than an inline RUN one-liner) to avoid # Dockerfile/PowerShell quoting collisions - see Install-Ruby.ps1 for why. diff --git a/docker/windows2022-core/Install-Ruby.ps1 b/docker/windows2022-core/Install-Ruby.ps1 index 3a18f2e2..d829368c 100644 --- a/docker/windows2022-core/Install-Ruby.ps1 +++ b/docker/windows2022-core/Install-Ruby.ps1 @@ -41,3 +41,24 @@ Write-Output "Resolved Chocolatey ruby package version '$RubyVersion' -> '$resol choco install ruby --version=$resolved --no-progress if ($LASTEXITCODE -ne 0) { throw "choco install ruby failed with exit code $LASTEXITCODE" } + +# The Chocolatey "ruby" package doesn't ship (or run) the MSYS2/MINGW devkit +# toolchain that "ridk install" normally sets up, and several gems pulled in +# below (json, racc, libyajl2, ffi, ffi-yajl) need it to compile native +# extensions. Buildkite's "rubydistros" agents provide this out of the box; +# this image needs to install it explicitly. "ridk install" is an +# interactive-only menu with no reliable non-interactive/piped mode on +# Windows, so instead install MSYS2 directly to C:\msys64 - one of the +# default locations RubyInstaller's Msys2Installation#iterate_msys_paths +# auto-detects - and provision the MINGW toolchain via pacman. +choco install msys2 -y --no-progress --params "/InstallDir:C:\msys64 /NoUpdate" +if ($LASTEXITCODE -ne 0) { throw "choco install msys2 failed with exit code $LASTEXITCODE" } + +$msysBash = "C:\msys64\usr\bin\bash.exe" + +# Ruby built for the x64-mingw-ucrt platform (what Chocolatey's ruby package +# targets) needs the "ucrt64" MINGW toolchain, not the older "mingw64" one. +& $msysBash -lc "pacman -Syu --noconfirm" +if ($LASTEXITCODE -ne 0) { throw "pacman -Syu failed with exit code $LASTEXITCODE" } +& $msysBash -lc "pacman -S --needed --noconfirm base-devel mingw-w64-ucrt-x86_64-toolchain" +if ($LASTEXITCODE -ne 0) { throw "pacman toolchain install failed with exit code $LASTEXITCODE" } From 562786c8c266ba8e53b09a979e5a8cec147e787c Mon Sep 17 00:00:00 2001 From: Thomas Powell <104777878+tpowell-progress@users.noreply.github.com> Date: Mon, 14 Sep 2026 13:20:35 -0400 Subject: [PATCH 7/8] Add a memory-pressure regression test for the heap-lifetime bug (PRs #187/#196/#205) PRs #187, #196, and #205 fixed a class of bug where the buffer the native wrapper used to hand a result back to Ruby could be collected/reused before Ruby finished reading it, silently corrupting or truncating the payload -- even for values as trivial as a bare boolean. It mostly showed up under memory pressure (e.g. constrained CI agents). Add chef-powershell/memory_pressure_test.rb, which round-trips a matrix of payload shapes (booleans, integers, floats, strings including multi-byte Unicode, arrays, hashtables, a large string) through both the :powershell and :pwsh interpreters many times, forcing a full GC before every single round-trip (or, opt-in via CHEF_POWERSHELL_GC_STRESS=1, the much slower GC.stress=true for the whole run), and asserts *exact* equality every time. Add .expeditor/local_low_memory_test.ps1, which runs that same script inside a memory-constrained (default 512MB) Windows Docker container -- reusing the windows2022-core build image -- against whatever DLLs are currently under chef-powershell/bin/ruby_bin_folder/ (i.e. the output of a local build_gems.ps1/local_build_gems.ps1 run), so the .NET/CLR side is put under genuine OS-level memory pressure too, not just the Ruby side. 256MB was observed to reliably OOM before a PowerShell runspace could even start, so 512MB is the floor used here. Document both in 'Testing chef-powershell with Chef-18 and Chef-19.md'. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Thomas Powell <104777878+tpowell-progress@users.noreply.github.com> --- .expeditor/local_low_memory_test.ps1 | 144 +++++++++ ...hef-powershell with Chef-18 and Chef-19.md | 35 +++ chef-powershell/memory_pressure_test.rb | 277 ++++++++++++++++++ 3 files changed, 456 insertions(+) create mode 100644 .expeditor/local_low_memory_test.ps1 create mode 100644 chef-powershell/memory_pressure_test.rb diff --git a/.expeditor/local_low_memory_test.ps1 b/.expeditor/local_low_memory_test.ps1 new file mode 100644 index 00000000..5fa3b9f8 --- /dev/null +++ b/.expeditor/local_low_memory_test.ps1 @@ -0,0 +1,144 @@ +#!/usr/bin/env powershell + +#Requires -Version 5 + +<# +.SYNOPSIS + Runs chef-powershell/memory_pressure_test.rb inside a memory-constrained Windows + container, using the DLLs produced by a local .expeditor/build_gems.ps1 run (see + local_build_gems.ps1), to guard against a regression of the heap-lifetime bug fixed + in PRs #187, #196, #205 (result payloads getting corrupted/lost on the way back from + the native wrapper to Ruby, particularly under memory pressure). + +.DESCRIPTION + 1. Requires that chef-powershell/bin/ruby_bin_folder/ already contains freshly + built DLLs (i.e. you've already run .\.expeditor\local_build_gems.ps1, or copied a + built Habitat package's bin/ over that folder yourself). + 2. Reuses the same minimal Windows Server 2022 Core + Ruby/Git/vcredist image that + local_build_gems.ps1 builds (docker/windows2022-core/Dockerfile) -- no Habitat/ + Chef-Client/Node install is needed just to run this Ruby-only test. + 3. Runs that image as a *memory-limited* container (-MemoryLimit, default 512MB) + with the repo mounted read-only at C:\workdir, and executes + chef-powershell/memory_pressure_test.rb inside it. The script itself additionally + forces Ruby-side GC churn (GC.stress + large discarded allocations) so both the + Ruby heap and, via reduced available system memory, the CLR heap the native + wrapper allocates from are put under realistic pressure. + 4. Surfaces the container's exit code as this script's exit code/throw, so it can be + wired into CI the same way build_gems.ps1's other verification steps are. + +.PARAMETER MemoryLimit + Docker `--memory` limit to apply to the test container, e.g. "384m" or "512m". Low + enough to force frequent GC on both the Ruby and CLR sides without being so low the + container fails to start PowerShell/.NET at all -- 256m was observed to reliably + OutOfMemoryException before a runspace could even come up (System.Management. + Automation itself needs real headroom), so 512m is the default floor. + +.PARAMETER ImageTag + Tag of the Windows Server 2022 Core build image to (re)use. Defaults to the same tag + local_build_gems.ps1 builds, so a prior local build run's image is reused as-is. + +.PARAMETER Isolation + Container isolation mode to use ("process" or "hyperv"). Defaults to "hyperv" for + broader host/image version compatibility -- see local_build_gems.ps1. + +.PARAMETER SkipImageBuild + Skips (re)building the Docker image and just runs -ImageTag as-is. + +.PARAMETER Iterations + Number of stress-test iterations to run inside the container (each iteration + round-trips the full payload matrix through both interpreters). Defaults to 40. + +.EXAMPLE + .\.expeditor\build_gems.ps1 # or local_build_gems.ps1, to produce fresh DLLs + .\.expeditor\local_low_memory_test.ps1 + +.EXAMPLE + .\.expeditor\local_low_memory_test.ps1 -MemoryLimit 128m -Iterations 100 +#> + +[CmdletBinding()] +param( + [Parameter()] [string] $MemoryLimit = '512m', + [Parameter()] [string] $ImageTag = 'chef-powershell-shim/windows2022-core:latest', + [Parameter()] [ValidateSet('process', 'hyperv')] [string] $Isolation = 'hyperv', + [Parameter()] [switch] $SkipImageBuild, + [Parameter()] [int] $Iterations = 40 +) + +$ErrorActionPreference = 'Stop' + +function Write-Step { + param([string] $Message) + Write-Output "--- :petri_dish: $Message" +} + +if (-not (Get-Command docker -ErrorAction SilentlyContinue)) { + throw 'Docker CLI not found. Install Docker Desktop (with Windows container support) and retry.' +} + +$dockerOsType = (docker info --format '{{.OSType}}' 2>$null) +if ($dockerOsType -ne 'windows') { + throw "Docker is currently configured for '$dockerOsType' containers. Switch Docker Desktop to Windows containers and retry." +} + +Write-Step 'Resolving project root' +$projectRoot = "$(git rev-parse --show-toplevel)" +if (-not $projectRoot) { + throw 'Unable to resolve the git project root. Run this script from within the chef-powershell-shim repo.' +} +$projectRoot = (Resolve-Path $projectRoot).Path +Write-Output "Project root: $projectRoot" +Write-Output "`r" + +$arch = if ($env:PROCESSOR_ARCHITECTURE) { $env:PROCESSOR_ARCHITECTURE } else { 'AMD64' } +$binDir = Join-Path $projectRoot "chef-powershell\bin\ruby_bin_folder\$arch" +$wrapperDll = Join-Path $binDir 'Chef.PowerShell.Wrapper.dll' +if (-not (Test-Path -Path $wrapperDll -PathType Leaf)) { + throw ( + "No locally built DLLs found at '$binDir'. Run '.\.expeditor\local_build_gems.ps1' " + + "(or '.\.expeditor\build_gems.ps1' directly) first to produce the DLLs this test " + + 'exercises -- this script intentionally does not build them itself.' + ) +} +Write-Output "Using locally built DLLs from: $binDir" +Write-Output "`r" + +$dockerfilePath = Join-Path $projectRoot 'docker\windows2022-core\Dockerfile' +if (-not (Test-Path -Path $dockerfilePath -PathType Leaf)) { + throw "Dockerfile not found at '$dockerfilePath'" +} + +if (-not $SkipImageBuild) { + Write-Step "Building/reusing the Windows Server 2022 Core test image ($ImageTag)" + & docker build --file $dockerfilePath --tag $ImageTag --isolation $Isolation (Split-Path -Path $dockerfilePath -Parent) + if ($LASTEXITCODE -ne 0) { + throw "Docker image build failed with exit code $LASTEXITCODE" + } + Write-Output "`r" +} +else { + Write-Step "Skipping image build, reusing existing image ($ImageTag)" + Write-Output "`r" +} + +Write-Step "Running memory_pressure_test.rb in a $MemoryLimit-limited container (isolation: $Isolation)" +$dockerRunArgs = @( + 'run', + '--rm', + '--isolation', $Isolation, + '--memory', $MemoryLimit, + '--volume', "${projectRoot}:C:\workdir:ro", + '--workdir', 'C:\workdir\chef-powershell', + $ImageTag, + 'ruby', 'memory_pressure_test.rb', "$Iterations" +) + +& docker @dockerRunArgs +$testExitCode = $LASTEXITCODE + +Write-Output "`r" +if ($testExitCode -ne 0) { + throw "memory_pressure_test.rb failed inside the $MemoryLimit-limited container (exit code $testExitCode) -- possible regression of the PR #187/#196/#205 heap-lifetime bug." +} + +Write-Output "--- :white_check_mark: All payload round-trips survived $MemoryLimit-limited-memory GC pressure intact." diff --git a/chef-powershell/Testing chef-powershell with Chef-18 and Chef-19.md b/chef-powershell/Testing chef-powershell with Chef-18 and Chef-19.md index 24a888b8..d019e1c4 100644 --- a/chef-powershell/Testing chef-powershell with Chef-18 and Chef-19.md +++ b/chef-powershell/Testing chef-powershell with Chef-18 and Chef-19.md @@ -95,6 +95,40 @@ This checks for the presence of: - `bin\host\fxr\10.0.0\hostfxr.dll` - VC++ CRT redist DLLs (`vcruntime140.dll`, `msvcp140.dll`, etc.) +### 1.4 Memory-pressure / heap-lifetime regression test + +PRs #187, #196, and #205 fixed a class of bug where the buffer the native wrapper used to hand a +result back to Ruby could be collected/reused before Ruby finished reading it -- silently +corrupting or truncating the payload, even for values as trivial as a bare boolean. This mostly +showed up under memory pressure (e.g. constrained CI agents). + +`memory_pressure_test.rb` guards against a regression of that bug: it round-trips a matrix of +payload shapes (booleans, integers, floats, strings including multi-byte Unicode, arrays, +hashtables, and a large string) through both interpreters, many times, forcing a full GC before +every single round-trip (or, opt-in, `GC.stress = true` for the whole run) and asserts *exact* +equality every time. + +```powershell +cd C:\localrepo\chef-powershell-shim\chef-powershell +ruby memory_pressure_test.rb # 15 iterations by default +ruby memory_pressure_test.rb 50 # or pass an iteration count +$env:CHEF_POWERSHELL_GC_STRESS = "1" # opt-in to the slower, more thorough GC.stress mode +ruby memory_pressure_test.rb +``` + +`.expeditor\local_low_memory_test.ps1` runs this same script inside a memory-constrained (default +512MB) Windows Docker container, using whatever DLLs are currently under +`chef-powershell\bin\ruby_bin_folder\` (i.e. the output of a local +`.expeditor\build_gems.ps1` / `local_build_gems.ps1` run), so the .NET/CLR side is also put under +genuine OS-level memory pressure, not just the Ruby side: + +```powershell +.\.expeditor\local_build_gems.ps1 # produces fresh DLLs under chef-powershell\bin\... +.\.expeditor\local_low_memory_test.ps1 # runs memory_pressure_test.rb in a 512MB container +``` + +Exit code 0 = every round-trip came back exact. + --- ## Part 2 — Testing against Chef-19 (Habitat only) @@ -538,6 +572,7 @@ hab pkg install C:\localrepo\chef-powershell-shim\results\chef-chef-powershell-s | Run RSpec unit tests | `bundle exec rake spec` (from `chef-powershell/`) | | Smoke test local Hab build | `bundle exec ruby smoke_test_dlls.rb \bin` | | Full layout verification | `bundle exec ruby verify_hab_build.rb \bin` | +| Memory-pressure / heap-lifetime regression test | `ruby memory_pressure_test.rb` (or `.expeditor\local_low_memory_test.ps1` for the low-memory container version) | | Test under Chef-19 Hab | `hab pkg exec chef/chef-infra-client ruby -e "..."` (`--channel stable` on install; replace vendored gem first, see 2.2) | | Test under Chef-18 Hab | `hab pkg exec $chef18Ident ruby -e "..."` (replace vendored gem first, see 3.2) | | Test under Chef-18 Omnibus | `C:\opscode\chef\embedded\bin\ruby -e "..."` | diff --git a/chef-powershell/memory_pressure_test.rb b/chef-powershell/memory_pressure_test.rb new file mode 100644 index 00000000..903c49d3 --- /dev/null +++ b/chef-powershell/memory_pressure_test.rb @@ -0,0 +1,277 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true +# +# Regression test for the class of bug fixed by PRs #187, #196, #205: under memory +# pressure, the buffer the native wrapper used to hand a result back to Ruby could be +# collected/reused before Ruby finished reading it, silently corrupting or truncating +# the payload -- even for values as trivial as a boolean. The fix made the native side +# call back *into* Ruby (StoreResultCallback in lib/chef-powershell/powershell.rb) so +# the bytes are copied into a Ruby string synchronously, while the buffer is still +# guaranteed valid, instead of Ruby reading a raw pointer whenever it got around to it. +# +# This script re-creates the conditions that exposed that bug: +# - the *process* is put under heavy GC churn (large, constantly-discarded Ruby +# allocations, plus GC.stress to force a collection around every allocation), and +# - it is intended to be run inside a memory-constrained container (see +# .expeditor/local_low_memory_test.ps1), so the .NET/CLR side is also forced to +# collect aggressively rather than comfortably holding on to short-lived buffers. +# +# It then repeatedly round-trips a variety of payload shapes/sizes (including the +# "trivial" boolean/integer cases from the original bug reports) through both +# interpreters and asserts *exact* equality on every single iteration. Any deviation +# indicates the result buffer was corrupted, truncated, or reused before Ruby captured +# it, i.e. a regression of the original heap-lifetime bug. +# +# Usage: +# ruby memory_pressure_test.rb [iterations] +# +# Or via environment variable: +# $env:CHEF_POWERSHELL_BIN = "C:\hab\pkgs\chef\chef-powershell-shim\X.Y.Z\timestamp\bin" +# ruby memory_pressure_test.rb +# +# Exit code: 0 = all round-trips were exact, 1 = one or more were corrupted/failed. + +original_gem_path = Gem.path.dup + +begin + require "bundler/setup" +rescue LoadError, StandardError + # Fall back to system/already-loaded gems if Bundler is unavailable, or if the + # committed Gemfile.lock doesn't match what's installed on this particular host + # (e.g. a bare low-memory container that only has ffi/ffi-yajl installed globally). + # Even when it ultimately raises, `bundler/setup` replaces Gem.path with just the + # (possibly gem-less, on a version mismatch) vendor bundle dir as a side effect -- + # restore it so the plain `require`s below can still find system-installed gems. + Gem.paths = { "GEM_PATH" => original_gem_path.join(File::PATH_SEPARATOR) } + Gem::Specification.reset +end + +BIN_DIR = if ARGV[0] && !ARGV[0].match?(/\A\d+\z/) + File.expand_path(ARGV[0]) + elsif ENV["CHEF_POWERSHELL_BIN"] + ENV["CHEF_POWERSHELL_BIN"] + else + File.expand_path(File.join("bin", "ruby_bin_folder", ENV.fetch("PROCESSOR_ARCHITECTURE", "AMD64")), __dir__) + end.freeze + +ITERATIONS = (ARGV.find { |a| a.match?(/\A\d+\z/) } || 15).to_i + +NET481_DLL = File.join(BIN_DIR, "Chef.PowerShell.Wrapper.dll").freeze +NET10_DLL = File.join(BIN_DIR, "shared", "Microsoft.NETCore.App", "10.0.0", + "Chef.PowerShell.Wrapper.Core.dll").freeze + +$LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) +$LOAD_PATH.unshift(File.expand_path("lib", __dir__)) + +# Same trick smoke_test_dlls.rb uses: when running from source (not installed as a +# gem), Gem.loaded_specs["chef-powershell"] is nil, which crashes powershell.rb at +# class-body load time. Register the gemspec manually. +unless Gem.loaded_specs["chef-powershell"] + gemspec_path = File.expand_path("chef-powershell.gemspec", __dir__) + if File.exist?(gemspec_path) + spec = Gem::Specification.load(gemspec_path) + spec.instance_variable_set(:@full_gem_path, File.expand_path(__dir__)) + Gem.loaded_specs[spec.name] = spec + end +end + +require "chef-powershell" + +ENV["CHEF_POWERSHELL_BIN"] = BIN_DIR + +class ChefPowerShell + class Pwsh + # AddDllDirectory allocates an OS-tracked "cookie" for every call and is never + # released (there is no matching RemoveDllDirectory call anywhere in this repo). + # Calling it fresh on every single :pwsh invocation -- as the upstream Pwsh#exec + # and smoke_test_dlls.rb do -- leaks one of those cookies per call. That's mostly + # invisible in normal use, but a long-running memory-pressure loop like this one + # calls Pwsh#exec many times in a tight loop and *will* eventually exhaust the + # per-process directory table (observed here as a "Failed to register DLL search + # directory" LoadError, not a crash, so it can't silently corrupt anything -- but + # it does stop the stress loop short). Work around it here, in the test harness + # only, by registering each directory at most once per process. + @registered_dll_dirs = {} + + def exec(script, timeout: -1) + original_dml = ENV["DOTNET_MULTILEVEL_LOOKUP"] + original_root = ENV["DOTNET_ROOT"] + original_root_x86 = ENV["DOTNET_ROOT(x86)"] + + ENV["DOTNET_MULTILEVEL_LOOKUP"] = "0" + ENV["DOTNET_ROOT"] = BIN_DIR + ENV["DOTNET_ROOT(x86)"] = BIN_DIR + @powershell_dll = NET10_DLL + + core_dir = File.dirname(@powershell_dll) + Kernel32.SetDefaultDllDirectories(Kernel32::LOAD_LIBRARY_SEARCH_DEFAULT_DIRS) + [BIN_DIR, core_dir].each do |dir| + next if self.class.instance_variable_get(:@registered_dll_dirs)[dir] + + Kernel32.register_search_directory(dir) + self.class.instance_variable_get(:@registered_dll_dirs)[dir] = true + end + Kernel32.SetDllDirectoryA(core_dir) + + ChefPowerShell::PowerShell.instance_method(:exec).bind(self).call(script, timeout: timeout) + ensure + ENV["DOTNET_MULTILEVEL_LOOKUP"] = original_dml + ENV["DOTNET_ROOT"] = original_root + ENV["DOTNET_ROOT(x86)"] = original_root_x86 + end + end +end + +def ps481(script, timeout: -1) + ps = ChefPowerShell::PowerShell.allocate + ps.instance_variable_set(:@powershell_dll, NET481_DLL) + ps.send(:exec, script, timeout: timeout) + ps +end + +def ps_core(script, timeout: -1) + ChefPowerShell::Pwsh.new(script, timeout: timeout) +end + +# --------------------------------------------------------------------------- +# Memory pressure helpers +# --------------------------------------------------------------------------- + +# GC.stress = true forces a full GC around *every single* Ruby object allocation, +# which is the most thorough way to catch anything relying on a Ruby object's +# memory staying put across a call boundary -- but it can slow execution down by +# multiple orders of magnitude, which isn't practical to run by default (including +# inside an already memory-constrained container). It's opt-in via +# CHEF_POWERSHELL_GC_STRESS=1 for maintainers who want the maximal, slow, "torture +# test" version of this script; the default mode below still forces a full GC +# around every payload round-trip, just not around every single allocation. +EXTREME_GC_STRESS = ENV["CHEF_POWERSHELL_GC_STRESS"] == "1" + +def under_gc_pressure + if EXTREME_GC_STRESS + original = GC.stress + GC.stress = true + begin + yield + ensure + GC.stress = original + end + else + yield + end +end + +# Churn a batch of large, immediately-discarded allocations and force a full, +# immediate GC to simulate a memory-constrained environment applying pressure to +# both the Ruby heap and, indirectly (via reduced available system memory), the +# CLR heap the native wrapper allocates its result buffers from. This runs around +# every payload round-trip (not just every iteration), maximizing the chance a GC +# lands in the window between the native side handing back a result and Ruby +# finishing reading it. +def churn_garbage(chunks: 4, chunk_size: 256 * 1024) + chunks.times { "x" * chunk_size } + GC.start(full_mark: true, immediate_sweep: true) +end + +# --------------------------------------------------------------------------- +# Harness +# --------------------------------------------------------------------------- + +PASS = "\e[32mPASS\e[0m" +FAIL = "\e[31mFAIL\e[0m" +BOLD = "\e[1m" +RESET = "\e[0m" + +$failures = 0 +$passes = 0 + +def check(description) + result = yield + if result + $passes += 1 + else + puts " #{FAIL} #{description}" + $failures += 1 + end +rescue => e + puts " #{FAIL} #{description}" + puts " #{e.class}: #{e.message.lines.first&.chomp}" + $failures += 1 +end + +def section(title) + puts "\n#{BOLD}#{title}#{RESET}" +end + +# Payload matrix: deliberately includes the "trivial" scalar cases called out in the +# original bug reports (bare booleans/integers), plus strings (including multi-byte +# UTF-8, to stress the UTF-16LE -> UTF-8 decode path), arrays, hashes, and a large +# string to stress bigger buffer allocations. +PAYLOADS = [ + ["$true", true], + ["$false", false], + ["42", 42], + ["0", 0], + ["-17", -17], + ["3.14", 3.14], + ["'hello from powershell'", "hello from powershell"], + ["'unicode: héllo wörld 日本語 🎉'", "unicode: héllo wörld 日本語 🎉"], + ["@(1, 2, 3, 4, 5)", [1, 2, 3, 4, 5]], + ["@{ Name = 'Chef'; Version = '19' }", { "Name" => "Chef", "Version" => "19" }], + ["'x' * 20000", nil], # validated separately below (length check) +].freeze + +def run_payload_round_trips(run_proc, label) + PAYLOADS.each do |script, expected| + # Churn + force a GC immediately before every single call, so a collection is + # highly likely to land right in the window between the native side handing + # back the result buffer and Ruby finishing reading it. + churn_garbage + + if script == "'x' * 20000" + check("#{label}: large string round-trips intact (20000 chars)") do + result = run_proc.call("('x' * 20000)") + result.errors.empty? && result.result.is_a?(String) && result.result.length == 20000 && result.result.chars.uniq == ["x"] + end + else + check("#{label}: #{script.inspect} round-trips as #{expected.inspect}") do + result = run_proc.call(script) + result.errors.empty? && result.result == expected + end + end + end +end + +net481_present = File.exist?(NET481_DLL) +net10_present = File.exist?(NET10_DLL) + +puts "DLL directory : #{BIN_DIR}" +puts "NET481 DLL : #{NET481_DLL} (#{net481_present ? "present" : "MISSING"})" +puts "NET10 DLL : #{NET10_DLL} (#{net10_present ? "present" : "MISSING"})" +puts "Iterations : #{ITERATIONS}" +puts "GC pressure : #{EXTREME_GC_STRESS ? "EXTREME (GC.stress=true for the whole run; CHEF_POWERSHELL_GC_STRESS=1)" : "full GC forced before every round-trip (set CHEF_POWERSHELL_GC_STRESS=1 for the slower, more thorough GC.stress mode)"}" + +abort "Neither wrapper DLL was found under #{BIN_DIR} -- nothing to test." unless net481_present || net10_present + +under_gc_pressure do + ITERATIONS.times do |i| + section("Iteration #{i + 1}/#{ITERATIONS}") + + run_payload_round_trips(method(:ps481), ":powershell") if net481_present + run_payload_round_trips(method(:ps_core), ":pwsh") if net10_present + end +end + +puts "\n#{"=" * 60}" +total = $passes + $failures +puts "Results: #{$passes}/#{total} round-trips exact, #{$failures} corrupted/failed" +puts "=" * 60 + +if $failures > 0 + puts "\n#{FAIL} Memory pressure test FAILED -- #{$failures} round-trip(s) came back " \ + "corrupted, truncated, or wrong. This is the class of bug fixed in PRs #187/#196/#205." + exit 1 +else + puts "\n#{PASS} All #{$passes} round-trips survived GC/memory pressure intact." + exit 0 +end From 0e5024eccf3f54daa9e0b1e16cba5782b2fe5d8f Mon Sep 17 00:00:00 2001 From: Thomas Powell <104777878+tpowell-progress@users.noreply.github.com> Date: Mon, 14 Sep 2026 22:39:20 -0400 Subject: [PATCH 8/8] Fix chefstyle Layout/ExtraSpacing offense in memory_pressure_test.rb The unicode payload row had extra alignment spacing that cookstyle flagged as Layout/ExtraSpacing, failing the gem_check style task. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Thomas Powell <104777878+tpowell-progress@users.noreply.github.com> --- chef-powershell/memory_pressure_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef-powershell/memory_pressure_test.rb b/chef-powershell/memory_pressure_test.rb index 903c49d3..65533a28 100644 --- a/chef-powershell/memory_pressure_test.rb +++ b/chef-powershell/memory_pressure_test.rb @@ -215,7 +215,7 @@ def section(title) ["-17", -17], ["3.14", 3.14], ["'hello from powershell'", "hello from powershell"], - ["'unicode: héllo wörld 日本語 🎉'", "unicode: héllo wörld 日本語 🎉"], + ["'unicode: héllo wörld 日本語 🎉'", "unicode: héllo wörld 日本語 🎉"], ["@(1, 2, 3, 4, 5)", [1, 2, 3, 4, 5]], ["@{ Name = 'Chef'; Version = '19' }", { "Name" => "Chef", "Version" => "19" }], ["'x' * 20000", nil], # validated separately below (length check)