Build Swift SDK #49
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Swift SDK | |
| # Builds the per-architecture Swift SDK artifactbundles and merges them into | |
| # one all-arch bundle. Split out of build-images.yml: the images and the SDKs | |
| # share nothing but the container, they fail for different reasons, and | |
| # keeping them apart means "the SDK is broken" and "the image is broken" are | |
| # separate red marks - and either can be re-run without paying for the other. | |
| # | |
| # The two jobs here stay together on purpose. combined-swift-sdk consumes the | |
| # per-arch bundles through download-artifact, which only sees artifacts from | |
| # its own workflow run; reaching into another workflow's run would need a run | |
| # id and a token for no benefit. | |
| # | |
| # Every architecture's SDK comes from the app-sdk profile (Swift plus the full | |
| # applibs graphics/app stack), so all four bundles carry the same libraries | |
| # and headers - libGL/EGL/GLES, SDL, wayland, X11, cairo, ALSA - rather than | |
| # arm64/x86_64 shipping only what the image profile happened to build. armv7 | |
| # and i386 have no bootable image target and exist only here. | |
| # | |
| # Each arch starts by seeding its output tree from the toolchain-latest | |
| # release that build-toolchain.yml publishes (br-seed), so only the app-sdk | |
| # profile's delta over the sdk profile compiles here. The trees persist on the | |
| # runner at /mnt/br/profiles/app-sdk/<arch>, bind-mounted to | |
| # /mnt/br/output/<arch> in the container because that is the prefix Buildroot | |
| # baked into the packaged toolchain - see docs/build.md. | |
| # | |
| # Runs on the self-hosted x86 runner (see docs/build.md), which is a single | |
| # machine: the four app-sdk jobs execute one after another rather than in | |
| # parallel. The ccache lives on that machine's own disk at | |
| # /mnt/br/ccache/<arch> - the same directory the toolchain and image workflows | |
| # use - so it survives between runs and there is nothing to upload or restore. | |
| # Four Buildroot trees, so hours per run: nightly, plus on demand from the | |
| # Actions tab ("gh workflow run build-swift-sdk.yml"). Scheduled two hours | |
| # before the image build so the two heavy workflows do not contend for | |
| # runners and both results are waiting in the morning. | |
| on: | |
| schedule: | |
| - cron: '0 5 * * *' | |
| workflow_dispatch: | |
| # One runner, so a nightly still running when the next fires would otherwise | |
| # queue behind itself. Supersede instead. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| # Not cancel-in-progress, for the same reason as build-toolchain.yml: the | |
| # daily 05:00 UTC schedule and a workflow_dispatch share this group, so the | |
| # schedule killed a dispatched run forty minutes in - seven seconds after | |
| # starting, on the same commit, discarding four arches of progress to | |
| # rebuild exactly the same thing. | |
| # | |
| # These jobs run on GitHub-hosted runners, so two runs of the same ref never | |
| # share a tree: /mnt is local to each ephemeral runner. Overlapping runs | |
| # cost machine time and nothing else, while a cancellation costs an hour or | |
| # more of work. | |
| cancel-in-progress: false | |
| env: | |
| # The caches are on the runner's disk, not in the 10 GB repo-wide | |
| # actions/cache budget, so they are sized for hit rate. | |
| CCACHE_MAX_SIZE: 20G | |
| jobs: | |
| app-sdk: | |
| name: App SDK (${{ matrix.arch }}) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: [x86_64, arm64, armv7, i386] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 350 | |
| # Plain base image; the toolchain comes from br-seed, not a per-arch tag. | |
| # /mnt is bind-mounted so the tree and ccache outlive the container, and | |
| # the per-profile host dir is mounted at the exact path the packaged | |
| # toolchain was built at, because Buildroot bakes that prefix into | |
| # .config, the host/ cmake caches and the gcc specs. | |
| container: | |
| image: docker.io/colemancda/buildroot-swift:latest | |
| options: >- | |
| --volume /mnt:/mnt | |
| --volume /mnt/br/profiles/app-sdk/${{ matrix.arch }}:/mnt/br/output/${{ matrix.arch }} | |
| --security-opt label=disable | |
| env: | |
| CCACHE_DIR: /mnt/br/ccache/${{ matrix.arch }} | |
| steps: | |
| - name: Checkout swift-linux | |
| uses: actions/checkout@v4 | |
| # Seed before br-setup: defconfig has to configure the seeded tree, not | |
| # be buried under the unpack. Steady state this only compares the | |
| # release's asset timestamps against the tree's stamp and moves on. | |
| - name: Seed from the toolchain release | |
| uses: ./.github/actions/br-seed | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| arch: ${{ matrix.arch }} | |
| dest-dir: /mnt/br/output/${{ matrix.arch }} | |
| - name: Prepare and configure the tree | |
| uses: ./.github/actions/br-setup | |
| with: | |
| output-dir: /mnt/br/output/${{ matrix.arch }} | |
| arch: ${{ matrix.arch }} | |
| profile: app-sdk | |
| ccache-dir: /mnt/br/ccache/${{ matrix.arch }} | |
| ccache-max-size: ${{ env.CCACHE_MAX_SIZE }} | |
| # GitHub-hosted runners have ~70G on /mnt, not the terabyte the | |
| # self-hosted box has, and the seeded tree already occupies a chunk | |
| # of it. The default 50G guard would abort a job that has ample room. | |
| min-free-gb: '20' | |
| # The seed ships host packages as stamps without sources, which the Swift | |
| # packages cannot live with: they read host-swift's build directory and | |
| # its swift-source tree, not just what it installed into host/. Without | |
| # this the target swift package configures against a compiler that is not | |
| # there and stops ninety minutes in. See the action for the detail. | |
| - name: Fetch the shared host-swift | |
| uses: ./.github/actions/host-swift | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| build-dir: /mnt/br/output/${{ matrix.arch }}/build | |
| # The seeded tree carries the sdk profile's stamps, so this replays them | |
| # and compiles only the app-sdk delta. ccache is the same per-arch | |
| # directory every workflow uses; compiler_check=content (set by | |
| # br-setup) is what lets hits survive path differences between them. | |
| - name: Build app-sdk (Swift + applibs) | |
| run: | | |
| set -eu | |
| FORCE_UNSAFE_CONFIGURE=1 make -C "$BR_B" O="$BR_O" BR2_EXTERNAL="$BR_EXT" \ | |
| BR2_DL_DIR=/mnt/br/dl BR2_CCACHE_DIR="$CCACHE_DIR" -j"$BR_JOBS" | |
| # Runs in the container deliberately: the sysroot contains symlinks to | |
| # absolute paths (/mnt/br/..., $GITHUB_WORKSPACE/...) for gcc's crt*.o, | |
| # libgcc.a and the libstdc++ headers, and those only resolve where the | |
| # same mounts exist. make-swift-sdk.sh turns them into real files for | |
| # the portable bundle (rsync --copy-unsafe-links); on the host they | |
| # would just dangle. | |
| - name: Generate Swift SDK | |
| run: | | |
| set -eu | |
| export OUTPUT_BASE=/mnt/br/output | |
| bundle="/tmp/swift-linux-${{ matrix.arch }}.artifactbundle" | |
| bash "$GITHUB_WORKSPACE/util/make-swift-sdk.sh" \ | |
| --arch ${{ matrix.arch }} --portable --out "$bundle" | |
| tar czf "$GITHUB_WORKSPACE/swift-linux-${{ matrix.arch }}-swift-sdk.tar.gz" \ | |
| -C "$(dirname "$bundle")" "$(basename "$bundle")" | |
| - name: Upload Swift SDK | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: swift-linux-${{ matrix.arch }}-swift-sdk | |
| path: swift-linux-${{ matrix.arch }}-swift-sdk.tar.gz | |
| if-no-files-found: error | |
| - name: ccache stats | |
| if: always() | |
| run: | | |
| ccache -d "$CCACHE_DIR" -s 2>/dev/null || true | |
| df -h /mnt | |
| # The container runs as root; the workspace belongs to the runner user on | |
| # the host and, unlike a hosted VM, survives the job. Without this the | |
| # next run's actions/checkout git-cleans as the runner user and hits EPERM | |
| # on the root-owned ports tree and tarball this job created. | |
| - name: Return the workspace to the runner user | |
| if: always() | |
| run: chown -R "$(stat -c '%u:%g' "$GITHUB_WORKSPACE")" "$GITHUB_WORKSPACE" || true | |
| # Merge the per-arch Swift SDKs into one artifactbundle that cross-compiles | |
| # for every architecture (swift build --swift-sdk aarch64/x86_64-...-linux-gnu). | |
| # No container - it only rearranges files (jq + tar), so it runs against the | |
| # runner machine's own tools. | |
| combined-swift-sdk: | |
| name: Combined Swift SDK | |
| needs: app-sdk | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| permissions: | |
| contents: write # uploads assets to the swift-sdk-latest release | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # This job used to inherit jq and gh from the ubuntu-latest image. On the | |
| # self-hosted machine they are the admin's to install, so say which one is | |
| # missing rather than failing later inside a pipeline. | |
| - name: Host prerequisites | |
| run: | | |
| set -eu | |
| missing= | |
| for c in jq gh tar; do | |
| command -v "$c" >/dev/null 2>&1 || missing="$missing $c" | |
| done | |
| if [ -n "$missing" ]; then | |
| echo "missing on the runner:$missing" >&2 | |
| echo "install with: sudo apt-get install -y$missing (see docs/build.md)" >&2 | |
| exit 1 | |
| fi | |
| - name: Download per-arch Swift SDKs | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: swift-linux-*-swift-sdk | |
| path: sdk-parts | |
| - name: Combine into an all-arch SDK | |
| run: | | |
| set -eu | |
| mkdir -p parts | |
| find sdk-parts -name '*-swift-sdk.tar.gz' -exec tar xzf {} -C parts \; | |
| bash util/combine-swift-sdk.sh parts/*.artifactbundle --out swift-linux.artifactbundle | |
| tar czf swift-linux-swift-sdk.tar.gz swift-linux.artifactbundle | |
| - name: Show what the bundle supports | |
| run: | | |
| jq -r '.targetTriples | keys[]' swift-linux.artifactbundle/swift-linux/swift-sdk.json | |
| - name: Upload combined Swift SDK | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: swift-linux-swift-sdk | |
| path: swift-linux-swift-sdk.tar.gz | |
| if-no-files-found: error | |
| # An artifact expires and needs a logged-in download, which is no way to | |
| # ship an SDK. Publish the same tarball to a rolling release instead, the | |
| # way build-toolchain.yml publishes its trees, so | |
| # "gh release download swift-sdk-latest" works for anyone. | |
| # | |
| # Release assets cap at 2 GiB. The toolchain trees are far past that and | |
| # always ship split; this bundle compresses well below it, so it is split | |
| # only if it has grown - consumers concatenate either way, and the | |
| # single-part case leaves one .part00 file rather than two spellings of | |
| # the same asset. | |
| - name: Publish to the swift-sdk-latest release | |
| if: github.event_name != 'pull_request' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -eu | |
| ls -lh swift-linux-swift-sdk.tar.gz | |
| split -b 1900M -d swift-linux-swift-sdk.tar.gz swift-linux-swift-sdk.tar.gz.part | |
| rm swift-linux-swift-sdk.tar.gz | |
| gh release view swift-sdk-latest --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1 || \ | |
| gh release create swift-sdk-latest --repo "$GITHUB_REPOSITORY" \ | |
| --title "Swift SDK (rolling)" \ | |
| --notes "Rolling all-architecture Swift SDK built by build-swift-sdk.yml, covering aarch64, x86_64, armv7 and i686. Assets are split tar.gz parts: | |
| cat swift-linux-swift-sdk.tar.gz.part* | tar xzf - | |
| swift sdk install swift-linux.artifactbundle" \ | |
| --latest=false | |
| # Overwrite each part, retrying until it lands. Deleting the old | |
| # assets up front and uploading afterwards leaves a window where the | |
| # release carries no SDK at all, and an upload that dies inside that | |
| # window leaves it that way - which is how the armv7 toolchain asset | |
| # was lost. --clobber is not a fix on its own: it also removes the | |
| # old asset before writing the new one, so the retry is the part that | |
| # matters. | |
| for f in swift-linux-swift-sdk.tar.gz.part*; do | |
| ok= | |
| for attempt in 1 2 3 4 5; do | |
| if gh release upload swift-sdk-latest "$f" \ | |
| --repo "$GITHUB_REPOSITORY" --clobber; then | |
| ok=yes | |
| break | |
| fi | |
| echo "upload of $f failed (attempt $attempt of 5), retrying" >&2 | |
| sleep $((attempt * 30)) | |
| done | |
| [ -n "$ok" ] || { echo "giving up on $f - the release is now missing this part" >&2; exit 1; } | |
| done | |
| # Confirm the full set is present at full size before pruning, so a | |
| # truncated upload fails here rather than when someone tries to | |
| # concatenate the parts. | |
| gh release view swift-sdk-latest --repo "$GITHUB_REPOSITORY" \ | |
| --json assets --jq '.assets[] | "\(.name) \(.size)"' > /tmp/sdk-assets.$$ | |
| for f in swift-linux-swift-sdk.tar.gz.part*; do | |
| want=$(stat -c %s "$f") | |
| got=$(awk -v n="$f" '$1 == n { print $2 }' /tmp/sdk-assets.$$) | |
| [ "$got" = "$want" ] || { | |
| echo "$f is ${got:-absent} on the release, expected $want bytes" >&2 | |
| rm -f /tmp/sdk-assets.$$ | |
| exit 1 | |
| } | |
| done | |
| # A bundle that now splits into fewer parts than last time would | |
| # otherwise leave the extras behind and corrupt the concatenation. | |
| keep=$(ls swift-linux-swift-sdk.tar.gz.part* | tr '\n' ' ') | |
| for a in $(awk '{ print $1 }' /tmp/sdk-assets.$$ | grep '^swift-linux-swift-sdk\.' || true); do | |
| case " $keep " in | |
| *" $a "*) ;; | |
| *) echo "removing stale asset $a" | |
| gh release delete-asset swift-sdk-latest "$a" --repo "$GITHUB_REPOSITORY" -y ;; | |
| esac | |
| done | |
| rm -f /tmp/sdk-assets.$$ |