diff --git a/.buildkite/ci.mjs b/.buildkite/ci.mjs index 3fb1e643f32f31..7f9aac99ff275f 100755 --- a/.buildkite/ci.mjs +++ b/.buildkite/ci.mjs @@ -322,7 +322,6 @@ function getCppAgent(platform, options) { * @returns {Agent} */ function getZigAgent(platform, options) { - const { arch } = platform; return { queue: "build-zig", }; @@ -334,13 +333,14 @@ function getZigAgent(platform, options) { * @returns {Agent} */ function getTestAgent(platform, options) { - const { os, arch } = platform; + const { os, arch, release } = platform; if (os === "darwin") { return { queue: `test-${os}`, os, arch, + release, }; } diff --git a/ci/README.md b/ci/README.md deleted file mode 100644 index fbd89a34dd78fc..00000000000000 --- a/ci/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# CI - -This directory contains scripts for building CI images for Bun. - -## Building - -### `macOS` - -On macOS, images are built using [`tart`](https://tart.run/), a tool that abstracts over the [`Virtualization.Framework`](https://developer.apple.com/documentation/virtualization) APIs, to run macOS VMs. - -To install the dependencies required, run: - -```sh -$ cd ci -$ bun run bootstrap -``` - -To build a vanilla macOS VM, run: - -```sh -$ bun run build:darwin-aarch64-vanilla -``` - -This builds a vanilla macOS VM with the current macOS release on your machine. It runs scripts to disable things like spotlight and siri, but it does not install any software. - -> Note: The image size is 50GB, so make sure you have enough disk space. - -If you want to build a specific macOS release, you can run: - -```sh -$ bun run build:darwin-aarch64-vanilla-15 -``` - -> Note: You cannot build a newer release of macOS on an older macOS machine. - -To build a macOS VM with software installed to build and test Bun, run: - -```sh -$ bun run build:darwin-aarch64 -``` - -## Running - -### `macOS` - -## How To - -### Support a new macOS release - -1. Visit [`ipsw.me`](https://ipsw.me/VirtualMac2,1) and find the IPSW of the macOS release you want to build. - -2. Add an entry to [`ci/darwin/variables.pkr.hcl`](/ci/darwin/variables.pkr.hcl) with the following format: - -```hcl -sonoma = { - distro = "sonoma" - release = "15" - ipsw = "https://updates.cdn-apple.com/..." -} -``` - -3. Add matching scripts to [`ci/package.json`](/ci/package.json) to build the image, then test it: - -```sh -$ bun run build:darwin-aarch64-vanilla-15 -``` - -> Note: If you need to troubleshoot the build, you can remove the `headless = true` property from [`ci/darwin/image-vanilla.pkr.hcl`](/ci/darwin/image-vanilla.pkr.hcl) and the VM's screen will be displayed. - -4. Test and build the non-vanilla image: - -```sh -$ bun run build:darwin-aarch64-15 -``` - -This will use the vanilla image and run the [`scripts/bootstrap.sh`](/scripts/bootstrap.sh) script to install the required software to build and test Bun. - -5. Publish the images: - -```sh -$ bun run login -$ bun run publish:darwin-aarch64-vanilla-15 -$ bun run publish:darwin-aarch64-15 -``` diff --git a/ci/alpine/build.Dockerfile b/ci/alpine/build.Dockerfile deleted file mode 100644 index f1f9aabb87ea86..00000000000000 --- a/ci/alpine/build.Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM alpine:edge AS build -ARG GIT_SHA -ENV GIT_SHA=${GIT_SHA} -WORKDIR /app/bun -ENV HOME=/root - -COPY . . -RUN touch $HOME/.bashrc -RUN ./scripts/bootstrap.sh -RUN . $HOME/.bashrc && bun run build:release - -RUN apk add file -RUN file ./build/release/bun -RUN ldd ./build/release/bun -RUN ./build/release/bun - -RUN cp -R /app/bun/build/* /output - -FROM scratch AS artifact -COPY --from=build /output / - -# docker build -f ./ci/alpine/build.Dockerfile --progress=plain --build-arg GIT_SHA="$(git rev-parse HEAD)" --target=artifact --output type=local,dest=./build-alpine . diff --git a/ci/alpine/test.Dockerfile b/ci/alpine/test.Dockerfile deleted file mode 100644 index e6836fe9d20133..00000000000000 --- a/ci/alpine/test.Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM alpine:edge -ENV HOME=/root -WORKDIR /root -COPY ./build-alpine/release/bun . -COPY ./test ./test -COPY ./scripts ./scripts -COPY ./package.json ./package.json -COPY ./packages ./packages - -RUN apk update -RUN apk add nodejs lsb-release-minimal git python3 npm make g++ -RUN apk add file - -RUN file /root/bun -RUN ldd /root/bun -RUN /root/bun - -RUN ./scripts/runner.node.mjs --exec-path /root/bun - -# docker build -f ./ci/alpine/test.Dockerfile --progress=plain . diff --git a/ci/darwin/image-vanilla.pkr.hcl b/ci/darwin/image-vanilla.pkr.hcl deleted file mode 100644 index 40455713b4a9b6..00000000000000 --- a/ci/darwin/image-vanilla.pkr.hcl +++ /dev/null @@ -1,46 +0,0 @@ -# Generates a vanilla macOS VM with optimized settings for virtualized environments. -# See login.sh and optimize.sh for details. - -data "external-raw" "boot-script" { - program = ["sh", "-c", templatefile("scripts/boot-image.sh", var)] -} - -source "tart-cli" "bun-darwin-aarch64-vanilla" { - vm_name = "bun-darwin-aarch64-vanilla-${local.release.distro}-${local.release.release}" - from_ipsw = local.release.ipsw - cpu_count = local.cpu_count - memory_gb = local.memory_gb - disk_size_gb = local.disk_size_gb - ssh_username = local.username - ssh_password = local.password - ssh_timeout = "120s" - create_grace_time = "30s" - boot_command = split("\n", data.external-raw.boot-script.result) - headless = true # Disable if you need to debug why the boot_command is not working -} - -build { - sources = ["source.tart-cli.bun-darwin-aarch64-vanilla"] - - provisioner "file" { - content = file("scripts/setup-login.sh") - destination = "/tmp/setup-login.sh" - } - - provisioner "shell" { - inline = ["echo \"${local.password}\" | sudo -S sh -c 'sh /tmp/setup-login.sh \"${local.username}\" \"${local.password}\"'"] - } - - provisioner "file" { - content = file("scripts/optimize-machine.sh") - destination = "/tmp/optimize-machine.sh" - } - - provisioner "shell" { - inline = ["sudo sh /tmp/optimize-machine.sh"] - } - - provisioner "shell" { - inline = ["sudo rm -rf /tmp/*"] - } -} diff --git a/ci/darwin/image.pkr.hcl b/ci/darwin/image.pkr.hcl deleted file mode 100644 index b536efbecb36e2..00000000000000 --- a/ci/darwin/image.pkr.hcl +++ /dev/null @@ -1,44 +0,0 @@ -# Generates a macOS VM with software installed to build and test Bun. - -source "tart-cli" "bun-darwin-aarch64" { - vm_name = "bun-darwin-aarch64-${local.release.distro}-${local.release.release}" - vm_base_name = "bun-darwin-aarch64-vanilla-${local.release.distro}-${local.release.release}" - cpu_count = local.cpu_count - memory_gb = local.memory_gb - disk_size_gb = local.disk_size_gb - ssh_username = local.username - ssh_password = local.password - ssh_timeout = "120s" - headless = true -} - -build { - sources = ["source.tart-cli.bun-darwin-aarch64"] - - provisioner "file" { - content = file("../../scripts/bootstrap.sh") - destination = "/tmp/bootstrap.sh" - } - - provisioner "shell" { - inline = ["CI=true sh /tmp/bootstrap.sh"] - } - - provisioner "file" { - source = "darwin/plists/" - destination = "/tmp/" - } - - provisioner "shell" { - inline = [ - "sudo ls /tmp/", - "sudo mv /tmp/*.plist /Library/LaunchDaemons/", - "sudo chown root:wheel /Library/LaunchDaemons/*.plist", - "sudo chmod 644 /Library/LaunchDaemons/*.plist", - ] - } - - provisioner "shell" { - inline = ["sudo rm -rf /tmp/*"] - } -} diff --git a/ci/darwin/plists/buildkite-agent.plist b/ci/darwin/plists/buildkite-agent.plist deleted file mode 100644 index 23c058913f7e3c..00000000000000 --- a/ci/darwin/plists/buildkite-agent.plist +++ /dev/null @@ -1,44 +0,0 @@ - - - - - Label - com.buildkite.buildkite-agent - - ProgramArguments - - /usr/local/bin/buildkite-agent - start - - - KeepAlive - - SuccessfulExit - - - - RunAtLoad - - - StandardOutPath - /var/buildkite-agent/logs/buildkite-agent.log - - StandardErrorPath - /var/buildkite-agent/logs/buildkite-agent.log - - EnvironmentVariables - - BUILDKITE_AGENT_CONFIG - /etc/buildkite-agent/buildkite-agent.cfg - - - LimitLoadToSessionType - - Aqua - LoginWindow - Background - StandardIO - System - - - \ No newline at end of file diff --git a/ci/darwin/plists/tailscale.plist b/ci/darwin/plists/tailscale.plist deleted file mode 100644 index cbe3f001b0c4ae..00000000000000 --- a/ci/darwin/plists/tailscale.plist +++ /dev/null @@ -1,20 +0,0 @@ - - - - - Label - com.tailscale.tailscaled - - ProgramArguments - - /usr/local/bin/tailscale - up - --ssh - --authkey - ${TAILSCALE_AUTHKEY} - - - RunAtLoad - - - \ No newline at end of file diff --git a/ci/darwin/plists/tailscaled.plist b/ci/darwin/plists/tailscaled.plist deleted file mode 100644 index 12d316f1abaad1..00000000000000 --- a/ci/darwin/plists/tailscaled.plist +++ /dev/null @@ -1,16 +0,0 @@ - - - - - Label - com.tailscale.tailscaled - - ProgramArguments - - /usr/local/bin/tailscaled - - - RunAtLoad - - - \ No newline at end of file diff --git a/ci/darwin/scripts/boot-image.sh b/ci/darwin/scripts/boot-image.sh deleted file mode 100755 index 02ae01db0345a3..00000000000000 --- a/ci/darwin/scripts/boot-image.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/sh - -# This script generates the boot commands for the macOS installer GUI. -# It is run on your local machine, not inside the VM. - -# Sources: -# - https://github.com/cirruslabs/macos-image-templates/blob/master/templates/vanilla-sequoia.pkr.hcl - -if ! [ "${release}" ] || ! [ "${username}" ] || ! [ "${password}" ]; then - echo "Script must be run with variables: release, username, and password" >&2 - exit 1 -fi - -# Hello, hola, bonjour, etc. -echo "" - -# Select Your Country and Region -echo "italianoenglish" -echo "united states" - -# Written and Spoken Languages -echo "" - -# Accessibility -echo "" - -# Data & Privacy -echo "" - -# Migration Assistant -echo "" - -# Sign In with Your Apple ID -echo "" - -# Are you sure you want to skip signing in with an Apple ID? -echo "" - -# Terms and Conditions -echo "" - -# I have read and agree to the macOS Software License Agreement -echo "" - -# Create a Computer Account -echo "${username}${password}${password}" - -# Enable Location Services -echo "" - -# Are you sure you don't want to use Location Services? -echo "" - -# Select Your Time Zone -echo "UTC" - -# Analytics -echo "" - -# Screen Time -echo "" - -# Siri -echo "" - -# Choose Your Look -echo "" - -if [ "${release}" = "13" ] || [ "${release}" = "14" ]; then - # Enable Voice Over - echo "v" -else - # Welcome to Mac - echo "" - - # Enable Keyboard navigation - echo "Terminal" - echo "defaults write NSGlobalDomain AppleKeyboardUIMode -int 3" - echo "q" -fi - -# Now that the installation is done, open "System Settings" -echo "System Settings" - -# Navigate to "Sharing" -echo "fsharing" - -if [ "${release}" = "13" ]; then - # Navigate to "Screen Sharing" and enable it - echo "" - - # Navigate to "Remote Login" and enable it - echo "" - - # Open "Remote Login" details - echo "" - - # Enable "Full Disk Access" - echo "" - - # Click "Done" - echo "" - - # Disable Voice Over - echo "" -elif [ "${release}" = "14" ]; then - # Navigate to "Screen Sharing" and enable it - echo "" - - # Navigate to "Remote Login" and enable it - echo "" - - # Disable Voice Over - echo "" -elif [ "${release}" = "15" ]; then - # Navigate to "Screen Sharing" and enable it - echo "" - - # Navigate to "Remote Login" and enable it - echo "" -fi - -# Quit System Settings -echo "q" diff --git a/ci/darwin/scripts/optimize-machine.sh b/ci/darwin/scripts/optimize-machine.sh deleted file mode 100644 index 1d58ff4bb349c0..00000000000000 --- a/ci/darwin/scripts/optimize-machine.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/sh - -# This script optimizes macOS for virtualized environments. -# It disables things like spotlight, screen saver, and sleep. - -# Sources: -# - https://github.com/sickcodes/osx-optimizer -# - https://github.com/koding88/MacBook-Optimization-Script -# - https://www.macstadium.com/blog/simple-optimizations-for-macos-and-ios-build-agents - -if [ "$(id -u)" != "0" ]; then - echo "This script must be run using sudo." >&2 - exit 1 -fi - -execute() { - echo "$ $@" >&2 - if ! "$@"; then - echo "Command failed: $@" >&2 - exit 1 - fi -} - -disable_software_update() { - execute softwareupdate --schedule off - execute defaults write com.apple.SoftwareUpdate AutomaticDownload -bool false - execute defaults write com.apple.SoftwareUpdate AutomaticCheckEnabled -bool false - execute defaults write com.apple.SoftwareUpdate ConfigDataInstall -int 0 - execute defaults write com.apple.SoftwareUpdate CriticalUpdateInstall -int 0 - execute defaults write com.apple.SoftwareUpdate ScheduleFrequency -int 0 - execute defaults write com.apple.SoftwareUpdate AutomaticDownload -int 0 - execute defaults write com.apple.commerce AutoUpdate -bool false - execute defaults write com.apple.commerce AutoUpdateRestartRequired -bool false -} - -disable_spotlight() { - execute mdutil -i off -a - execute mdutil -E / -} - -disable_siri() { - execute launchctl unload -w /System/Library/LaunchAgents/com.apple.Siri.agent.plist - execute defaults write com.apple.Siri StatusMenuVisible -bool false - execute defaults write com.apple.Siri UserHasDeclinedEnable -bool true - execute defaults write com.apple.assistant.support "Assistant Enabled" 0 -} - -disable_sleep() { - execute systemsetup -setsleep Never - execute systemsetup -setcomputersleep Never - execute systemsetup -setdisplaysleep Never - execute systemsetup -setharddisksleep Never -} - -disable_screen_saver() { - execute defaults write com.apple.screensaver loginWindowIdleTime 0 - execute defaults write com.apple.screensaver idleTime 0 -} - -disable_screen_lock() { - execute defaults write com.apple.loginwindow DisableScreenLock -bool true -} - -disable_wallpaper() { - execute defaults write com.apple.loginwindow DesktopPicture "" -} - -disable_application_state() { - execute defaults write com.apple.loginwindow TALLogoutSavesState -bool false -} - -disable_accessibility() { - execute defaults write com.apple.Accessibility DifferentiateWithoutColor -int 1 - execute defaults write com.apple.Accessibility ReduceMotionEnabled -int 1 - execute defaults write com.apple.universalaccess reduceMotion -int 1 - execute defaults write com.apple.universalaccess reduceTransparency -int 1 -} - -disable_dashboard() { - execute defaults write com.apple.dashboard mcx-disabled -boolean YES - execute killall Dock -} - -disable_animations() { - execute defaults write NSGlobalDomain NSAutomaticWindowAnimationsEnabled -bool false - execute defaults write -g QLPanelAnimationDuration -float 0 - execute defaults write com.apple.finder DisableAllAnimations -bool true -} - -disable_time_machine() { - execute tmutil disable -} - -enable_performance_mode() { - # https://support.apple.com/en-us/101992 - if ! [ $(nvram boot-args 2>/dev/null | grep -q serverperfmode) ]; then - execute nvram boot-args="serverperfmode=1 $(nvram boot-args 2>/dev/null | cut -f 2-)" - fi -} - -add_terminal_to_desktop() { - execute ln -sf /System/Applications/Utilities/Terminal.app ~/Desktop/Terminal -} - -main() { - disable_software_update - disable_spotlight - disable_siri - disable_sleep - disable_screen_saver - disable_screen_lock - disable_wallpaper - disable_application_state - disable_accessibility - disable_dashboard - disable_animations - disable_time_machine - enable_performance_mode - add_terminal_to_desktop -} - -main diff --git a/ci/darwin/scripts/setup-login.sh b/ci/darwin/scripts/setup-login.sh deleted file mode 100755 index f68beb26f2f2d8..00000000000000 --- a/ci/darwin/scripts/setup-login.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/sh - -# This script generates a /etc/kcpassword file to enable auto-login on macOS. -# Yes, this stores your password in plain text. Do NOT do this on your local machine. - -# Sources: -# - https://github.com/xfreebird/kcpassword/blob/master/kcpassword - -if [ "$(id -u)" != "0" ]; then - echo "This script must be run using sudo." >&2 - exit 1 -fi - -execute() { - echo "$ $@" >&2 - if ! "$@"; then - echo "Command failed: $@" >&2 - exit 1 - fi -} - -kcpassword() { - passwd="$1" - key="7d 89 52 23 d2 bc dd ea a3 b9 1f" - passwd_hex=$(printf "%s" "$passwd" | xxd -p | tr -d '\n') - - key_len=33 - passwd_len=${#passwd_hex} - remainder=$((passwd_len % key_len)) - if [ $remainder -ne 0 ]; then - padding=$((key_len - remainder)) - passwd_hex="${passwd_hex}$(printf '%0*x' $((padding / 2)) 0)" - fi - - result="" - i=0 - while [ $i -lt ${#passwd_hex} ]; do - for byte in $key; do - [ $i -ge ${#passwd_hex} ] && break - p="${passwd_hex:$i:2}" - r=$(printf '%02x' $((0x$p ^ 0x$byte))) - result="${result}${r}" - i=$((i + 2)) - done - done - - echo "$result" -} - -login() { - username="$1" - password="$2" - - enable_passwordless_sudo() { - execute mkdir -p /etc/sudoers.d/ - echo "${username} ALL=(ALL) NOPASSWD: ALL" | EDITOR=tee execute visudo "/etc/sudoers.d/${username}-nopasswd" - } - - enable_auto_login() { - echo "00000000: 1ced 3f4a bcbc ba2c caca 4e82" | execute xxd -r - /etc/kcpassword - execute defaults write /Library/Preferences/com.apple.loginwindow autoLoginUser "${username}" - } - - disable_screen_lock() { - execute sysadminctl -screenLock off -password "${password}" - } - - enable_passwordless_sudo - enable_auto_login - disable_screen_lock -} - -if [ $# -ne 2 ]; then - echo "Usage: $0 " >&2 - exit 1 -fi - -login "$@" diff --git a/ci/darwin/variables.pkr.hcl b/ci/darwin/variables.pkr.hcl deleted file mode 100644 index d1133eb04a5f21..00000000000000 --- a/ci/darwin/variables.pkr.hcl +++ /dev/null @@ -1,78 +0,0 @@ -packer { - required_plugins { - tart = { - version = ">= 1.12.0" - source = "github.com/cirruslabs/tart" - } - external = { - version = ">= 0.0.2" - source = "github.com/joomcode/external" - } - } -} - -variable "release" { - type = number - default = 13 -} - -variable "username" { - type = string - default = "admin" -} - -variable "password" { - type = string - default = "admin" -} - -variable "cpu_count" { - type = number - default = 2 -} - -variable "memory_gb" { - type = number - default = 4 -} - -variable "disk_size_gb" { - type = number - default = 50 -} - -locals { - sequoia = { - tier = 1 - distro = "sequoia" - release = "15" - ipsw = "https://updates.cdn-apple.com/2024FallFCS/fullrestores/062-78489/BDA44327-C79E-4608-A7E0-455A7E91911F/UniversalMac_15.0_24A335_Restore.ipsw" - } - - sonoma = { - tier = 2 - distro = "sonoma" - release = "14" - ipsw = "https://updates.cdn-apple.com/2023FallFCS/fullrestores/042-54934/0E101AD6-3117-4B63-9BF1-143B6DB9270A/UniversalMac_14.0_23A344_Restore.ipsw" - } - - ventura = { - tier = 2 - distro = "ventura" - release = "13" - ipsw = "https://updates.cdn-apple.com/2022FallFCS/fullrestores/012-92188/2C38BCD1-2BFF-4A10-B358-94E8E28BE805/UniversalMac_13.0_22A380_Restore.ipsw" - } - - releases = { - 15 = local.sequoia - 14 = local.sonoma - 13 = local.ventura - } - - release = local.releases[var.release] - username = var.username - password = var.password - cpu_count = var.cpu_count - memory_gb = var.memory_gb - disk_size_gb = var.disk_size_gb -} diff --git a/ci/linux/Dockerfile b/ci/linux/Dockerfile deleted file mode 100644 index 3b46e73f6ccdbd..00000000000000 --- a/ci/linux/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -ARG IMAGE=debian:11 -FROM $IMAGE -COPY ./scripts/bootstrap.sh /tmp/bootstrap.sh -ENV CI=true -RUN sh /tmp/bootstrap.sh && rm -rf /tmp/* -WORKDIR /workspace/bun -COPY bunfig.toml bunfig.toml -COPY package.json package.json -COPY CMakeLists.txt CMakeLists.txt -COPY cmake/ cmake/ -COPY scripts/ scripts/ -COPY patches/ patches/ -COPY *.zig ./ -COPY src/ src/ -COPY packages/ packages/ -COPY test/ test/ -RUN bun i -RUN bun run build:ci diff --git a/ci/linux/scripts/set-hostname.sh b/ci/linux/scripts/set-hostname.sh deleted file mode 100644 index e529f74ce01976..00000000000000 --- a/ci/linux/scripts/set-hostname.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh - -# This script sets the hostname of the current machine. - -execute() { - echo "$ $@" >&2 - if ! "$@"; then - echo "Command failed: $@" >&2 - exit 1 - fi -} - -main() { - if [ "$#" -ne 1 ]; then - echo "Usage: $0 " >&2 - exit 1 - fi - - if [ -f "$(which hostnamectl)" ]; then - execute hostnamectl set-hostname "$1" - else - echo "Error: hostnamectl is not installed." >&2 - exit 1 - fi -} - -main "$@" diff --git a/ci/linux/scripts/start-tailscale.sh b/ci/linux/scripts/start-tailscale.sh deleted file mode 100644 index 3b519bfdf59339..00000000000000 --- a/ci/linux/scripts/start-tailscale.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh - -# This script starts tailscale on the current machine. - -execute() { - echo "$ $@" >&2 - if ! "$@"; then - echo "Command failed: $@" >&2 - exit 1 - fi -} - -main() { - if [ "$#" -ne 1 ]; then - echo "Usage: $0 " >&2 - exit 1 - fi - - execute tailscale up --reset --ssh --accept-risk=lose-ssh --auth-key="$1" -} - -main "$@" diff --git a/ci/package.json b/ci/package.json deleted file mode 100644 index 28bd56c959fcd2..00000000000000 --- a/ci/package.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "private": true, - "scripts": { - "bootstrap": "brew install gh jq cirruslabs/cli/tart cirruslabs/cli/sshpass hashicorp/tap/packer && packer init darwin", - "login": "token=$(gh auth token); username=$(gh api user --jq .login); echo \"Login as $username...\"; echo \"$token\" | tart login ghcr.io --username \"$username\" --password-stdin; echo \"$token\" | docker login ghcr.io --username \"$username\" --password-stdin", - "fetch:image-name": "echo ghcr.io/oven-sh/bun-vm", - "fetch:darwin-version": "echo 1", - "fetch:macos-version": "sw_vers -productVersion | cut -d. -f1", - "fetch:script-version": "cat ../scripts/bootstrap.sh | grep 'v=' | sed 's/v=\"//;s/\"//' | head -n 1", - "build:darwin-aarch64-vanilla": "packer build '-only=*.bun-darwin-aarch64-vanilla' -var release=$(bun fetch:macos-version) darwin/", - "build:darwin-aarch64-vanilla-15": "packer build '-only=*.bun-darwin-aarch64-vanilla' -var release=15 darwin/", - "build:darwin-aarch64-vanilla-14": "packer build '-only=*.bun-darwin-aarch64-vanilla' -var release=14 darwin/", - "build:darwin-aarch64-vanilla-13": "packer build '-only=*.bun-darwin-aarch64-vanilla' -var release=13 darwin/", - "build:darwin-aarch64": "packer build '-only=*.bun-darwin-aarch64' -var release=$(bun fetch:macos-version) darwin/", - "build:darwin-aarch64-15": "packer build '-only=*.bun-darwin-aarch64' -var release=15 darwin/", - "build:darwin-aarch64-14": "packer build '-only=*.bun-darwin-aarch64' -var release=14 darwin/", - "build:darwin-aarch64-13": "packer build '-only=*.bun-darwin-aarch64' -var release=13 darwin/", - "publish:darwin-aarch64-vanilla": "image=$(tart list --format json | jq -r \".[] | select(.Name | test(\\\"^bun-darwin-aarch64-vanilla-.*-$(bun fetch:macos-version)$\\\")) | .Name\" | head -n 1 | sed 's/bun-//'); tart push \"bun-$image\" \"ghcr.io/oven-sh/bun-vm:$image-v$(bun fetch:darwin-version)\"", - "publish:darwin-aarch64-vanilla-15": "tart push bun-darwin-aarch64-vanilla-sequoia-15 \"$(bun fetch:image-name):darwin-aarch64-vanilla-sequoia-15-v$(bun fetch:darwin-version)\"", - "publish:darwin-aarch64-vanilla-14": "tart push bun-darwin-aarch64-vanilla-sonoma-14 \"$(bun fetch:image-name):darwin-aarch64-vanilla-sonoma-14-v$(bun fetch:darwin-version)\"", - "publish:darwin-aarch64-vanilla-13": "tart push bun-darwin-aarch64-vanilla-ventura-13 \"$(bun fetch:image-name):darwin-aarch64-vanilla-ventura-13-v$(bun fetch:darwin-version)\"", - "publish:darwin-aarch64": "image=$(tart list --format json | jq -r \".[] | select(.Name | test(\\\"^bun-darwin-aarch64-.*-$(bun fetch:macos-version)$\\\")) | .Name\" | head -n 1 | sed 's/bun-//'); tart push \"bun-$image\" \"ghcr.io/oven-sh/bun-vm:$image-v$(bun fetch:script-version)\"", - "publish:darwin-aarch64-15": "tart push bun-darwin-aarch64-sequoia-15 \"$(bun fetch:image-name):darwin-aarch64-sequoia-15-v$(bun fetch:script-version)\"", - "publish:darwin-aarch64-14": "tart push bun-darwin-aarch64-sonoma-14 \"$(bun fetch:image-name):darwin-aarch64-sonoma-14-v$(bun fetch:script-version)\"", - "publish:darwin-aarch64-13": "tart push bun-darwin-aarch64-ventura-13 \"$(bun fetch:image-name):darwin-aarch64-ventura-13-v$(bun fetch:script-version)\"" - } -} diff --git a/cmake/Globals.cmake b/cmake/Globals.cmake index af66b00f081762..5ae4088b22e109 100644 --- a/cmake/Globals.cmake +++ b/cmake/Globals.cmake @@ -743,6 +743,10 @@ function(register_cmake_command) list(APPEND MAKE_EFFECTIVE_ARGS --fresh) endif() + if(CMAKE_TOOLCHAIN_FILE) + list(APPEND MAKE_EFFECTIVE_ARGS "--toolchain=${CMAKE_TOOLCHAIN_FILE}") + endif() + register_command( COMMENT "Configuring ${MAKE_TARGET}" TARGET configure-${MAKE_TARGET} diff --git a/cmake/targets/BuildBun.cmake b/cmake/targets/BuildBun.cmake index 11759e2f7a9418..87774eca015958 100644 --- a/cmake/targets/BuildBun.cmake +++ b/cmake/targets/BuildBun.cmake @@ -330,7 +330,6 @@ register_command( ${BUN_BAKE_RUNTIME_CODEGEN_SOURCES} ${BUN_BAKE_RUNTIME_CODEGEN_SCRIPT} OUTPUTS - ${CODEGEN_PATH}/bake_empty_file ${BUN_BAKE_RUNTIME_OUTPUTS} ) @@ -507,9 +506,7 @@ set(BUN_ZIG_GENERATED_SOURCES ) # In debug builds, these are not embedded, but rather referenced at runtime. -if (DEBUG) - list(APPEND BUN_ZIG_GENERATED_SOURCES ${CODEGEN_PATH}/bake_empty_file) -else() +if(NOT DEBUG) list(APPEND BUN_ZIG_GENERATED_SOURCES ${BUN_BAKE_RUNTIME_OUTPUTS}) endif() diff --git a/cmake/toolchains/darwin-aarch64.cmake b/cmake/toolchains/darwin-aarch64.cmake index b5a52c3fb2ef5a..ee790a71486f5d 100644 --- a/cmake/toolchains/darwin-aarch64.cmake +++ b/cmake/toolchains/darwin-aarch64.cmake @@ -1,5 +1,4 @@ set(CMAKE_SYSTEM_NAME Darwin) set(CMAKE_SYSTEM_PROCESSOR aarch64) - set(CMAKE_C_COMPILER_WORKS ON) set(CMAKE_CXX_COMPILER_WORKS ON) diff --git a/cmake/toolchains/darwin-x64.cmake b/cmake/toolchains/darwin-x64.cmake index aef2c72d12d181..d4a8f185ecb306 100644 --- a/cmake/toolchains/darwin-x64.cmake +++ b/cmake/toolchains/darwin-x64.cmake @@ -1,6 +1,5 @@ set(CMAKE_SYSTEM_NAME Darwin) set(CMAKE_SYSTEM_PROCESSOR x64) set(CMAKE_OSX_ARCHITECTURES x86_64) - set(CMAKE_C_COMPILER_WORKS ON) set(CMAKE_CXX_COMPILER_WORKS ON) \ No newline at end of file diff --git a/cmake/toolchains/linux-aarch64-musl.cmake b/cmake/toolchains/linux-aarch64-musl.cmake index e4a33f709e88cd..53ea70d6061d11 100644 --- a/cmake/toolchains/linux-aarch64-musl.cmake +++ b/cmake/toolchains/linux-aarch64-musl.cmake @@ -1,6 +1,5 @@ set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR aarch64) set(ABI musl) - set(CMAKE_C_COMPILER_WORKS ON) set(CMAKE_CXX_COMPILER_WORKS ON) \ No newline at end of file diff --git a/cmake/toolchains/linux-aarch64.cmake b/cmake/toolchains/linux-aarch64.cmake index 657594dae8c513..292d403d9a12ea 100644 --- a/cmake/toolchains/linux-aarch64.cmake +++ b/cmake/toolchains/linux-aarch64.cmake @@ -1,6 +1,5 @@ set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR aarch64) set(ABI gnu) - set(CMAKE_C_COMPILER_WORKS ON) set(CMAKE_CXX_COMPILER_WORKS ON) \ No newline at end of file diff --git a/cmake/toolchains/linux-x64-baseline.cmake b/cmake/toolchains/linux-x64-baseline.cmake index 73d6bc61e4946e..81fdfabff66e8f 100644 --- a/cmake/toolchains/linux-x64-baseline.cmake +++ b/cmake/toolchains/linux-x64-baseline.cmake @@ -2,6 +2,5 @@ set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR x64) set(ENABLE_BASELINE ON) set(ABI gnu) - set(CMAKE_C_COMPILER_WORKS ON) set(CMAKE_CXX_COMPILER_WORKS ON) \ No newline at end of file diff --git a/cmake/toolchains/linux-x64-musl-baseline.cmake b/cmake/toolchains/linux-x64-musl-baseline.cmake index ea28a1757ac8d0..a36f60dbb4981b 100644 --- a/cmake/toolchains/linux-x64-musl-baseline.cmake +++ b/cmake/toolchains/linux-x64-musl-baseline.cmake @@ -2,6 +2,5 @@ set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR x64) set(ENABLE_BASELINE ON) set(ABI musl) - set(CMAKE_C_COMPILER_WORKS ON) set(CMAKE_CXX_COMPILER_WORKS ON) \ No newline at end of file diff --git a/cmake/toolchains/linux-x64-musl.cmake b/cmake/toolchains/linux-x64-musl.cmake index db4998bba9d510..57456126109be4 100644 --- a/cmake/toolchains/linux-x64-musl.cmake +++ b/cmake/toolchains/linux-x64-musl.cmake @@ -1,6 +1,5 @@ set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR x64) set(ABI musl) - set(CMAKE_C_COMPILER_WORKS ON) set(CMAKE_CXX_COMPILER_WORKS ON) diff --git a/cmake/toolchains/linux-x64.cmake b/cmake/toolchains/linux-x64.cmake index 4104a1c5df7396..81e7357baa51ef 100644 --- a/cmake/toolchains/linux-x64.cmake +++ b/cmake/toolchains/linux-x64.cmake @@ -1,6 +1,5 @@ set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR x64) set(ABI gnu) - set(CMAKE_C_COMPILER_WORKS ON) set(CMAKE_CXX_COMPILER_WORKS ON) diff --git a/cmake/toolchains/windows-x64-baseline.cmake b/cmake/toolchains/windows-x64-baseline.cmake index fe2df9a9307408..9581ed91735f23 100644 --- a/cmake/toolchains/windows-x64-baseline.cmake +++ b/cmake/toolchains/windows-x64-baseline.cmake @@ -1,6 +1,5 @@ set(CMAKE_SYSTEM_NAME Windows) set(CMAKE_SYSTEM_PROCESSOR x64) set(ENABLE_BASELINE ON) - set(CMAKE_C_COMPILER_WORKS ON) set(CMAKE_CXX_COMPILER_WORKS ON) \ No newline at end of file diff --git a/cmake/toolchains/windows-x64.cmake b/cmake/toolchains/windows-x64.cmake index bb239656dcd61b..4c61f19db494a1 100644 --- a/cmake/toolchains/windows-x64.cmake +++ b/cmake/toolchains/windows-x64.cmake @@ -1,5 +1,4 @@ set(CMAKE_SYSTEM_NAME Windows) set(CMAKE_SYSTEM_PROCESSOR x64) - set(CMAKE_C_COMPILER_WORKS ON) set(CMAKE_CXX_COMPILER_WORKS ON) \ No newline at end of file diff --git a/scripts/agent.mjs b/scripts/agent.mjs index ece3359cc79111..0594c71685550a 100755 --- a/scripts/agent.mjs +++ b/scripts/agent.mjs @@ -2,256 +2,922 @@ // An agent that starts buildkite-agent and runs others services. -import { join } from "node:path"; -import { realpathSync } from "node:fs"; -import { - isWindows, - getOs, - getArch, - getKernel, - getAbi, - getAbiVersion, - getDistro, - getDistroVersion, - getHostname, - getCloud, - getCloudMetadataTag, - which, - getEnv, - writeFile, - spawnSafe, - mkdir, -} from "./utils.mjs"; import { parseArgs } from "node:util"; +import { join, relative } from "node:path"; +import { existsSync, readdirSync, realpathSync } from "node:fs"; +import { getEnv, getUser, getUsername, parseOs, readFile, startGroup } from "./utils.mjs"; +import { rm, mkdir, writeFile, mkdtemp, which, spawn, spawnSafe, spawnSsh, spawnSshSafe, spawnScp } from "./utils.mjs"; +import { isPosix, isLinux, isMacOS, isWindows, getCloud, getCloudMetadataTag } from "./utils.mjs"; +import { getOs, getArch, getKernel, getAbi, getAbiVersion, getDistro, getDistroVersion } from "./utils.mjs"; /** - * @param {"install" | "start"} action + * @returns {string} */ -async function doBuildkiteAgent(action) { - const username = "buildkite-agent"; - const command = which("buildkite-agent", { required: true }); - - let homePath, cachePath, logsPath, agentLogPath, pidPath; - if (isWindows) { - homePath = "C:\\buildkite-agent"; - cachePath = join(homePath, "cache"); - logsPath = join(homePath, "logs"); - agentLogPath = join(logsPath, "buildkite-agent.log"); +function getAgentName() { + let name = `${getOs()}-${getArch()}`; + if (isLinux) { + name += `-${getDistro()}-${getDistroVersion()}`; } else { - homePath = "/var/lib/buildkite-agent"; - cachePath = "/var/cache/buildkite-agent"; - logsPath = "/var/log/buildkite-agent"; - agentLogPath = join(logsPath, "buildkite-agent.log"); - pidPath = join(logsPath, "buildkite-agent.pid"); + name += `-${getDistroVersion()}`; } + return name; +} - async function install() { - const command = process.execPath; - const args = [realpathSync(process.argv[1]), "start"]; - - if (isWindows) { - mkdir(logsPath); - - const nssm = which("nssm", { required: true }); - const nssmCommands = [ - [nssm, "install", "buildkite-agent", command, ...args], - [nssm, "set", "buildkite-agent", "Start", "SERVICE_AUTO_START"], - [nssm, "set", "buildkite-agent", "AppDirectory", homePath], - [nssm, "set", "buildkite-agent", "AppStdout", agentLogPath], - [nssm, "set", "buildkite-agent", "AppStderr", agentLogPath], - ]; - for (const command of nssmCommands) { - await spawnSafe(command, { stdio: "inherit" }); - } - } +/** + * @typedef {Object} AgentLocation + * @property {string} agentPath + * @property {string} homePath + * @property {string} configPath + * @property {string} cachePath + * @property {string} logsPath + * @property {string} agentLogPath + * @property {string} tmpPath + */ - if (isOpenRc()) { - const servicePath = "/etc/init.d/buildkite-agent"; - const service = `#!/sbin/openrc-run - name="buildkite-agent" - description="Buildkite Agent" - command=${escape(command)} - command_args=${escape(args.map(escape).join(" "))} - command_user=${escape(username)} - - pidfile=${escape(pidPath)} - start_stop_daemon_args=" \\ - --background \\ - --make-pidfile \\ - --stdout ${escape(agentLogPath)} \\ - --stderr ${escape(agentLogPath)}" - - depend() { - need net - use dns logger - } - `; - writeFile(servicePath, service, { mode: 0o755 }); - await spawnSafe(["rc-update", "add", "buildkite-agent", "default"], { stdio: "inherit", privileged: true }); - } +/** + * @param {"windows" | "linux" | "darwin"} [os] + * @param {string} [username] + * @returns {AgentLocation} + */ +function getAgentLocation(os = getOs(), username = getUsername()) { + const agentPath = which("buildkite-agent", { required: true }); + + if (os === "windows") { + const homePath = "C:\\buildkite-agent"; + return { + agentPath, + homePath, + configPath: join(homePath, "buildkite-agent.cfg"), + cachePath: join(homePath, "cache"), + logsPath: join(homePath, "logs"), + agentLogPath: join(homePath, "logs", "buildkite-agent.log"), + tmpPath: "C:\\Windows\\TEMP", + }; + } - if (isSystemd()) { - const servicePath = "/etc/systemd/system/buildkite-agent.service"; - const service = ` - [Unit] - Description=Buildkite Agent - After=syslog.target - After=network-online.target - - [Service] - Type=simple - User=${username} - ExecStart=${escape(command)} ${args.map(escape).join(" ")} - RestartSec=5 - Restart=on-failure - KillMode=process - - [Journal] - Storage=persistent - StateDirectory=${escape(agentLogPath)} - - [Install] - WantedBy=multi-user.target - `; - writeFile(servicePath, service); - await spawnSafe(["systemctl", "daemon-reload"], { stdio: "inherit", privileged: true }); - await spawnSafe(["systemctl", "enable", "buildkite-agent"], { stdio: "inherit", privileged: true }); - } + if (os === "darwin") { + const userPath = `/Users/${username === "root" ? "administrator" : username}`; + return { + agentPath, + // FIXME: Library/Application Support/buildkite-agent + // causes issues with the space in the path, fix this later. + homePath: join(userPath, "Library/Services/buildkite-agent"), + configPath: join(userPath, "Library/Preferences/buildkite-agent.cfg"), + cachePath: join(userPath, "Library/Caches/buildkite-agent"), + logsPath: join(userPath, "Library/Logs/buildkite-agent"), + agentLogPath: join(userPath, "Library/Logs/buildkite-agent/buildkite-agent.log"), + tmpPath: "/tmp", + }; } - async function start() { - const cloud = await getCloud(); + return { + agentPath, + homePath: "/var/lib/buildkite-agent", + configPath: "/etc/buildkite-agent/buildkite-agent.cfg", + cachePath: "/var/cache/buildkite-agent", + logsPath: "/var/log/buildkite-agent", + agentLogPath: "/var/log/buildkite-agent/buildkite-agent.log", + tmpPath: "/tmp", + }; +} - let token = getEnv("BUILDKITE_AGENT_TOKEN", false); - if (!token && cloud) { - token = await getCloudMetadataTag("buildkite:token"); - } +/** + * @param {Record} [options] + * @returns {string} + */ +function getAgentConfig(options = {}) { + const lines = Object.entries(options).map(([key, value]) => `${key}=${escape(value)}`); + return `# Generated by scripts/agent.mjs +# https://buildkite.com/docs/agent/v3/configuration + +${lines.join("\n")} +`; +} - if (!token) { - throw new Error( - "Buildkite token not found: either set BUILDKITE_AGENT_TOKEN or add a buildkite:token label to the instance", - ); +/** + * @param {AgentLocation} location + * @returns {Promise} + * @link https://buildkite.com/docs/agent/v3/cli-start + */ +async function getAgentCommand(location) { + const { agentPath, homePath, configPath, cachePath, logsPath } = location; + const cloud = getCloud(); + const command = [agentPath, "start"]; + + let name = getAgentName(); + if (existsSync(configPath)) { + command.push("--config", configPath); + if (readFile(configPath).includes("spawn=")) { + name += "-%spawn"; } + } + command.push("--name", name); + + // If the agent token is set, use it. + // If this is not set, the agent will fail to start. + const agentToken = getEnv("BUILDKITE_AGENT_TOKEN", false); + if (agentToken) { + command.push("--token", agentToken); + } else if (cloud) { + const agentToken = await getCloudMetadataTag("buildkite:token"); + if (agentToken) { + command.push("--token", agentToken); + } + } - let shell; - if (isWindows) { - // Command Prompt has a faster startup time than PowerShell. - // Also, it propogates the exit code of the command, which PowerShell does not. - const cmd = which("cmd", { required: true }); - shell = `"${cmd}" /S /C`; - } else { - const sh = which("sh", { required: true }); - shell = `${sh} -elc`; + // For ephemeral agents, they can be assigned a specific job ID to run. + // This prevents them from being assigned to other jobs. + let ephemeral = false; + if (cloud) { + const jobId = await getCloudMetadataTag("buildkite:job-uuid"); + if (jobId) { + command.push("--disconnect-after-job", "--acquire-job", jobId); + ephemeral = true; } + } - const flags = ["enable-job-log-tmpfile", "no-feature-reporting"]; - const options = { - "name": getHostname(), - "token": token || "xxx", - "shell": shell, - "job-log-path": logsPath, - "build-path": join(homePath, "builds"), - "hooks-path": join(homePath, "hooks"), - "plugins-path": join(homePath, "plugins"), - "experiment": "normalised-upload-paths,resolve-commit-after-checkout,agent-api", - }; + // If the agent is ephemeral, add extra flags to speed up the agent. + if (ephemeral) { + command.push("--git-clone-flags", "-v --depth=1"); + command.push("--git-fetch-flags", "-v --prune --depth=1"); + } + + // On Windows, use Command Prompt, since it's much faster on startup + // and it propogates the exit code of the command, which PowerShell does not. + // On macOS and Linux, use plain sh -l so ~/.profile is sourced. + if (isWindows) { + const cmd = which("cmd", { required: true }); + command.push("--shell", `"${cmd}" /S /C`); + } else { + const sh = which("sh", { required: true }); + command.push("--shell", `${sh} -elc`); + } - let ephemeral; - if (cloud) { - const jobId = await getCloudMetadataTag("buildkite:job-uuid"); - if (jobId) { - options["acquire-job"] = jobId; - flags.push("disconnect-after-job"); - ephemeral = true; + // Ensure that paths are set correctly. + command.push("--job-log-path", logsPath); + command.push("--build-path", join(homePath, "builds")); + command.push("--hooks-path", join(homePath, "hooks")); + command.push("--plugins-path", join(homePath, "plugins")); + if (!ephemeral) { + command.push("--git-mirrors-path", join(cachePath, "git")); + } + + // Enable various feature flags that are not required, but are useful. + command.push( + "--enable-job-log-tmpfile", + "--no-feature-reporting", + "--experiment", + "normalised-upload-paths,resolve-commit-after-checkout,agent-api", + ); + + // Define the tags that will be used to identify the agent. + // Steps can use these tags to specify which agent to run on. + const tags = { + "os": getOs(), + "arch": getArch(), + "posix": isPosix, + "windows": isWindows, + "kernel": getKernel(), + "abi": getAbi(), + "abi-version": getAbiVersion(), + "distro": getDistro(), + "distro-version": getDistroVersion(), + "cloud": getCloud(), + "ephemeral": ephemeral, + // Defined for legacy reasons. + "release": parseInt(getDistroVersion()) || undefined, + }; + + // Steps add these tags to tell robobun that it should create an agent. + // If that is the case, these tags need to be added to the metadata. + if (cloud) { + const extraTags = ["robobun", "robobun2"]; + for (const tag of extraTags) { + const value = await getCloudMetadataTag(tag); + if (typeof value === "string") { + metadata[tag] = value; } } + } - if (ephemeral) { - options["git-clone-flags"] = "-v --depth=1"; - options["git-fetch-flags"] = "-v --prune --depth=1"; - } else { - options["git-mirrors-path"] = join(cachePath, "git"); - } + // Add the tags to the command. + command.push( + "--tags", + Object.entries(tags) + .filter(([, value]) => typeof value !== "undefined" && value !== null) + .map(([key, value]) => `${key}=${value}`) + .join(","), + ); + + return command; +} - const tags = { - "os": getOs(), - "arch": getArch(), - "kernel": getKernel(), - "abi": getAbi(), - "abi-version": getAbiVersion(), - "distro": getDistro(), - "distro-version": getDistroVersion(), - "cloud": cloud, +/** + * @param {ServiceType} [type] + * @returns {Service} + */ +export function getAgentService(type) { + return getService("buildkite-agent", type); +} + +/** + * @param {AgentLocation} location + * @param {ServiceType} [type] + * @returns {Promise} + */ +export async function createAgentService(location, type) { + const service = getAgentService(type); + const { type: serviceType } = service; + const { homePath, logsPath, agentLogPath, configPath } = location; + + // Instead of running the agentPath directly, call this script with the "exec" arguments. + // This allows the agent command to be generated on each startup, instead of at install time, + // which is important because tags can change (for example, a macOS machine upgraded to a new release). + const { execPath, argv } = process; + const scriptPath = realpathSync(argv[1]); + const extraArgs = argv.slice(2); + const args = [scriptPath, "exec", ...extraArgs]; + + if (serviceType === "openrc") { + const pidPath = join(logsPath, "buildkite-agent.pid"); + const serviceConfig = `#!/sbin/openrc-run + name="buildkite-agent" + description="Buildkite Agent" + command=${escape(execPath)} + command_args=${escape(args.map(escape).join(" "))} + command_user=buildkite-agent + pidfile=${escape(pidPath)} + start_stop_daemon_args=" \\ + --background \\ + --make-pidfile \\ + --stdout ${escape(agentLogPath)} \\ + --stderr ${escape(agentLogPath)}" + depend() { + need net + use dns logger + } + `; + await service.install(serviceConfig); + } + + if (serviceType === "systemd") { + const serviceConfig = ` + [Unit] + Description=Buildkite Agent + After=syslog.target + After=network-online.target + + [Service] + Type=simple + User=buildkite-agent + ExecStart=${escape(execPath)} ${args.map(escape).join(" ")} + RestartSec=5 + Restart=on-failure + KillMode=process + + [Journal] + Storage=persistent + StateDirectory=${escape(agentLogPath)} + + [Install] + WantedBy=multi-user.target + `; + await service.install(serviceConfig); + } + + if (serviceType === "nssm") { + const serviceConfig = { + command: [execPath, ...args], + options: { + "AppDirectory": homePath, + "AppStdout": agentLogPath, + "AppStderr": agentLogPath, + }, }; + await service.install(serviceConfig); + } - if (cloud) { - const requiredTags = ["robobun", "robobun2"]; - for (const tag of requiredTags) { - const value = await getCloudMetadataTag(tag); - if (typeof value === "string") { - tags[tag] = value; + if (serviceType === "plist") { + const username = getUsername(); + const serviceConfig = ` + + + + Label + buildkite-agent + UserName + ${username === "root" ? "administrator" : username} + EnvironmentVariables + + PATH + ${process.env.PATH} + + KeepAlive + + SuccessfulExit + + + ProcessType + Interactive + ProgramArguments + + ${execPath} + ${scriptPath} + exec + + RunAtLoad + + StandardErrorPath + ${agentLogPath} + StandardOutPath + ${agentLogPath} + WorkingDirectory + ${homePath} + WatchPaths + + ${configPath} + + + +`; + await service.install(serviceConfig); + } + + return service; +} + +/** + * @param {AgentLocation} location + */ +export async function startAgent(location) { + const command = await getAgentCommand(location); + await spawnSafe(command, { stdio: "inherit" }); +} + +/** + * @param {AgentLocation} location + */ +export async function cleanAgent(location) { + const { homePath, cachePath, tmpPath } = location; + const buildPath = join(homePath, "builds"); + + // Remove the items in the directory, but not the directory itself. + for (const parentPath of [buildPath, cachePath, tmpPath]) { + try { + const entries = readdirSync(parentPath, { encoding: "utf8" }); + for (const entry of entries) { + const entryPath = join(parentPath, entry); + try { + rm(entryPath); + } catch (error) { + console.error(`Failed to remove ${entryPath}:`, error); } } + } catch (error) { + console.error(`Failed to clean ${parentPath}:`, error); } + } +} - options["tags"] = Object.entries(tags) - .filter(([, value]) => value) - .map(([key, value]) => `${key}=${value}`) - .join(","); - - await spawnSafe( - [ - command, - "start", - ...flags.map(flag => `--${flag}`), - ...Object.entries(options).map(([key, value]) => `--${key}=${value}`), - ], - { - stdio: "inherit", +/** + * @returns {Promise} + */ +async function getRebootService() { + if (!isMacOS) { + return; + } + + // Reboot every morning at 6am PT. + // Randomize the minute of the hour to avoid thundering herd. + const hourOfDay = 6; + const minuteOfHour = Math.floor(Math.random() * 60); + + const service = getService("reboot", "plist"); + const serviceConfig = ` + + + + Label + reboot + ProgramArguments + + /sbin/shutdown + -r + now + + StartCalendarInterval + + Hour + ${hourOfDay} + Minute + ${minuteOfHour} + + + +`; + await service.install(serviceConfig); + await service.enable(); + + return service; +} + +/** + * @typedef {"systemd" | "openrc" | "nssm" | "plist"} ServiceType + */ + +/** + * @typedef {Object} Service + * @property {string} name + * @property {ServiceType} type + * @property {(config: string) => Promise} install + * @property {() => Promise} status + * @property {() => Promise} enable + * @property {() => Promise} disable + * @property {() => Promise} start + * @property {() => Promise} stop + * @property {() => Promise} restart + * @property {() => Promise} uninstall + */ + +/** + * @param {string} name + * @param {ServiceType} [type] + * @returns {Service} + */ +function getService(name, type) { + const spawnOptions = { stdio: "inherit", privileged: true }; + + // https://docs.alpinelinux.org/user-handbook/0.1a/Working/openrc.html + if (type === "openrc" || getDistro() === "alpine") { + const rcService = which("rc-service", { required: true }); + const rcUpdate = which("rc-update", { required: true }); + return { + name, + type: "openrc", + async install(config) { + const servicePath = `/etc/init.d/${name}`; + writeFile(servicePath, config, { mode: 0o755 }); + }, + async status() { + await spawnSafe([rcService, "status", name], spawnOptions); + }, + async enable() { + await spawnSafe([rcUpdate, "add", name, "default"], spawnOptions); + }, + async disable() { + await spawnSafe([rcUpdate, "delete", name, "default"], spawnOptions); + }, + async start() { + await spawnSafe([rcService, "start", name], spawnOptions); + }, + async stop() { + await spawnSafe([rcService, "stop", name], spawnOptions); + }, + async restart() { + await spawnSafe([rcService, "restart", name], spawnOptions); + }, + async uninstall() { + await spawnSafe([rcService, "remove", name], spawnOptions); + }, + }; + } + + if (type === "systemd" || isLinux) { + const systemctl = which("systemctl", { required: true }); + return { + name, + type: "systemd", + async install(config) { + const servicePath = `/etc/systemd/system/${name}.service`; + writeFile(servicePath, config, { mode: 0o644 }); + await spawnSafe([systemctl, "daemon-reload"], spawnOptions); + await spawnSafe([systemctl, "enable", name], spawnOptions); + }, + async status() { + await spawnSafe([systemctl, "status", name], spawnOptions); }, - ); + async enable() { + await spawnSafe([systemctl, "enable", name], spawnOptions); + }, + async disable() { + await spawnSafe([systemctl, "disable", name], spawnOptions); + }, + async start() { + await spawnSafe([systemctl, "start", name], spawnOptions); + }, + async stop() { + await spawnSafe([systemctl, "stop", name], spawnOptions); + }, + async restart() { + await spawnSafe([systemctl, "restart", name], spawnOptions); + }, + async uninstall() { + await spawnSafe([systemctl, "disable", name], spawnOptions); + await spawnSafe([systemctl, "stop", name], spawnOptions); + await spawnSafe([systemctl, "remove", name], spawnOptions); + }, + }; } - if (action === "install") { - await install(); - } else if (action === "start") { - await start(); + // https://nssm.cc/commands + if (type === "nssm" || isWindows) { + const nssm = which("nssm", { required: true }); + return { + name, + type: "nssm", + async install(config) { + const { command, options } = config; + await spawnSafe([nssm, "install", name, ...command], spawnOptions); + if (options) { + for (const [key, value] of Object.entries(options)) { + await spawnSafe([nssm, "set", name, key, value], spawnOptions); + } + } + }, + async status() { + await spawnSafe([nssm, "get", name, "State"], spawnOptions); + }, + async enable() { + await spawnSafe([nssm, "set", name, "Start", "SERVICE_AUTO_START"], spawnOptions); + }, + async disable() { + await spawnSafe([nssm, "set", name, "Start", "SERVICE_DISABLED"], spawnOptions); + }, + async start() { + await spawnSafe([nssm, "start", name], spawnOptions); + }, + async stop() { + await spawnSafe([nssm, "stop", name], spawnOptions); + }, + async restart() { + await spawnSafe([nssm, "restart", name], spawnOptions); + }, + async uninstall() { + await spawnSafe([nssm, "remove", name, "confirm"], spawnOptions); + }, + }; } + + // https://developer.apple.com/library/archive/documentation/MacOSX/Conceptual/BPSystemStartup/Chapters/Introduction.html + if (type === "plist" || isMacOS) { + const launchctl = which("launchctl", { required: true }); + const serviceId = `system/${name}`; + return { + name, + type: "plist", + async install(config) { + const servicePath = `/Library/LaunchDaemons/${name}.plist`; + writeFile(servicePath, config, { mode: 0o644 }); + await spawnSafe(["chown", "root:wheel", servicePath], spawnOptions); + const plutil = which("plutil"); + if (plutil) { + await spawnSafe([plutil, "-lint", servicePath], spawnOptions); + } + // For some reason, it must be unloaded before it can be loaded, otherwise: + // Load failed: 5: Input/output error + await spawnSafe([launchctl, "unload", servicePath], spawnOptions); + const { error, exitCode } = await spawn([launchctl, "load", servicePath], spawnOptions); + if (!(exitCode === 0 || exitCode === 3)) { + throw error; + } + }, + async status() { + await spawnSafe([launchctl, "print", serviceId], spawnOptions); + }, + async enable() { + await spawnSafe([launchctl, "enable", serviceId], spawnOptions); + }, + async disable() { + await spawnSafe([launchctl, "disable", serviceId], spawnOptions); + }, + async start() { + const { error, exitCode } = await spawn([launchctl, "start", serviceId], spawnOptions); + if (!(exitCode === 0 || exitCode === 3)) { + throw error; + } + }, + async stop() { + await spawnSafe([launchctl, "stop", serviceId], spawnOptions); + }, + async restart() { + await spawnSafe([launchctl, "stop", serviceId], spawnOptions); + await spawnSafe([launchctl, "start", serviceId], spawnOptions); + }, + async uninstall() { + await spawnSafe([launchctl, "unload", serviceId], spawnOptions); + }, + }; + } + + throw new Error(`Unsupported service type: ${type}`); } /** - * @returns {boolean} + * @param {string} string + * @returns {string} */ -function isSystemd() { - return !!which("systemctl"); +function escape(string) { + return JSON.stringify(string); } /** - * @returns {boolean} + * @param {import("./utils.mjs").SshOptions} sshOptions + * @returns {Promise<"windows" | "linux" | "darwin">} */ -function isOpenRc() { - return !!which("rc-service"); +async function getOsSsh(sshOptions) { + try { + const { error: unameError, stdout: uname } = await spawnSsh({ ...sshOptions, command: ["uname", "-s"] }); + if (!unameError) { + return parseOs(uname); + } + const { error: cmdError, stdout: cmd } = await spawnSsh({ ...sshOptions, command: ["cmd", "/c", "ver"] }); + if (!cmdError) { + return parseOs(cmd); + } + throw unameError || cmdError; + } catch (cause) { + const { hostname } = sshOptions; + throw new Error(`Failed to determine the machine's platform: ${hostname}`, { cause }); + } } -function escape(string) { - return JSON.stringify(string); +/** + * @param {import("./utils.mjs").SshOptions} sshOptions + * @returns {Promise} + */ +async function getSudoCommand(sshOptions) { + const { password } = sshOptions; + const { exitCode } = await spawnSsh({ ...sshOptions, command: ["sudo", "echo", "1"] }); + if (exitCode === 0) { + return ["sudo"]; + } + + if (password) { + const { exitCode } = await spawnSsh({ + ...sshOptions, + command: [ + "sh", + "-c", + `echo '${password}' | sudo -S sh -c 'echo \"%admin ALL=(ALL) NOPASSWD: ALL\" | tee -a /etc/sudoers.d/nopasswd'`, + ], + }); + if (exitCode === 0) { + return ["sudo"]; + } + } + + return []; +} + +/** + * @param {import("./utils.mjs").SshOptions} sshOptions + * @returns {Promise} + */ +async function uploadBootstrap(sshOptions) { + const os = await getOsSsh(sshOptions); + const { tmpPath } = getAgentLocation(os); + + const filename = os === "windows" ? "bootstrap.ps1" : "bootstrap.sh"; + const scriptPath = join(import.meta.dirname, filename); + const bootstrapPath = join(tmpPath, filename); + await spawnScp({ + ...sshOptions, + source: scriptPath, + destination: bootstrapPath, + }); + + if (os === "windows") { + return ["powershell", "-ExecutionPolicy", "Bypass", "-File", bootstrapPath, "-CI"]; + } + + const _ = await getSudoCommand(sshOptions); + return ["sh", bootstrapPath, "--ci"]; +} + +/** + * @param {import("./utils.mjs").SshOptions} sshOptions + * @returns {Promise} + */ +async function uploadScript(sshOptions) { + const os = await getOsSsh(sshOptions); + + const runnerPath = which(["bunx", "npx"], { required: true }); + const scriptPath = realpathSync(process.argv[1]); + const localTmpScriptPath = mkdtemp("agent-", "agent.mjs"); + await spawnSafe([ + runnerPath, + "esbuild", + scriptPath, + "--bundle", + "--platform=node", + "--format=esm", + `--outfile=${localTmpScriptPath}`, + ]); + + const { username } = sshOptions; + const location = getAgentLocation(os, username); + const { tmpPath, homePath } = location; + + const tmpScriptPath = join(tmpPath, "agent.mjs"); + await spawnScp({ + ...sshOptions, + source: localTmpScriptPath, + destination: tmpScriptPath, + }); + + const command = []; + if (os !== "windows") { + const sudoCommand = await getSudoCommand(sshOptions); + command.push(...sudoCommand); + } + + await spawnSshSafe({ + ...sshOptions, + command: [...command, "mkdir", "-p", homePath], + }); + + const agentScriptPath = join(homePath, "agent.mjs"); + await spawnSshSafe({ + ...sshOptions, + command: [...command, "cp", tmpScriptPath, agentScriptPath], + }); + + const { stdout: nodeStdout } = await spawnSsh({ ...sshOptions, command: ["node", "-v"] }); + const nodeVersion = parseInt(nodeStdout.trim().replace(/^v/, "")); + if (isNaN(nodeVersion) || nodeVersion < 20) { + command.push("bun"); + } else { + command.push("node"); + } + + return [...command, agentScriptPath]; } async function main() { - const { positionals: args } = parseArgs({ + const { positionals: args, values: options } = parseArgs({ allowPositionals: true, + options: { + "hostname": { type: "string", multiple: true }, + "username": { type: "string" }, + "password": { type: "string" }, + "bootstrap": { type: "boolean" }, + "token": { type: "string" }, + "queue": { type: "string" }, + "spawn": { type: "string" }, + "tailscale-authkey": { type: "string" }, + }, }); - if (!args.length || args.includes("install")) { - console.log("Installing agent..."); - await doBuildkiteAgent("install"); - console.log("Agent installed."); + if (!args.length) { + const scriptPath = relative(process.cwd(), process.argv[1]); + console.error(`Usage: ${scriptPath} [install|enable|disable|start|stop|restart|uninstall]`); + console.error(); + console.error(`Options:`); + console.error(` --hostname=string: The hostname of the machine to connect to (can specify multiple).`); + console.error(` --username=string: The username of the machine to connect to.`); + console.error(` --password=string: The password of the machine to connect to.`); + console.error(` --bootstrap: If true, run the bootstrap script on the machine.`); + console.error(` --token=string: The Buildkite token to use.`); + console.error(` --queue=string: The name of the Buildkite queue to use (e.g. "build-zig").`); + console.error(` --spawn=number: The number of agents to run on the machine (default: 1).`); + console.error(` --tailscale-authkey=string: The Tailscale authkey to use.`); + console.error(); + console.error(`Examples:`); + console.error(`1. Install the agent on the local machine:`); + console.error(` ${scriptPath} install`); + console.error(); + console.error(`2. Install the agent on a remote machine:`); + console.error(` ${scriptPath} --hostname=127.0.0.1 --username=admin install`); + console.error(); + console.error(`3. Start the agent on a remote machine:`); + console.error(` ${scriptPath} --hostname=127.0.0.1 --username=admin start`); + console.error(); + console.error(`4. Stop the agent on a remote machine (with a password):`); + console.error(` ${scriptPath} --hostname=127.0.0.1 --username=administrator --password=admin stop`); + process.exit(1); } - if (args.includes("start")) { - console.log("Starting agent..."); - await doBuildkiteAgent("start"); - console.log("Agent started."); + const { hostname: hostnames, username, password, bootstrap, ...agentOptions } = options; + + // When a hostname is defined, connect to the machine, + // bundle and upload this script, then run it on that machine. + if (hostnames?.length) { + for (const hostname of hostnames) { + const sshOptions = { hostname, username, password }; + + if (bootstrap) { + await startGroup(`Running bootstrap script on ${hostname}...`, async () => { + const command = await uploadBootstrap(sshOptions); + await spawnSshSafe( + { + ...sshOptions, + command, + }, + { + stdio: "inherit", + }, + ); + }); + } + + await startGroup(`Uploading script to ${hostname}...`, async () => { + const command = await uploadScript(sshOptions); + const commandWithoutSsh = [ + ...command, + ...args, + ...Object.entries(agentOptions).map(([key, value]) => `--${key}=${value}`), + ]; + await spawnSshSafe( + { + ...sshOptions, + command: commandWithoutSsh, + }, + { + stdio: "inherit", + }, + ); + }); + } + + // Return after running the script on the remote machine. + return; + } + + const { uid, gid } = getUser(); + const fileOptions = { mode: 0o777, uid, gid }; + + const location = getAgentLocation(); + const { "tailscale-authkey": tailscaleAuthkey, ...agentConfig } = agentOptions; + const { tmpPath, agentPath, configPath, agentLogPath, ...locationPaths } = location; + + if (args.includes("clean") || args.includes("exec")) { + await startGroup("Cleaning agent...", async () => { + await cleanAgent(location); + }); + } + + if (args.includes("exec")) { + return startGroup("Running agent...", async () => { + await startAgent(location); + }); + } + + if (args.includes("install")) { + for (const path of Object.values(locationPaths)) { + await startGroup(`Creating directory: ${path}`, async () => { + mkdir(path, fileOptions); + }); + } + + if (!existsSync(configPath) || Object.values(agentConfig).some(Boolean)) { + await startGroup(`Creating config: ${configPath}`, async () => { + const configFile = getAgentConfig(agentConfig); + writeFile(configPath, configFile, fileOptions); + }); + } + + if (tailscaleAuthkey) { + await startGroup("Setting up Tailscale...", async () => { + const tailscale = which("tailscale", { required: true }); + const hostname = getAgentName(); + await spawnSafe( + [tailscale, "up", "--accept-risk=all", "--ssh", "--hostname", hostname, "--auth-key", tailscaleAuthkey], + { + stdio: "inherit", + }, + ); + }); + } + + await startGroup("Installing agent...", async () => { + const service = await createAgentService(location); + const { name, type } = service; + console.log(`Created agent: ${name} (${type})`); + }); + + if (isMacOS) { + await startGroup("Installing reboot service...", async () => { + const rebootService = await getRebootService(); + if (rebootService) { + const { name, type } = rebootService; + console.log(`Created service: ${name} (${type})`); + } + }); + } + } else { + const service = getAgentService(); + const { name, type } = service; + console.log(`Found agent: ${name} (${type})`); + + for (const arg of args) { + startGroup(`Running command: ${arg}...`, async () => { + if (service[arg]) { + await service[arg](); + } else { + throw new Error(`Unsupported command: ${arg}`); + } + }); + } } } diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index f5f793fa3aa2a8..3646bd4444c454 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -19,9 +19,7 @@ print() { error() { print "error: $@" >&2 - if ! [ "$$" = "$pid" ]; then - kill -s TERM "$pid" - fi + kill -s TERM "$pid" exit 1 } @@ -59,14 +57,14 @@ execute_as_user() { } grant_to_user() { - path="$1" - if ! [ -f "$path" ] && ! [ -d "$path" ]; then - error "Could not find file or directory: \"$path\"" + grant_path="$1" + if ! [ -f "$grant_path" ] && ! [ -d "$grant_path" ]; then + error "Could not find file or directory: \"$grant_path\"" fi chown="$(require chown)" - execute_sudo "$chown" -R "$user:$group" "$path" - execute_sudo chmod -R 777 "$path" + execute_sudo "$chown" -R "$user:$group" "$grant_path" + execute_sudo chmod -R 777 "$grant_path" } which() { @@ -74,11 +72,11 @@ which() { } require() { - path="$(which "$1")" - if ! [ -f "$path" ]; then + exe_path="$(which "$1")" + if ! [ -f "$exe_path" ]; then error "Command \"$1\" is required, but is not installed." fi - print "$path" + print "$exe_path" } fetch() { @@ -106,79 +104,79 @@ compare_version() { } create_directory() { - path="$1" - path_dir="$path" - while ! [ -d "$path_dir" ]; do - path_dir="$(dirname "$path_dir")" + dir_path="$1" + dir_parent_path="$dir_path" + while ! [ -d "$dir_parent_path" ]; do + dir_parent_path="$(dirname "$dir_parent_path")" done - path_needs_sudo="0" - if ! [ -r "$path_dir" ] || ! [ -w "$path_dir" ]; then - path_needs_sudo="1" + dir_needs_sudo="0" + if ! [ -r "$dir_parent_path" ] || ! [ -w "$dir_parent_path" ]; then + dir_needs_sudo="1" fi mkdir="$(require mkdir)" - if [ "$path_needs_sudo" = "1" ]; then - execute_sudo "$mkdir" -p "$path" + if [ "$dir_needs_sudo" = "1" ]; then + execute_sudo "$mkdir" -p "$dir_path" else - execute "$mkdir" -p "$path" + execute "$mkdir" -p "$dir_path" fi - grant_to_user "$path" + grant_to_user "$dir_path" } create_tmp_directory() { mktemp="$(require mktemp)" - path="$(execute "$mktemp" -d)" - grant_to_user "$path" - print "$path" + tmp_path="$(execute "$mktemp" -d)" + grant_to_user "$tmp_path" + print "$tmp_path" } create_file() { - path="$1" - path_dir="$(dirname "$path")" - if ! [ -d "$path_dir" ]; then - create_directory "$path_dir" + file_path="$1" + file_parent_path="$(dirname "$file_path")" + if ! [ -d "$file_parent_path" ]; then + create_directory "$file_parent_path" fi - path_needs_sudo="0" - if ! [ -r "$path" ] || ! [ -w "$path" ]; then - path_needs_sudo="1" + file_needs_sudo="0" + if ! [ -r "$file_path" ] || ! [ -w "$file_path" ]; then + file_needs_sudo="1" fi - if [ "$path_needs_sudo" = "1" ]; then - execute_sudo touch "$path" + if [ "$file_needs_sudo" = "1" ]; then + execute_sudo touch "$file_path" else - execute touch "$path" + execute touch "$file_path" fi - content="$2" - if [ -n "$content" ]; then - append_file "$path" "$content" + file_content="$2" + if [ -n "$file_content" ]; then + append_file "$file_path" "$file_content" fi - grant_to_user "$path" + grant_to_user "$file_path" } append_file() { - path="$1" - if ! [ -f "$path" ]; then - create_file "$path" + file_path="$1" + if ! [ -f "$file_path" ]; then + create_file "$file_path" fi - path_needs_sudo="0" - if ! [ -r "$path" ] || ! [ -w "$path" ]; then - path_needs_sudo="1" + file_needs_sudo="0" + if ! [ -r "$file_path" ] || ! [ -w "$file_path" ]; then + file_needs_sudo="1" fi content="$2" print "$content" | while read -r line; do - if ! grep -q "$line" "$path"; then + if ! grep -q "$line" "$file_path"; then sh="$(require sh)" - if [ "$path_needs_sudo" = "1" ]; then - execute_sudo "$sh" -c "echo '$line' >> '$path'" + if [ "$file_needs_sudo" = "1" ]; then + execute_sudo "$sh" -c "echo '$line' >> '$file_path'" else - execute "$sh" -c "echo '$line' >> '$path'" + execute "$sh" -c "echo '$line' >> '$file_path'" fi fi done @@ -197,7 +195,7 @@ download_file() { append_to_profile() { content="$1" - profiles=".profile .zprofile .bash_profile .bashrc .zshrc" + profiles=".profile .zprofile .bash_profile .bashrc .zshrc .zshenv" for profile in $profiles; do for profile_path in "$current_home/$profile" "$home/$profile"; do if [ "$ci" = "1" ] || [ -f "$profile_path" ]; then @@ -208,13 +206,13 @@ append_to_profile() { } append_to_path() { - path="$1" - if ! [ -d "$path" ]; then - error "Could not find directory: \"$path\"" + bin_path="$1" + if ! [ -d "$bin_path" ]; then + error "Could not find directory: \"$bin_path\"" fi - append_to_profile "export PATH=\"$path:\$PATH\"" - export PATH="$path:$PATH" + append_to_profile "export PATH=\"$bin_path:\$PATH\"" + export PATH="$bin_path:$PATH" } move_to_bin() { @@ -231,6 +229,13 @@ move_to_bin() { done grant_to_user "$exe_path" + + if ! [ -d "$usr_path" ]; then + execute_sudo mkdir -p "$usr_path" + grant_to_user "$usr_path" + fi + + append_to_path "$usr_path" execute_sudo mv -f "$exe_path" "$usr_path/$(basename "$exe_path")" } @@ -430,17 +435,18 @@ check_package_manager() { check_user() { print "Checking user..." + id="$(require id)" if [ -n "$SUDO_USER" ]; then user="$SUDO_USER" else - id="$(require id)" user="$("$id" -un)" - group="$("$id" -gn)" fi if [ -z "$user" ]; then error "Could not determine user" fi print "User: $user" + + group="$("$id" -gn "$user")" print "Group: $group" home="$(execute_as_user echo '~')" @@ -449,13 +455,14 @@ check_user() { fi print "Home: $home" - id="$(which id)" - if [ -f "$id" ] && [ "$($id -u)" = "0" ]; then - sudo=1 - print "Sudo: enabled" - elif [ -f "$(which sudo)" ] && [ "$(sudo -n echo 1 2>/dev/null)" = "1" ]; then - can_sudo=1 - print "Sudo: can be used" + if [ -f "$(which sudo)" ] && [ -f "$(which id)" ]; then + if [ "$(id -u)" = "0" ]; then + sudo=1 + print "Sudo: enabled" + elif [ "$(sudo -n echo 1 2>/dev/null)" = "1" ]; then + can_sudo=1 + print "Sudo: can be used" + fi fi current_user="$user" @@ -639,7 +646,7 @@ install_brew() { bash="$(require bash)" script=$(download_file "https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh") - execute_as_user "$bash" -lc "NONINTERACTIVE=1 $script" + execute_as_user "$bash" -lc "$script" case "$arch" in x64) @@ -793,7 +800,7 @@ install_bun() { bun_zip="$(download_file "$bun_download_url")" bun_tmpdir="$(dirname "$bun_zip")" execute "$unzip" -o "$bun_zip" -d "$bun_tmpdir" - + move_to_bin "$bun_tmpdir/$bun_triplet/bun" bun_path="$(require bun)" execute_sudo ln -sf "$bun_path" "$(dirname "$bun_path")/bunx" @@ -1021,7 +1028,7 @@ install_rust() { sh="$(require sh)" rustup_script=$(download_file "https://sh.rustup.rs") - execute "$sh" -lc "$rustup_script -y --no-modify-path" + execute "$sh" -lc "CARGO_HOME=$rust_home RUSTUP_HOME=$rust_home $rustup_script -y --no-modify-path" append_to_path "$rust_home/bin" ;; esac @@ -1065,7 +1072,7 @@ install_docker() { fi getent="$(which getent)" - if [ -n "$("$getent" group docker)" ]; then + if [ -f "$getent" ] && [ -n "$("$getent" group docker)" ]; then usermod="$(which usermod)" if [ -f "$usermod" ]; then execute_sudo "$usermod" -aG docker "$user" @@ -1128,6 +1135,10 @@ install_tailscale() { install_packages go execute_as_user go install tailscale.com/cmd/tailscale{,d}@latest append_to_path "$home/go/bin" + tailscaled_path="$(which tailscaled)" + if [ -f "$tailscaled_path" ]; then + execute_sudo "$tailscaled_path" install-system-daemon + fi ;; esac } @@ -1312,6 +1323,112 @@ clean_system() { done } +optimize_system() { + if ! [ "$ci" = "1" ]; then + return + fi + + case "$os" in + darwin) + optimize_system_darwin + ;; + esac +} + +optimize_system_darwin() { + print "Optimizing macOS system..." + + disable_software_update() { + execute_sudo softwareupdate --schedule off + execute_sudo defaults write com.apple.SoftwareUpdate AutomaticDownload -bool false + execute_sudo defaults write com.apple.SoftwareUpdate AutomaticCheckEnabled -bool false + execute_sudo defaults write com.apple.SoftwareUpdate ConfigDataInstall -int 0 + execute_sudo defaults write com.apple.SoftwareUpdate CriticalUpdateInstall -int 0 + execute_sudo defaults write com.apple.SoftwareUpdate ScheduleFrequency -int 0 + execute_sudo defaults write com.apple.SoftwareUpdate AutomaticDownload -int 0 + execute_sudo defaults write com.apple.commerce AutoUpdate -bool false + execute_sudo defaults write com.apple.commerce AutoUpdateRestartRequired -bool false + } + + disable_spotlight() { + execute_sudo mdutil -i off -a + execute_sudo mdutil -E / + } + + disable_siri() { + execute_sudo launchctl unload -w /System/Library/LaunchAgents/com.apple.Siri.agent.plist + execute_sudo defaults write com.apple.Siri StatusMenuVisible -bool false + execute_sudo defaults write com.apple.Siri UserHasDeclinedEnable -bool true + execute_sudo defaults write com.apple.assistant.support "Assistant Enabled" 0 + } + + disable_sleep() { + execute_sudo systemsetup -setsleep Never + execute_sudo systemsetup -setcomputersleep Never + execute_sudo systemsetup -setdisplaysleep Never + execute_sudo systemsetup -setharddisksleep Never + } + + disable_screen_saver() { + execute_sudo defaults write com.apple.screensaver loginWindowIdleTime 0 + execute_sudo defaults write com.apple.screensaver idleTime 0 + } + + disable_screen_lock() { + execute_sudo defaults write com.apple.loginwindow DisableScreenLock -bool true + } + + disable_wallpaper() { + execute_sudo defaults write com.apple.loginwindow DesktopPicture "" + } + + disable_application_state() { + execute_sudo defaults write com.apple.loginwindow TALLogoutSavesState -bool false + } + + disable_accessibility() { + execute_sudo defaults write com.apple.Accessibility DifferentiateWithoutColor -int 1 + execute_sudo defaults write com.apple.Accessibility ReduceMotionEnabled -int 1 + execute_sudo defaults write com.apple.universalaccess reduceMotion -int 1 + execute_sudo defaults write com.apple.universalaccess reduceTransparency -int 1 + } + + disable_dashboard() { + execute_sudo defaults write com.apple.dashboard mcx-disabled -boolean YES + } + + disable_animations() { + execute_sudo defaults write NSGlobalDomain NSAutomaticWindowAnimationsEnabled -bool false + execute_sudo defaults write -g QLPanelAnimationDuration -float 0 + execute_sudo defaults write com.apple.finder DisableAllAnimations -bool true + } + + disable_time_machine() { + execute_sudo tmutil disable + } + + enable_performance_mode() { + # https://support.apple.com/en-us/101992 + if ! [ $(nvram boot-args 2>/dev/null | grep -q serverperfmode) ]; then + execute_sudo nvram boot-args="serverperfmode=1 $(nvram boot-args 2>/dev/null | cut -f 2-)" + fi + } + + disable_software_update + disable_spotlight + disable_siri + disable_sleep + disable_screen_saver + disable_screen_lock + disable_wallpaper + disable_application_state + disable_accessibility + disable_dashboard + disable_animations + disable_time_machine + enable_performance_mode +} + main() { check_features "$@" check_operating_system @@ -1324,6 +1441,7 @@ main() { install_build_essentials install_chromium clean_system + optimize_system } main "$@" diff --git a/scripts/build.mjs b/scripts/build.mjs index 2fab14a9596712..cea7a88c9395bd 100755 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -59,10 +59,16 @@ async function build(args) { generateOptions["-S"] = process.cwd(); } + const toolchain = generateOptions["--toolchain"]; + if (toolchain) { + const toolchainPath = resolve(import.meta.dirname, "..", "cmake", "toolchains", `${toolchain}.cmake`); + generateOptions["--toolchain"] = toolchainPath; + } + const cacheRead = isCacheReadEnabled(); const cacheWrite = isCacheWriteEnabled(); if (cacheRead || cacheWrite) { - const cachePath = getCachePath(); + const cachePath = getCachePath(undefined); if (cacheRead && !existsSync(cachePath)) { const mainCachePath = getCachePath(getDefaultBranch()); if (existsSync(mainCachePath)) { @@ -98,12 +104,6 @@ async function build(args) { } } - const toolchain = generateOptions["--toolchain"]; - if (toolchain) { - const toolchainPath = resolve(import.meta.dirname, "..", "cmake", "toolchains", `${toolchain}.cmake`); - generateOptions["--toolchain"] = toolchainPath; - } - const generateArgs = Object.entries(generateOptions).flatMap(([flag, value]) => flag.startsWith("-D") ? [`${flag}=${value}`] : [flag, value], ); @@ -132,7 +132,7 @@ function cmakePath(path) { return path.replace(/\\/g, "/"); } -function getCachePath(branch) { +function getCachePath(branch, toolchain) { const buildPath = process.env.BUILDKITE_BUILD_PATH; const repository = process.env.BUILDKITE_REPO; const fork = process.env.BUILDKITE_PULL_REQUEST_REPO; diff --git a/scripts/machine.mjs b/scripts/machine.mjs index 8d0ae49ec0dbd5..c8d1aa2b8f0384 100755 --- a/scripts/machine.mjs +++ b/scripts/machine.mjs @@ -16,8 +16,8 @@ import { startGroup, spawnSshSafe, spawnSsh, + spawnScp, tmpdir, - waitForPort, which, escapePowershell, getGithubUrl, @@ -33,7 +33,6 @@ import { isWindows, setupUserData, sha256, - isPrivileged, getUsernameForDistro, } from "./utils.mjs"; import { basename, extname, join, relative, resolve } from "node:path"; @@ -908,74 +907,6 @@ async function getGithubOrgSshKeys(organization) { return sshKeys.flat(); } -/** - * @typedef SshOptions - * @property {string} hostname - * @property {number} [port] - * @property {string} [username] - * @property {string} [password] - * @property {string[]} [command] - * @property {string[]} [identityPaths] - * @property {number} [retries] - */ - -/** - * @typedef ScpOptions - * @property {string} hostname - * @property {string} source - * @property {string} destination - * @property {string[]} [identityPaths] - * @property {string} [port] - * @property {string} [username] - * @property {number} [retries] - */ - -/** - * @param {ScpOptions} options - * @returns {Promise} - */ -async function spawnScp(options) { - const { hostname, port, username, identityPaths, password, source, destination, retries = 10 } = options; - await waitForPort({ hostname, port: port || 22 }); - - const command = ["scp", "-o", "StrictHostKeyChecking=no"]; - if (!password) { - command.push("-o", "BatchMode=yes"); - } - if (port) { - command.push("-P", port); - } - if (password) { - const sshPass = which("sshpass", { required: true }); - command.unshift(sshPass, "-p", password); - } else if (identityPaths) { - command.push(...identityPaths.flatMap(path => ["-i", path])); - } - command.push(resolve(source)); - if (username) { - command.push(`${username}@${hostname}:${destination}`); - } else { - command.push(`${hostname}:${destination}`); - } - - let cause; - for (let i = 0; i < retries; i++) { - const result = await spawn(command, { stdio: "inherit" }); - const { exitCode, stderr } = result; - if (exitCode === 0) { - return; - } - - cause = stderr.trim() || undefined; - if (/(bad configuration option)|(no such file or directory)/i.test(stderr)) { - break; - } - await new Promise(resolve => setTimeout(resolve, Math.pow(2, i) * 1000)); - } - - throw new Error(`SCP failed: ${source} -> ${username}@${hostname}:${destination}`, { cause }); -} - /** * @param {string} passwordData * @param {string} privateKeyPath diff --git a/scripts/utils.mjs b/scripts/utils.mjs index 18aa8c51c83a27..eaf70a9f69e4fd 100755 --- a/scripts/utils.mjs +++ b/scripts/utils.mjs @@ -6,6 +6,7 @@ import { createHash } from "node:crypto"; import { appendFileSync, chmodSync, + chownSync, copyFileSync, existsSync, mkdirSync, @@ -936,6 +937,13 @@ export function readFile(filename, options = {}) { return content; } +/** + * @typedef {object} FileOptions + * @property {number} [mode] + * @property {number} [uid] + * @property {number} [gid] + */ + /** * @param {string} path * @param {number} mode @@ -945,11 +953,20 @@ export function chmod(path, mode) { chmodSync(path, mode); } +/** + * @param {string} path + * @param {number | string} [uid] + * @param {number | string} [gid] + */ +export function chown(path, uid, gid) { + debugLog("$", "chown", path, uid, gid); + chownSync(path, uid, gid); +} + /** * @param {string} filename * @param {string | Buffer} content - * @param {object} [options] - * @param {number} [options.mode] + * @param {FileOptions} [options] */ export function writeFile(filename, content, options) { mkdir(dirname(filename)); @@ -959,14 +976,15 @@ export function writeFile(filename, content, options) { if (options?.mode) { chmod(filename, options.mode); + } else if (options?.uid || options?.gid) { + chown(filename, options.uid, options.gid); } } /** * @param {string} source * @param {string} destination - * @param {object} [options] - * @param {number} [options.mode] + * @param {FileOptions} [options] */ export function copyFile(source, destination, options) { mkdir(dirname(destination)); @@ -976,21 +994,26 @@ export function copyFile(source, destination, options) { if (options?.mode) { chmod(destination, options.mode); + } else if (options?.uid || options?.gid) { + chown(destination, options.uid, options.gid); } } /** * @param {string} path - * @param {object} [options] - * @param {number} [options.mode] + * @param {FileOptions} [options] */ export function mkdir(path, options = {}) { - if (existsSync(path)) { - return; + if (!existsSync(path)) { + debugLog("$", "mkdir", path); + mkdirSync(path, { ...options, recursive: true }); } - debugLog("$", "mkdir", path); - mkdirSync(path, { ...options, recursive: true }); + if (options?.mode) { + chmod(path, options.mode); + } else if (options?.uid || options?.gid) { + chown(path, options.uid, options.gid); + } } /** @@ -1853,14 +1876,63 @@ export function getHostname() { return hostname(); } +/** + * @returns {number} + */ +export function getUid() { + const originalUid = getEnv("SUDO_UID", false); + if (originalUid) { + return parseInt(originalUid); + } + + const { uid } = userInfo(); + return uid; +} + /** * @returns {string} */ export function getUsername() { + const originalUsername = getEnv("SUDO_USER", false); + if (originalUsername) { + return originalUsername; + } + const { username } = userInfo(); return username; } +/** + * @returns {number} + */ +export function getGid() { + const originalGid = getEnv("SUDO_GID", false); + if (originalGid) { + return parseInt(originalGid); + } + + const { gid } = userInfo(); + return gid; +} + +/** + * @typedef {object} User + * @property {string} username + * @property {number} uid + * @property {number} gid + */ + +/** + * @returns {User} + */ +export function getUser() { + return { + username: getUsername(), + uid: getUid(), + gid: getGid(), + }; +} + /** * @param {string} distro * @returns {string} @@ -1889,30 +1961,6 @@ export function getUsernameForDistro(distro) { throw new Error(`Unsupported distro: ${distro}`); } -/** - * @typedef {object} User - * @property {string} username - * @property {number} uid - * @property {number} gid - */ - -/** - * @param {string} username - * @returns {Promise} - */ -export async function getUser(username) { - if (isWindows) { - throw new Error("TODO: Windows"); - } - - const [uid, gid] = await Promise.all([ - spawnSafe(["id", "-u", username]).then(({ stdout }) => parseInt(stdout.trim())), - spawnSafe(["id", "-g", username]).then(({ stdout }) => parseInt(stdout.trim())), - ]); - - return { username, uid, gid }; -} - /** * @returns {string | undefined} */ @@ -2026,19 +2074,19 @@ let detectedCloud; /** * @returns {Promise} */ -export async function isAws() { +export function isAws() { if (typeof detectedCloud === "string") { return detectedCloud === "aws"; } - async function checkAws() { + function checkAws() { if (isLinux) { const kernel = release(); if (kernel.endsWith("-aws")) { return true; } - const { error: systemdError, stdout } = await spawn(["systemd-detect-virt"]); + const { error: systemdError, stdout } = spawnSync(["systemd-detect-virt"]); if (!systemdError) { if (stdout.includes("amazon")) { return true; @@ -2047,7 +2095,7 @@ export async function isAws() { const dmiPath = "/sys/devices/virtual/dmi/id/board_asset_tag"; if (existsSync(dmiPath)) { - const dmiFile = readFileSync(dmiPath, { encoding: "utf-8" }); + const dmiFile = readFile(dmiPath); if (dmiFile.startsWith("i-")) { return true; } @@ -2060,7 +2108,7 @@ export async function isAws() { return true; } - const { error: powershellError, stdout } = await spawn([ + const { error: powershellError, stdout } = spawnSync([ "powershell", "-Command", "Get-CimInstance -ClassName Win32_ComputerSystem | Select-Object Manufacturer", @@ -2071,7 +2119,7 @@ export async function isAws() { } } - if (await checkAws()) { + if (checkAws()) { detectedCloud = "aws"; return true; } @@ -2080,12 +2128,12 @@ export async function isAws() { /** * @returns {Promise} */ -export async function isGoogleCloud() { +export function isGoogleCloud() { if (typeof detectedCloud === "string") { return detectedCloud === "google"; } - async function detectGoogleCloud() { + function detectGoogleCloud() { if (isLinux) { const vendorPaths = [ "/sys/class/dmi/id/sys_vendor", @@ -2095,7 +2143,7 @@ export async function isGoogleCloud() { for (const vendorPath of vendorPaths) { if (existsSync(vendorPath)) { - const vendorFile = readFileSync(vendorPath, { encoding: "utf-8" }); + const vendorFile = readFile(vendorPath); if (vendorFile.includes("Google")) { return true; } @@ -2104,25 +2152,25 @@ export async function isGoogleCloud() { } } - if (await detectGoogleCloud()) { + if (detectGoogleCloud()) { detectedCloud = "google"; return true; } } /** - * @returns {Promise} + * @returns {Cloud | undefined} */ -export async function getCloud() { +export function getCloud() { if (typeof detectedCloud === "string") { return detectedCloud; } - if (await isAws()) { + if (isAws()) { return "aws"; } - if (await isGoogleCloud()) { + if (isGoogleCloud()) { return "google"; } } @@ -2133,7 +2181,7 @@ export async function getCloud() { * @returns {Promise} */ export async function getCloudMetadata(name, cloud) { - cloud ??= await getCloud(); + cloud ??= getCloud(); if (!cloud) { return; } @@ -2802,10 +2850,21 @@ export function getBuildkiteEmoji(emoji) { return name ? `:${name}:` : ""; } +/** + * @typedef SshOptions + * @property {string} hostname + * @property {number} [port] + * @property {string} [username] + * @property {string} [password] + * @property {string[]} [command] + * @property {string[]} [identityPaths] + * @property {number} [retries] + */ + /** * @param {SshOptions} options - * @param {import("./utils.mjs").SpawnOptions} [spawnOptions] - * @returns {Promise} + * @param {SpawnOptions} [spawnOptions] + * @returns {Promise} */ export async function spawnSshSafe(options, spawnOptions = {}) { return spawnSsh(options, { throwOnError: true, ...spawnOptions }); @@ -2813,8 +2872,8 @@ export async function spawnSshSafe(options, spawnOptions = {}) { /** * @param {SshOptions} options - * @param {import("./utils.mjs").SpawnOptions} [spawnOptions] - * @returns {Promise} + * @param {SpawnOptions} [spawnOptions] + * @returns {Promise} */ export async function spawnSsh(options, spawnOptions = {}) { const { hostname, port, username, identityPaths, password, retries = 10, command: spawnCommand } = options; @@ -2845,10 +2904,10 @@ export async function spawnSsh(options, spawnOptions = {}) { } const stdio = spawnCommand ? "pipe" : "inherit"; if (spawnCommand) { - command.push(...spawnCommand); + command.push(...spawnCommand.map(arg => (arg.includes(" ") ? JSON.stringify(arg) : arg))); } - /** @type {import("./utils.mjs").SpawnResult} */ + /** @type {SpawnResult} */ let result; for (let i = 0; i < retries; i++) { result = await spawn(command, { stdio, ...spawnOptions, throwOnError: undefined }); @@ -2876,6 +2935,63 @@ export async function spawnSsh(options, spawnOptions = {}) { return result; } +/** + * @typedef ScpOptions + * @property {string} hostname + * @property {string} source + * @property {string} destination + * @property {string[]} [identityPaths] + * @property {string} [port] + * @property {string} [username] + * @property {number} [retries] + */ + +/** + * @param {ScpOptions} options + * @returns {Promise} + */ +export async function spawnScp(options) { + const { hostname, port, username, identityPaths, password, source, destination, retries = 10 } = options; + await waitForPort({ hostname, port: port || 22 }); + + const command = ["scp", "-o", "StrictHostKeyChecking=no"]; + if (!password) { + command.push("-o", "BatchMode=yes"); + } + if (port) { + command.push("-P", port); + } + if (password) { + const sshPass = which("sshpass", { required: true }); + command.unshift(sshPass, "-p", password); + } else if (identityPaths) { + command.push(...identityPaths.flatMap(path => ["-i", path])); + } + command.push(resolve(source)); + if (username) { + command.push(`${username}@${hostname}:${destination}`); + } else { + command.push(`${hostname}:${destination}`); + } + + let cause; + for (let i = 0; i < retries; i++) { + const result = await spawn(command, { stdio: "inherit" }); + const { exitCode, stderr } = result; + if (exitCode === 0) { + return; + } + + cause = stderr.trim() || undefined; + if (/(bad configuration option)|(no such file or directory)/i.test(stderr)) { + break; + } + await new Promise(resolve => setTimeout(resolve, Math.pow(2, i) * 1000)); + } + + throw new Error(`SCP failed: ${source} -> ${username}@${hostname}:${destination}`, { cause }); +} + /** * @param {MachineOptions} options * @returns {Promise} diff --git a/src/bun.js/bindings/BunProcess.cpp b/src/bun.js/bindings/BunProcess.cpp index 452c89b97ee0d2..3f8447232f27ba 100644 --- a/src/bun.js/bindings/BunProcess.cpp +++ b/src/bun.js/bindings/BunProcess.cpp @@ -135,9 +135,9 @@ extern "C" void Process__emitErrorEvent(Zig::GlobalObject* global, EncodedJSValu static JSValue constructArch(VM& vm, JSObject* processObject) { -#if CPU(X86_64) +#if defined(__x86_64__) return JSC::jsString(vm, makeAtomString("x64"_s)); -#elif CPU(ARM64) +#elif defined(__aarch64__) return JSC::jsString(vm, makeAtomString("arm64"_s)); #else #error "Unknown architecture" diff --git a/src/bun.js/bindings/CPUFeatures.cpp b/src/bun.js/bindings/CPUFeatures.cpp index d6665cd6b7de60..85023153359a00 100644 --- a/src/bun.js/bindings/CPUFeatures.cpp +++ b/src/bun.js/bindings/CPUFeatures.cpp @@ -17,7 +17,7 @@ enum class AArch64CPUFeature : uint8_t { sve = 6, }; -#if CPU(X86_64) +#if defined(__x86_64__) #if OS(WINDOWS) @@ -66,7 +66,7 @@ static uint8_t x86_cpu_features() #endif -#if CPU(ARM64) +#if defined(__aarch64__) static uint8_t aarch64_cpu_features() { @@ -100,9 +100,9 @@ static uint8_t aarch64_cpu_features() extern "C" uint8_t bun_cpu_features() { -#if CPU(X86_64) +#if defined(__x86_64__) return x86_cpu_features(); -#elif CPU(ARM64) +#elif defined(__aarch64__) return aarch64_cpu_features(); #else return 0; diff --git a/src/bun.js/bindings/c-bindings.cpp b/src/bun.js/bindings/c-bindings.cpp index 8268b1cab293e0..b35059cb592e72 100644 --- a/src/bun.js/bindings/c-bindings.cpp +++ b/src/bun.js/bindings/c-bindings.cpp @@ -22,7 +22,7 @@ #endif // !OS(WINDOWS) #include -#if CPU(X86_64) && !OS(WINDOWS) +#if defined(__x86_64__) && !OS(WINDOWS) extern "C" void bun_warn_avx_missing(const char* url) { __builtin_cpu_init();