Add ASAN builds

It compiles some of Ruby's key dependencies with ASAN enabled, ships
those dynamic libraries to a location in Ruby's prefix (which is part of
the tarball deployed with setup-ruby). -Wl,-rpath is used to make sure
that these ASANified libs are used in preference to any
distribution-provided libs at runtime.

Only Ubuntu 24.04 is supported, because the version of clang in 22.04 is
too old. MacOS is also not yet supported.

Author: KJTsanaktsidis

.github/workflows/build.yml

@@ -80,13 +80,19 @@ jobs:
       matrix:
         os: [ ubuntu-20.04, ubuntu-22.04, ubuntu-24.04, macos-11, macos-arm-oss ]
         name: [ head, debug ]
+        include:
+          - { os: ubuntu-24.04, name: asan }
     runs-on: ${{ matrix.os }}
     steps:
     - name: Clone ruby
       uses: actions/checkout@v4
       with:
         repository: ruby/ruby
         ref: ${{ needs.prepare.outputs.commit }}
+    - name: Clone ruby-dev-builder
+      uses: actions/checkout@v4
+      with:
+        path: ruby-dev-builder
 
     - name: Set platform
       id: platform
@@ -128,6 +134,36 @@ jobs:
         echo "cppflags=-DENABLE_PATH_CHECK=0 -DRUBY_DEBUG=1" >> $GITHUB_ENV
         echo "optflags=-O3 -fno-inline" >> $GITHUB_ENV
       if: matrix.name == 'debug'
+    - name: Build dependencies and set configure flags (asan)
+      run: |
+        set -ex
+        # We only test ASAN with Clang. The version of Clang needs to be > 18, which the version in
+        # Ubuntu 24.04's repo is
+        sudo apt-get install -y clang
+
+        # ASAN builds need to compile (some of) their own dependencies with ASAN enabled, so that it can
+        # catch memory errors caused by passing invalid parameters into other libraries.
+        ASAN_LIB_PREFIX="$HOME/.rubies/ruby-${{ matrix.name }}"
+        ./ruby-dev-builder/asan_libs.rb \
+          --prefix="$ASAN_LIB_PREFIX" \
+          --cc=clang \
+          --cflags="-fsanitize=address -fno-omit-frame-pointer -ggdb3 -O3" \
+          --ldflags="-Wl,-rpath=$ASAN_LIB_PREFIX/lib" \
+          --makeopts="-j4"
+
+        # Set Ruby configure flags
+        # Clang > 17 does not work with M:N threading, so we disable it: https://bugs.ruby-lang.org/issues/20243
+        echo "cppflags=-DENABLE_PATH_CHECK=0 -DRUBY_DEBUG=1 -DVM_CHECK_MODE=1 -DUSE_MN_THREADS=0" >> $GITHUB_ENV
+        echo "optflags=-O3 -fno-omit-frame-pointer" >> $GITHUB_ENV
+        echo "debugflags=-fsanitize=address -ggdb3" >> $GITHUB_ENV
+        echo "CC=clang" >> $GITHUB_ENV
+        # Make sure we link against the ASAN libs we built
+        echo "cflags=-I$ASAN_LIB_PREFIX/include" >> $GITHUB_ENV
+        echo "LDFLAGS=-L$ASAN_LIB_PREFIX/lib -Wl,-rpath=$ASAN_LIB_PREFIX/lib" >> $GITHUB_ENV
+        # Make the test timeouts more generous too (ASAN is slower)
+        echo "RUBY_TEST_TIMEOUT_SCALE=5" >> $GITHUB_ENV
+        echo "SYNTAX_SUGGEST_TIMEOUT=600" >> $GITHUB_ENV
+      if: matrix.name == 'asan'
 
     # Build
     - run: chmod 755 $HOME # https://github.com/actions/virtual-environments/issues/267

asan_libs.rb

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'fileutils'
+require 'tmpdir'
+require 'shellwords'
+
+$prefix = $cflags = $ldflags = $cc = $makeopts = nil
+OptionParser.new do |opts|
+  opts.banner = "Usage: asan_libs.rb --prefix=$RUBY_PREFIX"
+  opts.on('--prefix=PREFIX') { $prefix = _1 }
+  opts.on('--cflags=CFLAGS') { $cflags = _1 }
+  opts.on('--ldflags=LDFLAGS') { $ldflags= _1 }
+  opts.on('--cc=CC') { $cc = _1 }
+  opts.on('--makeopts=MAKEOPTS') { $makeopts = _1 }
+end.parse!
+raise "--prefix must be specified" if $prefix.nil?
+
+
+OPENSSL_VERSION = "3.3.0"
+LIBYAML_VERSION = "0.2.5"
+
+$cflags_args = []
+$cflags_args << "CC=#{$cc}" if $cc
+$cflags_args << "CFLAGS=#{$cflags}" if $cflags
+$cflags_args << "LDFLAGS=#{$ldflags}" if $ldflags
+$makeopts = Shellwords.split($makeopts || '')
+
+def sh!(*args, **kwargs)
+  puts "==> #{Shellwords.join(args)}"
+  system(*args, **kwargs, exception: true) 
+end
+
+def chdir!(dir, &block)
+  puts "==> cd #{File.realpath dir}" 
+  Dir.chdir dir, &block
+end
+
+def rmglob!(what)
+  Dir.glob(what).each do |file|
+    puts "==> rm #{file}"
+    FileUtils.rm_f file
+  end
+end
+
+def compile_openssl
+  # Make sure we compile OpenSSL to look for certificates in the same place that the
+  # distribution provided OpenSSl would.
+  openssldir = Shellwords.split(`openssl version -d`.match(/^\s*OPENSSLDIR:(.*)$/)[1].strip).first
+
+  sh! *%w[curl -fsSLO], "https://www.openssl.org/source/openssl-#{OPENSSL_VERSION}.tar.gz"
+  sh! *%w[tar -xf], "openssl-#{OPENSSL_VERSION}.tar.gz"
+  chdir!("openssl-#{OPENSSL_VERSION}") do
+    sh! './Configure', "--prefix=#{$prefix}", '--libdir=lib', "--openssldir=#{openssldir}",
+      *%w[shared no-tests no-apps], *$cflags_args
+    sh! 'make', *$makeopts
+    # make install_sw will try and write config to OPENSSLDIR, which we don't want to do.
+    sh! *%w[make install_dev], exception: true
+    # OpenSSL make install_dev will also install static libraries, which we don't need
+    rmglob! File.join($prefix, "lib/*.a")
+  end
+end
+
+def compile_libyaml
+  sh! *%w[curl -fsSLO], "http://pyyaml.org/download/libyaml/yaml-#{LIBYAML_VERSION}.tar.gz"
+  sh! *%w[tar -xf], "yaml-#{LIBYAML_VERSION}.tar.gz"
+  chdir!("yaml-#{LIBYAML_VERSION}") do
+    sh! './configure', "--prefix=#{$prefix}", *%w[--disable-static --enable-shared], *$cflags_args
+    sh! 'make', *$makeopts
+    sh! 'make', 'install'
+  end
+end
+
+Dir.mktmpdir('asan_libs') do |build_dir|
+  Dir.chdir(build_dir) do
+    compile_openssl
+    compile_libyaml
+  end
+end