From f39648577e6ddad845774eb95669baecca6ef541 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Tue, 11 Jul 2023 13:17:30 -0700 Subject: [PATCH 1/8] Vendor re2 and abseil If `--disable-system-libraries` is specified, this change will download and build abseil and re2. `cmake` and a C++17 compiler is required for this to work. This makes it possible to ensure all the required dependencies are contained within the C extension rather than depend on system libraries, which may break the extension when they are updated. The building of these libraries uses mini_portile2 and techniques borrowed from the nokogiri and ruby-magic gems. Closes https://github.com/mudge/re2/issues/61 --- Rakefile | 11 ++ dependencies.yml | 9 ++ ext/re2/extconf.rb | 281 +++++++++++++++++++++++++++++++++++---------- re2.gemspec | 1 + 4 files changed, 244 insertions(+), 58 deletions(-) create mode 100644 dependencies.yml diff --git a/Rakefile b/Rakefile index 957a0c2..c6fa8e9 100644 --- a/Rakefile +++ b/Rakefile @@ -1,6 +1,17 @@ require 'rake/extensiontask' require 'rspec/core/rake_task' +CLEAN.include FileList['**/*{.o,.so,.dylib,.bundle}'], + FileList['**/extconf.h'], + FileList['**/Makefile'], + FileList['pkg/'] + +CLOBBER.include FileList['**/tmp'], + FileList['**/*.log'], + FileList['doc/**'], + FileList['tmp/'] +CLOBBER.add("ports/*").exclude(%r{ports/archives$}) + Rake::ExtensionTask.new('re2') RSpec::Core::RakeTask.new(:spec) diff --git a/dependencies.yml b/dependencies.yml new file mode 100644 index 0000000..1043178 --- /dev/null +++ b/dependencies.yml @@ -0,0 +1,9 @@ +libre2: + version: "2023-07-01" + sha256: "18cf85922e27fad3ed9c96a27733037da445f35eb1a2744c306a37c6d11e95c4" + # sha-256 hash provided in https://github.com/google/re2/releases/download/2023-07-01/re2-2023-07-01.tar.gz + +abseil: + version: "20230125.3" + sha256: 5366d7e7fa7ba0d915014d387b66d0d002c03236448e1ba9ef98122c13b35c36 + # sha-256 hash provided in https://github.com/abseil/abseil-cpp/archive/refs/tags/20230125.3.tar.gz diff --git a/ext/re2/extconf.rb b/ext/re2/extconf.rb index 9887232..6968fdf 100644 --- a/ext/re2/extconf.rb +++ b/ext/re2/extconf.rb @@ -6,6 +6,70 @@ require 'mkmf' +PACKAGE_ROOT_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..', '..')) + +REQUIRED_MINI_PORTILE_VERSION = "~> 2.8.2" # keep this version in sync with the one in the gemspec + +RE2_HELP_MESSAGE = <<~HELP + USAGE: ruby #{$0} [options] + + Flags that are always valid: + + --use-system-libraries + --enable-system-libraries + Use system libraries instead of building and using the packaged libraries. This is the default. + + --disable-system-libraries + Use the packaged libraries, and ignore the system libraries. This overrides `--use-system-libraries`. + + Flags only used when using system libraries: + + Related to re2 library: + + --with-re2-dir=DIRECTORY + Look for re2 headers and library in DIRECTORY. + + Environment variables used: + + CC + Use this path to invoke the compiler instead of `RbConfig::CONFIG['CC']` + + CPPFLAGS + If this string is accepted by the C preprocessor, add it to the flags passed to the C preprocessor + + CFLAGS + If this string is accepted by the compiler, add it to the flags passed to the compiler + + LDFLAGS + If this string is accepted by the linker, add it to the flags passed to the linker + + LIBS + Add this string to the flags passed to the linker +HELP + +# +# utility functions +# +def config_system_libraries? + enable_config("system-libraries", true) do |_, default| + arg_config("--use-system-libraries", default) + end +end + +def concat_flags(*args) + args.compact.join(" ") +end + +def do_help + print(RE2_HELP_MESSAGE) + exit!(0) +end + +# +# main +# +do_help if arg_config('--help') + if ENV["CC"] RbConfig::MAKEFILE_CONFIG["CC"] = ENV["CC"] RbConfig::CONFIG["CC"] = ENV["CC"] @@ -16,70 +80,57 @@ RbConfig::CONFIG["CXX"] = ENV["CXX"] end -header_dirs = [ - "/usr/local/include", - "/opt/homebrew/include", - "/usr/include" -] - -lib_dirs = [ - "/usr/local/lib", - "/opt/homebrew/lib", - "/usr/lib" -] +def build_extension + $CFLAGS << " -Wall -Wextra -funroll-loops" -dir_config("re2", header_dirs, lib_dirs) + # Pass -x c++ to force gcc to compile the test program + # as C++ (as it will end in .c by default). + compile_options = "-x c++" -$CFLAGS << " -Wall -Wextra -funroll-loops" + have_library("stdc++") + have_header("stdint.h") + have_func("rb_str_sublen") -# Pass -x c++ to force gcc to compile the test program -# as C++ (as it will end in .c by default). -compile_options = "-x c++" - -have_library("stdc++") -have_header("stdint.h") -have_func("rb_str_sublen") - -unless have_library("re2") - abort "You must have re2 installed and specified with --with-re2-dir, please see https://github.com/google/re2/wiki/Install" -end + unless have_library("re2") + abort "You must have re2 installed and specified with --with-re2-dir, please see https://github.com/google/re2/wiki/Install" + end -minimal_program = < int main() { return 0; } SRC -re2_requires_version_flag = checking_for("re2 that requires explicit C++ version flag") do - !try_compile(minimal_program, compile_options) -end + re2_requires_version_flag = checking_for("re2 that requires explicit C++ version flag") do + !try_compile(minimal_program, compile_options) + end + + if re2_requires_version_flag + # Recent versions of re2 depend directly on abseil, which requires a + # compiler with C++14 support (see + # https://github.com/abseil/abseil-cpp/issues/1127 and + # https://github.com/abseil/abseil-cpp/issues/1431). However, the + # `std=c++14` flag doesn't appear to suffice; we need at least + # `std=c++17`. + abort "Cannot compile re2 with your compiler: recent versions require C++14 support." unless %w[c++20 c++17 c++11 c++0x].any? do |std| + checking_for("re2 that compiles with #{std} standard") do + if try_compile(minimal_program, compile_options + " -std=#{std}") + compile_options << " -std=#{std}" + $CPPFLAGS << " -std=#{std}" -if re2_requires_version_flag - # Recent versions of re2 depend directly on abseil, which requires a - # compiler with C++14 support (see - # https://github.com/abseil/abseil-cpp/issues/1127 and - # https://github.com/abseil/abseil-cpp/issues/1431). However, the - # `std=c++14` flag doesn't appear to suffice; we need at least - # `std=c++17`. - abort "Cannot compile re2 with your compiler: recent versions require C++14 support." unless %w[c++20 c++17 c++11 c++0x].any? do |std| - checking_for("re2 that compiles with #{std} standard") do - if try_compile(minimal_program, compile_options + " -std=#{std}") - compile_options << " -std=#{std}" - $CPPFLAGS << " -std=#{std}" - - true + true + end end end end -end -# Determine which version of re2 the user has installed. -# Revision d9f8806c004d added an `endpos` argument to the -# generic Match() function. -# -# To test for this, try to compile a simple program that uses -# the newer form of Match() and set a flag if it is successful. -checking_for("RE2::Match() with endpos argument") do - test_re2_match_signature = < int main() { @@ -91,13 +142,13 @@ } SRC - if try_compile(test_re2_match_signature, compile_options) - $defs.push("-DHAVE_ENDPOS_ARGUMENT") + if try_compile(test_re2_match_signature, compile_options) + $defs.push("-DHAVE_ENDPOS_ARGUMENT") + end end -end -checking_for("RE2::Set::Match() with error information") do - test_re2_set_match_signature = < #include #include @@ -115,9 +166,123 @@ } SRC - if try_compile(test_re2_set_match_signature, compile_options) - $defs.push("-DHAVE_ERROR_INFO_ARGUMENT") + if try_compile(test_re2_set_match_signature, compile_options) + $defs.push("-DHAVE_ERROR_INFO_ARGUMENT") + end + end +end + +def process_recipe(name, version) + require "rubygems" + gem("mini_portile2", REQUIRED_MINI_PORTILE_VERSION) # gemspec is not respected at install time + require "mini_portile2" + message("Using mini_portile version #{MiniPortile::VERSION}\n") + + MiniPortileCMake.new(name, version).tap do |recipe| + recipe.target = File.join(PACKAGE_ROOT_DIR, "ports") + recipe.configure_options += [ + # abseil needs a C++14 compiler + '-DCMAKE_CXX_STANDARD=17', + # needed for building the C extension shared library with -fPIC + '-DCMAKE_POSITION_INDEPENDENT_CODE=ON', + # ensures pkg-config and installed libraries will be in lib, not lib64 + '-DCMAKE_INSTALL_LIBDIR=lib' + ] + + yield recipe + + checkpoint = "#{recipe.target}/#{recipe.name}-#{recipe.version}-#{recipe.host}.installed" + + if File.exist?(checkpoint) + message("Building re2 with a packaged version of #{name}-#{version}.\n") + else + message(<<~EOM) + ---------- IMPORTANT NOTICE ---------- + Building re2 with a packaged version of #{name}-#{version}. + Configuration options: #{recipe.configure_options.shelljoin} + EOM + + unless recipe.patch_files.empty? + message("The following patches are being applied:\n") + + recipe.patch_files.each do |patch| + message(" - %s\n" % File.basename(patch)) + end + end + + recipe.cook + + FileUtils.touch(checkpoint) + end + + recipe.activate + end +end + +def build_with_system_libraries + header_dirs = [ + "/usr/local/include", + "/opt/homebrew/include", + "/usr/include" + ] + + lib_dirs = [ + "/usr/local/lib", + "/opt/homebrew/lib", + "/usr/lib" + ] + + dir_config("re2", header_dirs, lib_dirs) + + build_extension +end + +def build_with_vendored_libraries + message "Building re2 using packaged libraries.\n" + + require 'yaml' + dependencies = YAML.load_file(File.join(PACKAGE_ROOT_DIR, 'dependencies.yml')) + + abseil_recipe = process_recipe('abseil', dependencies['abseil']['version']) do |recipe| + recipe.files = [{ + url: "https://github.com/abseil/abseil-cpp/archive/refs/tags/#{recipe.version}.tar.gz", + sha256: dependencies['abseil']['sha256'] + }] + recipe.configure_options += ['-DABSL_PROPAGATE_CXX_STD=ON'] end + + re2_recipe = process_recipe('libre2', dependencies['libre2']['version']) do |recipe| + recipe.files = [{ + url: "https://github.com/google/re2/releases/download/#{recipe.version}/re2-#{recipe.version}.tar.gz", + sha256: dependencies['libre2']['sha256'] + }] + recipe.configure_options += ["-DCMAKE_PREFIX_PATH=#{abseil_recipe.path}", '-DCMAKE_CXX_FLAGS=-DNDEBUG'] + end + + pkg_config_paths = [ + "#{abseil_recipe.path}/lib/pkgconfig", + "#{re2_recipe.path}/lib/pkgconfig" + ].join(':') + + pkg_config_paths = "#{ENV['PKG_CONFIG_PATH']}:#{pkg_config_paths}" if ENV['PKG_CONFIG_PATH'] + + ENV['PKG_CONFIG_PATH'] = pkg_config_paths + pc_file = File.join(re2_recipe.path, 'lib', 'pkgconfig', 're2.pc') + if pkg_config(pc_file) + # See https://bugs.ruby-lang.org/issues/18490, broken in Ruby 3.1 but fixed in Ruby 3.2. + flags = xpopen(['pkg-config', '--libs', '--static', pc_file], err: %i[child out], &:read) + flags.split.each { |flag| append_ldflags(flag) } if $?.success? + else + raise 'Please install the `pkg-config` utility!' + end + + build_extension +end + +if config_system_libraries? + build_with_system_libraries +else + build_with_vendored_libraries end create_makefile("re2") diff --git a/re2.gemspec b/re2.gemspec index e64b5a6..355dd63 100644 --- a/re2.gemspec +++ b/re2.gemspec @@ -29,4 +29,5 @@ Gem::Specification.new do |s| ] s.add_development_dependency("rake-compiler", "~> 0.9") s.add_development_dependency("rspec", "~> 3.2") + s.add_runtime_dependency("mini_portile2", "~> 2.8.2") # keep version in sync with extconf.rb end From cb320ca0ad4e2a9601b157b76efba372d6f694b4 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Tue, 11 Jul 2023 21:26:53 -0700 Subject: [PATCH 2/8] ci: drop support for Ruby 1.9 - 2.2 mini_portile2 requires >= Ruby 2.3, so we need to drop support for obsolete Ruby versions. --- .github/workflows/tests.yml | 47 ------------------------------------- 1 file changed, 47 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6a644b4..326dff8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,8 +19,6 @@ jobs: - 2.5 - 2.4 - 2.3 - - 2.2 - - 2.1.9 libre2: - version: "20150501" soname: 0 @@ -56,48 +54,3 @@ jobs: ruby-version: "${{ matrix.ruby }}" bundler-cache: true - run: bundle exec rake - - legacy: - name: Legacy Ruby ${{ matrix.ruby }} - libre2 ABI version ${{ matrix.libre2.soname }} - runs-on: ubuntu-20.04 - container: - image: "mudge/re2-ci:${{ matrix.ruby }}" - options: "--add-host rubygems.org:151.101.129.227 --add-host api.rubygems.org:151.101.129.227" - strategy: - matrix: - ruby: - - 1.8 - - 1.9.1 - - '2.0' - libre2: - - version: "20150501" - soname: 0 - - version: "20200302" - soname: 1 - - version: "20200303" - soname: 6 - - version: "20200501" - soname: 7 - - version: "20200706" - soname: 8 - - version: "20201101" - soname: 9 - - version: "20221201" - soname: 10 - steps: - - uses: actions/checkout@v3 - - name: Download and install specific release of libre2 - run: | - curl -Lo libre2-dev.deb https://github.com/mudge/re2-ci/releases/download/v1/libre2-dev_${{ matrix.libre2.version }}_amd64.deb - dpkg -i libre2-dev.deb - - name: Configure Bundler for Ruby dependencies - run: bundle config --local path vendor/bundle - - name: Generate Gemfile.lock - run: bundle lock - - name: Cache Ruby dependencies - uses: actions/cache@v3 - with: - path: vendor/bundle - key: gems-v1-${{ runner.os }}-${{ matrix.ruby }}-${{ hashFiles('Gemfile.lock') }} - - run: bundle install --jobs 4 - - run: bundle exec rake From 13c1b50710a30f94e407edc4cac9f827219767ca Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Tue, 11 Jul 2023 21:28:36 -0700 Subject: [PATCH 3/8] ci: add jobs to test with --disable-system-libraries --- .github/workflows/tests.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 326dff8..ab65d07 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,3 +54,39 @@ jobs: ruby-version: "${{ matrix.ruby }}" bundler-cache: true - run: bundle exec rake + + vendor: + name: Ruby ${{ matrix.ruby }} - vendored libre2 and abseil + runs-on: ubuntu-20.04 + strategy: + matrix: + ruby: + - '3.2' + - '3.1' + - '3.0' + - 2.7 + - 2.6 + - 2.5 + - 2.4 + - 2.3 + steps: + - uses: actions/checkout@v3 + - name: Install build dependencies + run: | + sudo apt-get install -y build-essential cmake + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "${{ matrix.ruby }}" + bundler-cache: true + - name: Configure Bundler for Ruby dependencies + run: bundle config --local path vendor/bundle + - name: Generate Gemfile.lock + run: bundle lock + - name: Cache Ruby dependencies + uses: actions/cache@v3 + with: + path: vendor/bundle + key: gems-v1-${{ runner.os }}-${{ matrix.ruby }}-${{ hashFiles('Gemfile.lock') }} + - run: bundle install --jobs 4 + - run: bundle exec rake compile -- --disable-system-libraries + - run: bundle exec rake From cad059357033fb1c56063f087baec8349d13e582 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Tue, 11 Jul 2023 21:40:41 -0700 Subject: [PATCH 4/8] Make `rake gem` vendor libre2 and abseil tarballs This increases the size of the gem from 32K to 2.4 MB, but this will significantly reduce network transfer and avoid rate limits. --- Rakefile | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index c6fa8e9..d32bbf3 100644 --- a/Rakefile +++ b/Rakefile @@ -12,10 +12,44 @@ CLOBBER.include FileList['**/tmp'], FileList['tmp/'] CLOBBER.add("ports/*").exclude(%r{ports/archives$}) +RE2_GEM_SPEC = Gem::Specification.load('re2.gemspec') + Rake::ExtensionTask.new('re2') +Gem::PackageTask.new(RE2_GEM_SPEC) do |p| + p.need_zip = false + p.need_tar = false +end + RSpec::Core::RakeTask.new(:spec) +def gem_build_path + File.join 'pkg', RE2_GEM_SPEC.full_name +end + +def add_file_to_gem(relative_source_path) + dest_path = File.join(gem_build_path, relative_source_path) + dest_dir = File.dirname(dest_path) + + mkdir_p dest_dir unless Dir.exist?(dest_dir) + rm_f dest_path if File.exist?(dest_path) + safe_ln relative_source_path, dest_path + + RE2_GEM_SPEC.files << relative_source_path +end + +def add_vendored_libraries + dependencies = YAML.load_file(File.join(File.dirname(__FILE__), 'dependencies.yml')) + abseil_archive = File.join('ports', 'archives', "#{dependencies['abseil']['version']}.tar.gz") + libre2_archive = File.join('ports', 'archives', "re2-#{dependencies['libre2']['version']}.tar.gz") + + add_file_to_gem(abseil_archive) + add_file_to_gem(libre2_archive) +end + +task gem_build_path do + add_vendored_libraries +end + task :spec => :compile task :default => :spec - From 26e68b7df4206693e11ebc2a36c40e850b8c2544 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 13 Jul 2023 09:37:43 -0700 Subject: [PATCH 5/8] Add support for building precompiled gems Running `rake gem:native` will attempt to cross-compile gems for the following platforms: - aarch64-linux - arm64-darwin - x86_64-darwin - x86_64-linux `rake gem:` also will compile one specific platform. For example: `rake gem:arm64-darwin`. --- Rakefile | 63 +++++++++++++++++++++++++++++++++++++++++++++++++---- lib/re2.rb | 8 ++++++- re2.gemspec | 3 ++- 3 files changed, 68 insertions(+), 6 deletions(-) diff --git a/Rakefile b/Rakefile index d32bbf3..89280b7 100644 --- a/Rakefile +++ b/Rakefile @@ -1,5 +1,8 @@ +# frozen_string_literal: true + require 'rake/extensiontask' require 'rspec/core/rake_task' +require 'rake_compiler_dock' CLEAN.include FileList['**/*{.o,.so,.dylib,.bundle}'], FileList['**/extconf.h'], @@ -14,17 +17,65 @@ CLOBBER.add("ports/*").exclude(%r{ports/archives$}) RE2_GEM_SPEC = Gem::Specification.load('re2.gemspec') -Rake::ExtensionTask.new('re2') - Gem::PackageTask.new(RE2_GEM_SPEC) do |p| p.need_zip = false p.need_tar = false end +CROSS_RUBY_VERSIONS = %w[3.2.0 3.1.0 3.0.0 2.7.0].join(':') +CROSS_RUBY_PLATFORMS = %w[ + aarch64-linux + arm64-darwin + x86_64-darwin + x86_64-linux +].freeze + +ENV['RUBY_CC_VERSION'] = CROSS_RUBY_VERSIONS + +Rake::ExtensionTask.new('re2', RE2_GEM_SPEC) do |e| + e.cross_compile = true + e.cross_config_options << '--enable-cross-build' + e.config_options << '--disable-system-libraries' + e.cross_platform = CROSS_RUBY_PLATFORMS + e.cross_compiling do |spec| + spec.files.reject! { |path| File.fnmatch?('ports/*', path) } + spec.dependencies.reject! { |dep| dep.name == 'mini_portile2' } + end +end + RSpec::Core::RakeTask.new(:spec) -def gem_build_path - File.join 'pkg', RE2_GEM_SPEC.full_name +namespace 'gem' do + def gem_builder(platform) + # use Task#invoke because the pkg/*gem task is defined at runtime + Rake::Task["native:#{platform}"].invoke + Rake::Task["pkg/#{RE2_GEM_SPEC.full_name}-#{Gem::Platform.new(platform)}.gem"].invoke + end + + CROSS_RUBY_PLATFORMS.each do |platform| + # The Linux x86 image (ghcr.io/rake-compiler/rake-compiler-dock-image:1.3.0-mri-x86_64-linux) + # is based on CentOS 7 and has two versions of cmake installed: + # a 2.8 version in /usr/bin and a 3.25 in /usr/local/bin. The latter is needed by abseil. + cmake = platform == 'x86_64-linux' ? '/usr/local/bin/cmake' : 'cmake' + desc "build native gem for #{platform} platform" + task platform do + RakeCompilerDock.sh <<~SCRIPT, platform: platform, verbose: true + gem install bundler --no-document && + bundle && + CMAKE=#{cmake} bundle exec rake gem:#{platform}:builder MAKE='nice make -j`nproc`' + SCRIPT + end + + namespace platform do + desc "build native gem for #{platform} platform (guest container)" + task 'builder' do + gem_builder(platform) + end + end + end + + desc 'build all native gems' + task 'native' => CROSS_RUBY_PLATFORMS end def add_file_to_gem(relative_source_path) @@ -38,6 +89,10 @@ def add_file_to_gem(relative_source_path) RE2_GEM_SPEC.files << relative_source_path end +def gem_build_path + File.join 'pkg', RE2_GEM_SPEC.full_name +end + def add_vendored_libraries dependencies = YAML.load_file(File.join(File.dirname(__FILE__), 'dependencies.yml')) abseil_archive = File.join('ports', 'archives', "#{dependencies['abseil']['version']}.tar.gz") diff --git a/lib/re2.rb b/lib/re2.rb index 5870b47..9c903ff 100644 --- a/lib/re2.rb +++ b/lib/re2.rb @@ -3,5 +3,11 @@ # # Copyright (c) 2010-2014, Paul Mucur (http://mudge.name) # Released under the BSD Licence, please see LICENSE.txt -require "re2.so" +begin + ::RUBY_VERSION =~ /(\d+\.\d+)/ + require_relative "#{Regexp.last_match(1)}/re2.so" +rescue LoadError + require 're2.so' +end + require "re2/scanner" diff --git a/re2.gemspec b/re2.gemspec index 355dd63..873ce5e 100644 --- a/re2.gemspec +++ b/re2.gemspec @@ -27,7 +27,8 @@ Gem::Specification.new do |s| "spec/re2/set_spec.rb", "spec/re2/scanner_spec.rb" ] - s.add_development_dependency("rake-compiler", "~> 0.9") + s.add_development_dependency "rake-compiler", "~> 1.2.1" + s.add_development_dependency "rake-compiler-dock", "~> 1.3.0" s.add_development_dependency("rspec", "~> 3.2") s.add_runtime_dependency("mini_portile2", "~> 2.8.2") # keep version in sync with extconf.rb end From f0e2f35e141d2433bc952de5518c39d6f2a49853 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 13 Jul 2023 22:41:01 -0700 Subject: [PATCH 6/8] Make cross-compilation for native gems work mini_portile2 provides `configure` script with the `--host` parameter, and the script figures out the target compilers to use. With CMake, we don't have that benefit so we need to set the C and C++ compilers ourselves. We use a simple heuristic to find the target compilers: 1. For macOS, we use `clang` and `clang++. Otherwise we use `gcc` and `gcc++`. 2. We search the PATH for `-compiler`. Use that if we find it. Otherwise default to the base compilers. To make CMake work with cross-compilation, a number of variables have to be set: - CMAKE_SYSTEM_PROCESSOR - CMAKE_SYSTEM_NAME - CMAKE_C_COMPILER - CMAKE_CXX_COMPILER --- Rakefile | 2 +- ext/re2/extconf.rb | 65 +++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/Rakefile b/Rakefile index 89280b7..daee260 100644 --- a/Rakefile +++ b/Rakefile @@ -62,7 +62,7 @@ namespace 'gem' do RakeCompilerDock.sh <<~SCRIPT, platform: platform, verbose: true gem install bundler --no-document && bundle && - CMAKE=#{cmake} bundle exec rake gem:#{platform}:builder MAKE='nice make -j`nproc`' + bundle exec rake gem:#{platform}:builder CMAKE=#{cmake} SCRIPT end diff --git a/ext/re2/extconf.rb b/ext/re2/extconf.rb index 6968fdf..d05723f 100644 --- a/ext/re2/extconf.rb +++ b/ext/re2/extconf.rb @@ -65,6 +65,55 @@ def do_help exit!(0) end +def darwin? + RbConfig::CONFIG["target_os"].include?("darwin") +end + +def target_host + # We use 'host' to set compiler prefix for cross-compiling. Prefer host_alias over host. And + # prefer i686 (what external dev tools use) to i386 (what ruby's configure.ac emits). + host = RbConfig::CONFIG["host_alias"].empty? ? RbConfig::CONFIG["host"] : RbConfig::CONFIG["host_alias"] + host.gsub(/i386/, "i686") +end + +def find_compiler(compilers) + compilers.find { |binary| find_executable(binary) } +end + +# configure automatically searches for the right compiler based on the +# `--host` parameter. However, we don't have that feature with +# cmake. Search for the right compiler for the target architecture using +# some basic heruistics. +def find_c_and_cxx_compilers(host) + if darwin? + c_compiler = 'clang' + cxx_compiler = 'clang++' + else + c_compiler = 'gcc' + cxx_compiler = 'g++' + end + + c_platform_compiler = "#{host}-#{c_compiler}" + cxx_platform_compiler = "#{host}-#{cxx_compiler}" + c_compiler = find_compiler([c_platform_compiler, c_compiler]) + cxx_compiler = find_compiler([cxx_platform_compiler, cxx_compiler]) + + [c_compiler, cxx_compiler] +end + +def cmake_compile_flags(host) + system_name = darwin? ? 'Darwin' : 'Linux' + c_compiler, cxx_compiler = find_c_and_cxx_compilers(host) + + # needed to ensure cross-compilation with CMake targets the right CPU and compilers + [ + "-DCMAKE_SYSTEM_PROCESSOR=#{RbConfig::CONFIG['target_cpu']}", + "-DCMAKE_SYSTEM_NAME=#{system_name}", + "-DCMAKE_C_COMPILER=#{c_compiler}", + "-DCMAKE_CXX_COMPILER=#{cxx_compiler}" + ] +end + # # main # @@ -179,7 +228,9 @@ def process_recipe(name, version) message("Using mini_portile version #{MiniPortile::VERSION}\n") MiniPortileCMake.new(name, version).tap do |recipe| + recipe.host = target_host recipe.target = File.join(PACKAGE_ROOT_DIR, "ports") + recipe.configure_options += [ # abseil needs a C++14 compiler '-DCMAKE_CXX_STANDARD=17', @@ -188,6 +239,7 @@ def process_recipe(name, version) # ensures pkg-config and installed libraries will be in lib, not lib64 '-DCMAKE_INSTALL_LIBDIR=lib' ] + recipe.configure_options += cmake_compile_flags(recipe.host) yield recipe @@ -268,13 +320,12 @@ def build_with_vendored_libraries ENV['PKG_CONFIG_PATH'] = pkg_config_paths pc_file = File.join(re2_recipe.path, 'lib', 'pkgconfig', 're2.pc') - if pkg_config(pc_file) - # See https://bugs.ruby-lang.org/issues/18490, broken in Ruby 3.1 but fixed in Ruby 3.2. - flags = xpopen(['pkg-config', '--libs', '--static', pc_file], err: %i[child out], &:read) - flags.split.each { |flag| append_ldflags(flag) } if $?.success? - else - raise 'Please install the `pkg-config` utility!' - end + + raise 'Please install the `pkg-config` utility!' unless pkg_config('re2') + + # See https://bugs.ruby-lang.org/issues/18490, broken in Ruby 3.1 but fixed in Ruby 3.2. + flags = xpopen(['pkg-config', '--libs', '--static', pc_file], err: %i[child out], &:read) + flags.split.each { |flag| append_ldflags(flag) } if $?.success? build_extension end From 8f85df081a842f6279f4c97000afc38a9fdc3bb7 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 14 Jul 2023 00:41:22 -0700 Subject: [PATCH 7/8] Add Windows and other cross-compilation targets Windows builds were previously failing because mini_portile2 adds an unnecessary option to the `cmake` configure step. This commit adds other targets: - arm-linux - arm64-darwin - x64-mingw-ucrt - x86-linux - x86-mingw32 --- Rakefile | 13 ++++++++++++- ext/re2/extconf.rb | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index daee260..e942972 100644 --- a/Rakefile +++ b/Rakefile @@ -25,7 +25,11 @@ end CROSS_RUBY_VERSIONS = %w[3.2.0 3.1.0 3.0.0 2.7.0].join(':') CROSS_RUBY_PLATFORMS = %w[ aarch64-linux + arm-linux arm64-darwin + x64-mingw-ucrt + x86-linux + x86-mingw32 x86_64-darwin x86_64-linux ].freeze @@ -56,7 +60,14 @@ namespace 'gem' do # The Linux x86 image (ghcr.io/rake-compiler/rake-compiler-dock-image:1.3.0-mri-x86_64-linux) # is based on CentOS 7 and has two versions of cmake installed: # a 2.8 version in /usr/bin and a 3.25 in /usr/local/bin. The latter is needed by abseil. - cmake = platform == 'x86_64-linux' ? '/usr/local/bin/cmake' : 'cmake' + cmake = + case platform + when 'x86_64-linux', 'x86-linux' + '/usr/local/bin/cmake' + else + 'cmake' + end + desc "build native gem for #{platform} platform" task platform do RakeCompilerDock.sh <<~SCRIPT, platform: platform, verbose: true diff --git a/ext/re2/extconf.rb b/ext/re2/extconf.rb index d05723f..8673902 100644 --- a/ext/re2/extconf.rb +++ b/ext/re2/extconf.rb @@ -114,6 +114,21 @@ def cmake_compile_flags(host) ] end +# By default, mini_portile2 might add an unnecessary option: +# https://github.com/flavorjones/mini_portile/blob/5084a2aeab12076f534cf0cabc81a4d5f84b5c25/lib/mini_portile2/mini_portile_cmake.rb#L17 +def delete_cmake_generator_option!(options) + indices = [] + + options.each_with_index do |element, index| + if element == '-G' && index + 1 < options.length + indices << index + indices << index + 1 + end + end + + indices.reverse_each { |index| options.delete_at(index) } +end + # # main # @@ -240,6 +255,7 @@ def process_recipe(name, version) '-DCMAKE_INSTALL_LIBDIR=lib' ] recipe.configure_options += cmake_compile_flags(recipe.host) + delete_cmake_generator_option!(recipe.configure_options) yield recipe From ed95b46f77debaa0ff4b222cd547f3d890774d15 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 14 Jul 2023 09:55:51 -0700 Subject: [PATCH 8/8] Make --disable-system-libraries the default parameter This commit also adds an environment variable, `RE2_USE_SYSTEM_LIBRARIES`, to enable system libraries. Add some documentation about mini_portile2 workarounds and improve CMake cross-compilation usage. --- .github/workflows/tests.yml | 1 + ext/re2/extconf.rb | 47 +++++++++++++++++++++++++++---------- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ab65d07..d662617 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -53,6 +53,7 @@ jobs: with: ruby-version: "${{ matrix.ruby }}" bundler-cache: true + - run: bundle exec rake compile -- --enable-system-libraries - run: bundle exec rake vendor: diff --git a/ext/re2/extconf.rb b/ext/re2/extconf.rb index 8673902..f1f4bea 100644 --- a/ext/re2/extconf.rb +++ b/ext/re2/extconf.rb @@ -15,12 +15,12 @@ Flags that are always valid: - --use-system-libraries --enable-system-libraries - Use system libraries instead of building and using the packaged libraries. This is the default. + Use system libraries instead of building and using the packaged libraries. --disable-system-libraries - Use the packaged libraries, and ignore the system libraries. This overrides `--use-system-libraries`. + Use the packaged libraries, and ignore the system libraries. This is the default. + Flags only used when using system libraries: @@ -51,9 +51,7 @@ # utility functions # def config_system_libraries? - enable_config("system-libraries", true) do |_, default| - arg_config("--use-system-libraries", default) - end + enable_config("system-libraries", ENV.key?('RE2_USE_SYSTEM_LIBRARIES')) end def concat_flags(*args) @@ -69,6 +67,15 @@ def darwin? RbConfig::CONFIG["target_os"].include?("darwin") end +def windows? + # Use =~ instead of match? to preserve Ruby 2.3 compatibility + RbConfig::CONFIG["target_os"] =~ /mingw|mswin/ +end + +def freebsd? + RbConfig::CONFIG["target_os"].include?("freebsd") +end + def target_host # We use 'host' to set compiler prefix for cross-compiling. Prefer host_alias over host. And # prefer i686 (what external dev tools use) to i386 (what ruby's configure.ac emits). @@ -84,13 +91,17 @@ def find_compiler(compilers) # `--host` parameter. However, we don't have that feature with # cmake. Search for the right compiler for the target architecture using # some basic heruistics. +# See https://github.com/flavorjones/mini_portile/issues/128. def find_c_and_cxx_compilers(host) + c_compiler = ENV["CC"] + cxx_compiler = ENV["CXX"] + if darwin? - c_compiler = 'clang' - cxx_compiler = 'clang++' + c_compiler ||= 'clang' + cxx_compiler ||='clang++' else - c_compiler = 'gcc' - cxx_compiler = 'g++' + c_compiler ||= 'gcc' + cxx_compiler ||= 'g++' end c_platform_compiler = "#{host}-#{c_compiler}" @@ -101,14 +112,25 @@ def find_c_and_cxx_compilers(host) [c_compiler, cxx_compiler] end +def cmake_system_name + if darwin? + 'Darwin' + elsif windows? + 'Windows' + elsif freebsd? + 'FreeBSD' + else + 'Linux' + end +end + def cmake_compile_flags(host) - system_name = darwin? ? 'Darwin' : 'Linux' c_compiler, cxx_compiler = find_c_and_cxx_compilers(host) # needed to ensure cross-compilation with CMake targets the right CPU and compilers [ "-DCMAKE_SYSTEM_PROCESSOR=#{RbConfig::CONFIG['target_cpu']}", - "-DCMAKE_SYSTEM_NAME=#{system_name}", + "-DCMAKE_SYSTEM_NAME=#{cmake_system_name}", "-DCMAKE_C_COMPILER=#{c_compiler}", "-DCMAKE_CXX_COMPILER=#{cxx_compiler}" ] @@ -116,6 +138,7 @@ def cmake_compile_flags(host) # By default, mini_portile2 might add an unnecessary option: # https://github.com/flavorjones/mini_portile/blob/5084a2aeab12076f534cf0cabc81a4d5f84b5c25/lib/mini_portile2/mini_portile_cmake.rb#L17 +# See https://github.com/flavorjones/mini_portile/issues/127. def delete_cmake_generator_option!(options) indices = []