From aa7790f10fa94fbc032a60ce4ce196f5ad57c93b Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Tue, 11 Jul 2023 13:17:30 -0700 Subject: [PATCH] Vendor re2 and abseil If `--disable-system-libraries` is specified, this change will download and build abseil and re2. `cmake` and a C++17 compiler is required for this to work. This makes it possible to ensure all the required dependencies are contained within the C extension rather than depend on system libraries, which may break the extension when updated. The building of these libraries uses mini_portile2 and techniques borrowed from the nokogiri and ruby-magic gems. Closes https://github.com/mudge/re2/issues/61 --- Rakefile | 11 ++ dependencies.yml | 9 ++ ext/re2/extconf.rb | 279 +++++++++++++++++++++++++++++++++++---------- re2.gemspec | 2 + 4 files changed, 243 insertions(+), 58 deletions(-) create mode 100644 dependencies.yml diff --git a/Rakefile b/Rakefile index 957a0c2..c6fa8e9 100644 --- a/Rakefile +++ b/Rakefile @@ -1,6 +1,17 @@ require 'rake/extensiontask' require 'rspec/core/rake_task' +CLEAN.include FileList['**/*{.o,.so,.dylib,.bundle}'], + FileList['**/extconf.h'], + FileList['**/Makefile'], + FileList['pkg/'] + +CLOBBER.include FileList['**/tmp'], + FileList['**/*.log'], + FileList['doc/**'], + FileList['tmp/'] +CLOBBER.add("ports/*").exclude(%r{ports/archives$}) + Rake::ExtensionTask.new('re2') RSpec::Core::RakeTask.new(:spec) diff --git a/dependencies.yml b/dependencies.yml new file mode 100644 index 0000000..1043178 --- /dev/null +++ b/dependencies.yml @@ -0,0 +1,9 @@ +libre2: + version: "2023-07-01" + sha256: "18cf85922e27fad3ed9c96a27733037da445f35eb1a2744c306a37c6d11e95c4" + # sha-256 hash provided in https://github.com/google/re2/releases/download/2023-07-01/re2-2023-07-01.tar.gz + +abseil: + version: "20230125.3" + sha256: 5366d7e7fa7ba0d915014d387b66d0d002c03236448e1ba9ef98122c13b35c36 + # sha-256 hash provided in https://github.com/abseil/abseil-cpp/archive/refs/tags/20230125.3.tar.gz diff --git a/ext/re2/extconf.rb b/ext/re2/extconf.rb index 9887232..d1b3285 100644 --- a/ext/re2/extconf.rb +++ b/ext/re2/extconf.rb @@ -6,6 +6,70 @@ require 'mkmf' +PACKAGE_ROOT_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..', '..')) + +REQUIRED_MINI_PORTILE_VERSION = "~> 2.8.2" # keep this version in sync with the one in the gemspec + +RE2_HELP_MESSAGE = <<~HELP + USAGE: ruby #{$0} [options] + + Flags that are always valid: + + --use-system-libraries + --enable-system-libraries + Use system libraries instead of building and using the packaged libraries. This is the default. + + --disable-system-libraries + Use the packaged libraries, and ignore the system libraries. This overrides `--use-system-libraries`. + + Flags only used when using system libraries: + + Related to re2 library: + + --with-re2-dir=DIRECTORY + Look for re2 headers and library in DIRECTORY. + + Environment variables used: + + CC + Use this path to invoke the compiler instead of `RbConfig::CONFIG['CC']` + + CPPFLAGS + If this string is accepted by the C preprocessor, add it to the flags passed to the C preprocessor + + CFLAGS + If this string is accepted by the compiler, add it to the flags passed to the compiler + + LDFLAGS + If this string is accepted by the linker, add it to the flags passed to the linker + + LIBS + Add this string to the flags passed to the linker +HELP + +# +# utility functions +# +def config_system_libraries? + enable_config("system-libraries", true) do |_, default| + arg_config("--use-system-libraries", default) + end +end + +def concat_flags(*args) + args.compact.join(" ") +end + +def do_help + print(RE2_HELP_MESSAGE) + exit!(0) +end + +# +# main +# +do_help if arg_config('--help') + if ENV["CC"] RbConfig::MAKEFILE_CONFIG["CC"] = ENV["CC"] RbConfig::CONFIG["CC"] = ENV["CC"] @@ -16,70 +80,57 @@ RbConfig::CONFIG["CXX"] = ENV["CXX"] end -header_dirs = [ - "/usr/local/include", - "/opt/homebrew/include", - "/usr/include" -] - -lib_dirs = [ - "/usr/local/lib", - "/opt/homebrew/lib", - "/usr/lib" -] +def build_extension + $CFLAGS << " -Wall -Wextra -funroll-loops" -dir_config("re2", header_dirs, lib_dirs) + # Pass -x c++ to force gcc to compile the test program + # as C++ (as it will end in .c by default). + compile_options = "-x c++" -$CFLAGS << " -Wall -Wextra -funroll-loops" + have_library("stdc++") + have_header("stdint.h") + have_func("rb_str_sublen") -# Pass -x c++ to force gcc to compile the test program -# as C++ (as it will end in .c by default). -compile_options = "-x c++" - -have_library("stdc++") -have_header("stdint.h") -have_func("rb_str_sublen") - -unless have_library("re2") - abort "You must have re2 installed and specified with --with-re2-dir, please see https://github.com/google/re2/wiki/Install" -end + unless have_library("re2") + abort "You must have re2 installed and specified with --with-re2-dir, please see https://github.com/google/re2/wiki/Install" + end -minimal_program = < int main() { return 0; } SRC -re2_requires_version_flag = checking_for("re2 that requires explicit C++ version flag") do - !try_compile(minimal_program, compile_options) -end + re2_requires_version_flag = checking_for("re2 that requires explicit C++ version flag") do + !try_compile(minimal_program, compile_options) + end + + if re2_requires_version_flag + # Recent versions of re2 depend directly on abseil, which requires a + # compiler with C++14 support (see + # https://github.com/abseil/abseil-cpp/issues/1127 and + # https://github.com/abseil/abseil-cpp/issues/1431). However, the + # `std=c++14` flag doesn't appear to suffice; we need at least + # `std=c++17`. + abort "Cannot compile re2 with your compiler: recent versions require C++14 support." unless %w[c++20 c++17 c++11 c++0x].any? do |std| + checking_for("re2 that compiles with #{std} standard") do + if try_compile(minimal_program, compile_options + " -std=#{std}") + compile_options << " -std=#{std}" + $CPPFLAGS << " -std=#{std}" -if re2_requires_version_flag - # Recent versions of re2 depend directly on abseil, which requires a - # compiler with C++14 support (see - # https://github.com/abseil/abseil-cpp/issues/1127 and - # https://github.com/abseil/abseil-cpp/issues/1431). However, the - # `std=c++14` flag doesn't appear to suffice; we need at least - # `std=c++17`. - abort "Cannot compile re2 with your compiler: recent versions require C++14 support." unless %w[c++20 c++17 c++11 c++0x].any? do |std| - checking_for("re2 that compiles with #{std} standard") do - if try_compile(minimal_program, compile_options + " -std=#{std}") - compile_options << " -std=#{std}" - $CPPFLAGS << " -std=#{std}" - - true + true + end end end end -end -# Determine which version of re2 the user has installed. -# Revision d9f8806c004d added an `endpos` argument to the -# generic Match() function. -# -# To test for this, try to compile a simple program that uses -# the newer form of Match() and set a flag if it is successful. -checking_for("RE2::Match() with endpos argument") do - test_re2_match_signature = < int main() { @@ -91,13 +142,13 @@ } SRC - if try_compile(test_re2_match_signature, compile_options) - $defs.push("-DHAVE_ENDPOS_ARGUMENT") + if try_compile(test_re2_match_signature, compile_options) + $defs.push("-DHAVE_ENDPOS_ARGUMENT") + end end -end -checking_for("RE2::Set::Match() with error information") do - test_re2_set_match_signature = < #include #include @@ -115,9 +166,121 @@ } SRC - if try_compile(test_re2_set_match_signature, compile_options) - $defs.push("-DHAVE_ERROR_INFO_ARGUMENT") + if try_compile(test_re2_set_match_signature, compile_options) + $defs.push("-DHAVE_ERROR_INFO_ARGUMENT") + end end end +def process_recipe(name, version) + require "rubygems" + gem("mini_portile2", REQUIRED_MINI_PORTILE_VERSION) # gemspec is not respected at install time + require "mini_portile2" + message("Using mini_portile version #{MiniPortile::VERSION}\n") + + MiniPortileCMake.new(name, version).tap do |recipe| + recipe.target = File.join(PACKAGE_ROOT_DIR, "ports") + recipe.configure_options += ['-DCMAKE_CXX_STANDARD=17', '-DCMAKE_POSITION_INDEPENDENT_CODE=ON'] + + yield recipe + + checkpoint = "#{recipe.target}/#{recipe.name}-#{recipe.version}-#{recipe.host}.installed" + + if File.exist?(checkpoint) + message("Building re2 with a packaged version of #{name}-#{version}.\n") + else + message(<<~EOM) + ---------- IMPORTANT NOTICE ---------- + Building re2 with a packaged version of #{name}-#{version}. + Configuration options: #{recipe.configure_options.shelljoin} + EOM + + unless recipe.patch_files.empty? + message("The following patches are being applied:\n") + + recipe.patch_files.each do |patch| + message(" - %s\n" % File.basename(patch)) + end + end + + recipe.cook + + FileUtils.touch(checkpoint) + end + + recipe.activate + end +end + +def build_with_system_libraries + header_dirs = [ + "/usr/local/include", + "/opt/homebrew/include", + "/usr/include" + ] + + lib_dirs = [ + "/usr/local/lib", + "/opt/homebrew/lib", + "/usr/lib" + ] + + dir_config("re2", header_dirs, lib_dirs) + + build_extension +end + +def build_with_vendored_libraries + message "Building re2 using packaged libraries.\n" + + require 'yaml' + dependencies = YAML.load_file(File.join(PACKAGE_ROOT_DIR, 'dependencies.yml')) + + abseil_recipe = process_recipe('abseil', dependencies['abseil']['version']) do |recipe| + recipe.files = [{ + url: "https://github.com/abseil/abseil-cpp/archive/refs/tags/#{recipe.version}.tar.gz", + sha256: dependencies['abseil']['sha256'] + }] + recipe.configure_options += ['-DABSL_PROPAGATE_CXX_STD=ON'] + end + + re2_recipe = process_recipe('libre2', dependencies['libre2']['version']) do |recipe| + recipe.files = [{ + url: "https://github.com/google/re2/releases/download/#{recipe.version}/re2-#{recipe.version}.tar.gz", + sha256: dependencies['libre2']['sha256'] + }] + recipe.configure_options += ["-DCMAKE_PREFIX_PATH=#{abseil_recipe.path}", '-DCMAKE_CXX_FLAGS=-DNDEBUG'] + end + + recipes = [abseil_recipe, re2_recipe] + include_dirs = recipes.map { |recipe| File.join(recipe.path, 'include') } + lib_dirs = recipes.map { |recipe| File.join(recipe.path, 'lib') } + dir_config('re2', include_dirs, lib_dirs) + + pkg_config_paths = [ + "#{abseil_recipe.path}/lib/pkgconfig", + "#{re2_recipe.path}/lib/pkgconfig" + ].join(':') + + pkg_config_paths = "#{ENV['PKG_CONFIG_PATH']}:#{pkg_config_paths}" if ENV['PKG_CONFIG_PATH'] + + ENV['PKG_CONFIG_PATH'] = pkg_config_paths + pc_file = File.join(re2_recipe.path, 'lib', 'pkgconfig', 're2.pc') + if pkg_config(pc_file) + # See https://bugs.ruby-lang.org/issues/18490, broken in Ruby 3.1 but fixed in Ruby 3.2. + flags = xpopen(['pkg-config', '--libs', '--static', pc_file], err: %i[child out], &:read) + flags.split.each { |flag| append_ldflags(flag) } if $?.success? + else + raise 'Please install the `pkg-config` utility!' + end + + build_extension +end + +if config_system_libraries? + build_with_system_libraries +else + build_with_vendored_libraries +end + create_makefile("re2") diff --git a/re2.gemspec b/re2.gemspec index e64b5a6..18a43e6 100644 --- a/re2.gemspec +++ b/re2.gemspec @@ -13,6 +13,7 @@ Gem::Specification.new do |s| "lib/re2.rb", "lib/re2/scanner.rb", "lib/re2/string.rb", + "dependencies.yml", "LICENSE.txt", "README.md", "Rakefile" @@ -29,4 +30,5 @@ Gem::Specification.new do |s| ] s.add_development_dependency("rake-compiler", "~> 0.9") s.add_development_dependency("rspec", "~> 3.2") + s.add_runtime_dependency("mini_portile2", "~> 2.8.2") # keep version in sync with extconf.rb end