diff --git a/.github/workflows/precompiled.yml b/.github/workflows/precompiled.yml new file mode 100644 index 0000000..228be08 --- /dev/null +++ b/.github/workflows/precompiled.yml @@ -0,0 +1,348 @@ +name: precompiled +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + cancel-in-progress: true +on: + workflow_dispatch: + schedule: + - cron: "0 8 * * 3" # At 08:00 on Wednesday # https://crontab.guru/#0_8_*_*_3 + push: + branches: + - main + - v*.*.x + tags: + - v*.*.* + pull_request: + types: [opened, synchronize] + branches: + - '*' + +jobs: + precompiled: + strategy: + fail-fast: false + matrix: + runs-on: ["ubuntu-latest", "macos-latest"] + ruby: ["2.7", "3.0", "3.1", "3.2"] + include: + - ruby: "2.7" + runs-on: "windows-2019" + - ruby: "3.0" + runs-on: "windows-2019" + - ruby: "3.1" + runs-on: "windows-2022" + - ruby: "3.2" + runs-on: "windows-2022" + runs-on: ${{matrix.runs-on}} + steps: + - uses: actions/checkout@v2 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{matrix.ruby}} + bundler-cache: true + - uses: actions/cache@v2 + with: + path: ports + key: ports-${{matrix.runs-on}}-${{hashFiles('ext/re2/extconf.rb')}} + - run: bundle exec rake compile spec + + cruby-package: + runs-on: "ubuntu-latest" + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: ports/archives + key: archives-ubuntu-${{hashFiles('ext/re2/extconf.rb')}} + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.2" + bundler-cache: true + - run: ./scripts/test-gem-build gems ruby + - uses: actions/upload-artifact@v2 + with: + name: cruby-gem + path: gems + retention-days: 1 + + cruby-linux-install: + needs: ["cruby-package"] + strategy: + fail-fast: false + matrix: + ruby: ["2.7", "3.0", "3.1", "3.2"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "${{matrix.ruby}}" + - uses: actions/download-artifact@v2 + with: + name: cruby-gem + path: gems + - run: ./scripts/test-gem-install gems + + cruby-osx-install: + needs: ["cruby-package"] + strategy: + fail-fast: false + matrix: + ruby: ["3.2"] + sys: ["enable", "disable"] + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "${{matrix.ruby}}" + - uses: actions/download-artifact@v2 + with: + name: cruby-gem + path: gems + - run: ./scripts/test-gem-install gems + + cruby-windows-install: + needs: ["cruby-package"] + strategy: + fail-fast: false + matrix: + ruby: ["2.7", "3.0"] + sys: ["enable", "disable"] + runs-on: windows-latest + steps: + - uses: actions/checkout@v2 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "${{matrix.ruby}}" + - uses: actions/download-artifact@v2 + with: + name: cruby-gem + path: gems + - run: ./scripts/test-gem-install gems + shell: bash + + cruby-windows-install-ucrt: + needs: ["cruby-package"] + strategy: + fail-fast: false + matrix: + ruby: ["3.1", "3.2"] + sys: ["enable", "disable"] + runs-on: windows-2022 + steps: + - uses: actions/checkout@v2 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "${{matrix.ruby}}" + - uses: actions/download-artifact@v2 + with: + name: cruby-gem + path: gems + - run: ./scripts/test-gem-install gems + shell: bash + + cruby-native-package: + needs: ["rcd_image_version"] + strategy: + fail-fast: false + matrix: + plat: + - "aarch64-linux" + - "arm-linux" + - "arm64-darwin" + - "x64-mingw-ucrt" + - "x64-mingw32" + - "x86-linux" + - "x86_64-darwin" + - "x86_64-linux" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: ports/archives + key: archives-ubuntu-${{hashFiles('ext/re2/extconf.rb')}} + - env: + DOCKER_IMAGE: "ghcr.io/rake-compiler/rake-compiler-dock-image:${{needs.rcd_image_version.outputs.rcd_image_version}}-mri-${{matrix.plat}}" + run: | + docker run --rm -v "$(pwd):/re2" -w /re2 \ + ${DOCKER_IMAGE} \ + ./scripts/test-gem-build gems ${{matrix.plat}} + - uses: actions/upload-artifact@v3 + with: + name: "cruby-${{matrix.plat}}-gem" + path: gems + retention-days: 1 + + cruby-x86_64-linux-install: + needs: ["cruby-native-package"] + strategy: + fail-fast: false + matrix: + ruby: ["2.7", "3.0", "3.1", "3.2"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "${{matrix.ruby}}" + - uses: actions/download-artifact@v2 + with: + name: cruby-x86_64-linux-gem + path: gems + - run: ./scripts/test-gem-install gems + + cruby-x86-linux-install: + needs: ["cruby-native-package"] + strategy: + fail-fast: false + matrix: + ruby: ["2.7", "3.0", "3.1", "3.2"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 + with: + name: cruby-x86-linux-gem + path: gems + - run: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + docker run --rm -v "$(pwd):/re2" -w /re2 \ + --platform=linux/386 \ + ruby:${{matrix.ruby}} \ + ./scripts/test-gem-install ./gems + + cruby-aarch64-linux-install: + needs: ["cruby-native-package"] + strategy: + fail-fast: false + matrix: + ruby: ["2.7", "3.0", "3.1", "3.2"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 + with: + name: cruby-aarch64-linux-gem + path: gems + - run: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + docker run --rm -v "$(pwd):/re2" -w /re2 \ + --platform=linux/arm64/v8 \ + ruby:${{matrix.ruby}} \ + ./scripts/test-gem-install ./gems + + cruby-arm-linux-install: + needs: ["cruby-native-package"] + strategy: + fail-fast: false + matrix: + ruby: ["2.7", "3.0", "3.1", "3.2"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 + with: + name: cruby-arm-linux-gem + path: gems + - run: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + docker run --rm -v "$(pwd):/re2" -w /re2 \ + --platform=linux/arm/v7 \ + ruby:${{matrix.ruby}} \ + ./scripts/test-gem-install ./gems + + cruby-x86_64-musl-install: + needs: ["cruby-native-package"] + strategy: + fail-fast: false + matrix: + ruby: ["2.7", "3.0", "3.1", "3.2"] + runs-on: ubuntu-latest + container: + image: "ruby:${{matrix.ruby}}-alpine" + steps: + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 + with: + name: cruby-x86_64-linux-gem + path: gems + - run: apk add bash libstdc++ gcompat + - run: ./scripts/test-gem-install gems + + cruby-x86_64-darwin-install: + needs: ["cruby-native-package"] + strategy: + fail-fast: false + matrix: + ruby: ["2.7", "3.0", "3.1", "3.2"] + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "${{matrix.ruby}}" + - uses: actions/download-artifact@v2 + with: + name: cruby-x86_64-darwin-gem + path: gems + - run: ./scripts/test-gem-install gems + +## arm64-darwin installation testing is omitted until github actions supports it +# cruby-arm64-darwin-install: +# ... + + cruby-x64-mingw32-install: + needs: ["cruby-native-package"] + strategy: + fail-fast: false + matrix: + ruby: ["2.7", "3.0", "3.1", "3.2"] + runs-on: windows-2019 + steps: + - uses: actions/checkout@v2 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "${{matrix.ruby}}" + - uses: actions/download-artifact@v2 + with: + name: cruby-x64-mingw32-gem + path: gems + - run: ./scripts/test-gem-install gems + + cruby-x64-mingw-ucrt-install: + needs: ["cruby-native-package"] + strategy: + fail-fast: false + matrix: + ruby: ["3.2"] + runs-on: windows-2022 + steps: + - uses: actions/checkout@v2 + - uses: MSP-Greg/setup-ruby-pkgs@v1 + with: + ruby-version: "${{matrix.ruby}}" + - uses: actions/download-artifact@v2 + with: + name: cruby-x64-mingw-ucrt-gem + path: gems + - run: ./scripts/test-gem-install gems + shell: bash + + # + # SECTION the end-to-end gem installation tests + # + rcd_image_version: + runs-on: ubuntu-latest + outputs: + rcd_image_version: ${{steps.rcd_image_version.outputs.rcd_image_version}} + steps: + - uses: actions/checkout@v3 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.2" + bundler-cache: true + bundler: latest + - id: rcd_image_version + run: bundle exec ruby -e 'require "rake_compiler_dock"; puts "rcd_image_version=#{RakeCompilerDock::IMAGE_VERSION}"' >> $GITHUB_OUTPUT diff --git a/Rakefile b/Rakefile index 7c9c7d6..170a067 100644 --- a/Rakefile +++ b/Rakefile @@ -29,6 +29,7 @@ CROSS_RUBY_PLATFORMS = %w[ arm-linux arm64-darwin x64-mingw-ucrt + x64-mingw32 x86-linux x86-mingw32 x86_64-darwin @@ -118,5 +119,27 @@ task gem_build_path do add_vendored_libraries end -task :spec => :compile -task :default => :spec +desc "Temporarily set VERSION to a unique timestamp" +task "set-version-to-timestamp" do + # this task is used by bin/test-gem-build + # to test building, packaging, and installing a precompiled gem + version_constant_re = /^\s*VERSION\s*=\s*["'](.*)["']$/ + + version_file_path = File.join(__dir__, "lib/re2/version.rb") + version_file_contents = File.read(version_file_path) + + current_version_string = version_constant_re.match(version_file_contents)[1] + current_version = Gem::Version.new(current_version_string) + + fake_version = Gem::Version.new(format("%s.test.%s", current_version.bump, Time.now.strftime("%Y.%m%d.%H%M"))) + + unless version_file_contents.gsub!(version_constant_re, " VERSION = \"#{fake_version}\"") + raise("Could not hack the VERSION constant") + end + + File.open(version_file_path, "w") { |f| f.write(version_file_contents) } + + puts "NOTE: wrote version as \"#{fake_version}\"" +end + +task default: [:compile, :spec] diff --git a/ext/re2/extconf.rb b/ext/re2/extconf.rb index cd8a508..d3b53a4 100644 --- a/ext/re2/extconf.rb +++ b/ext/re2/extconf.rb @@ -8,7 +8,7 @@ PACKAGE_ROOT_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..', '..')) -REQUIRED_MINI_PORTILE_VERSION = "~> 2.8.2" # keep this version in sync with the one in the gemspec +REQUIRED_MINI_PORTILE_VERSION = "~> 2.8.4" # keep this version in sync with the one in the gemspec RE2_HELP_MESSAGE = <<~HELP USAGE: ruby #{$0} [options] @@ -82,73 +82,12 @@ def target_host host.gsub(/i386/, "i686") end -def find_compiler(compilers) - compilers.find { |binary| find_executable(binary) } -end - -# configure automatically searches for the right compiler based on the -# `--host` parameter. However, we don't have that feature with -# cmake. Search for the right compiler for the target architecture using -# some basic heruistics. -# See https://github.com/flavorjones/mini_portile/issues/128. -def find_c_and_cxx_compilers(host) - c_compiler = ENV["CC"] - cxx_compiler = ENV["CXX"] - - if darwin? - c_compiler ||= 'clang' - cxx_compiler ||='clang++' - else - c_compiler ||= 'gcc' - cxx_compiler ||= 'g++' - end - - c_platform_compiler = "#{host}-#{c_compiler}" - cxx_platform_compiler = "#{host}-#{cxx_compiler}" - c_compiler = find_compiler([c_platform_compiler, c_compiler]) - cxx_compiler = find_compiler([cxx_platform_compiler, cxx_compiler]) - - [c_compiler, cxx_compiler] -end - -def cmake_system_name - if darwin? - 'Darwin' - elsif windows? - 'Windows' - elsif freebsd? - 'FreeBSD' - else - 'Linux' - end -end - -def cmake_compile_flags(host) - c_compiler, cxx_compiler = find_c_and_cxx_compilers(host) - - # needed to ensure cross-compilation with CMake targets the right CPU and compilers - [ - "-DCMAKE_SYSTEM_PROCESSOR=#{RbConfig::CONFIG['target_cpu']}", - "-DCMAKE_SYSTEM_NAME=#{cmake_system_name}", - "-DCMAKE_C_COMPILER=#{c_compiler}", - "-DCMAKE_CXX_COMPILER=#{cxx_compiler}" - ] -end - -# By default, mini_portile2 might add an unnecessary option: -# https://github.com/flavorjones/mini_portile/blob/5084a2aeab12076f534cf0cabc81a4d5f84b5c25/lib/mini_portile2/mini_portile_cmake.rb#L17 -# See https://github.com/flavorjones/mini_portile/issues/127. -def delete_cmake_generator_option!(options) - indices = [] - - options.each_with_index do |element, index| - if element == '-G' && index + 1 < options.length - indices << index - indices << index + 1 +def with_temp_dir + Dir.mktmpdir do |temp_dir| + Dir.chdir(temp_dir) do + yield end end - - indices.reverse_each { |index| options.delete_at(index) } end # @@ -276,8 +215,6 @@ def process_recipe(name, version) # ensures pkg-config and installed libraries will be in lib, not lib64 '-DCMAKE_INSTALL_LIBDIR=lib' ] - recipe.configure_options += cmake_compile_flags(recipe.host) - delete_cmake_generator_option!(recipe.configure_options) yield recipe @@ -300,7 +237,9 @@ def process_recipe(name, version) end end - recipe.cook + # Use a temporary base directory to reduce filename lengths since + # Windows can hit a limit of 250 characters (CMAKE_OBJECT_PATH_MAX). + with_temp_dir { recipe.cook } FileUtils.touch(checkpoint) end @@ -327,6 +266,69 @@ def build_with_system_libraries build_extension end +# pkgconf v1.9.3 on Windows incorrectly sorts the output of `pkg-config +# --libs --static`, resulting in build failures: https://github.com/pkgconf/pkgconf/issues/268. +# To work around the issue, store the correct order of abseil flags here and add them manually +# for Windows. +ABSL_LDFLAGS = %w[ + -labsl_flags + -labsl_flags_internal + -labsl_flags_marshalling + -labsl_flags_reflection + -labsl_flags_private_handle_accessor + -labsl_flags_commandlineflag + -labsl_flags_commandlineflag_internal + -labsl_flags_config + -labsl_flags_program_name + -labsl_cord + -labsl_cordz_info + -labsl_cord_internal + -labsl_cordz_functions + -labsl_cordz_handle + -labsl_crc_cord_state + -labsl_crc32c + -labsl_crc_internal + -labsl_crc_cpu_detect + -labsl_hash + -labsl_city + -labsl_bad_variant_access + -labsl_low_level_hash + -labsl_raw_hash_set + -labsl_hashtablez_sampler + -labsl_exponential_biased + -labsl_bad_optional_access + -labsl_str_format_internal + -labsl_synchronization + -labsl_graphcycles_internal + -labsl_stacktrace + -labsl_symbolize + -labsl_debugging_internal + -labsl_demangle_internal + -labsl_malloc_internal + -labsl_time + -labsl_civil_time + -labsl_strings + -labsl_strings_internal + -labsl_base + -labsl_spinlock_wait + -labsl_int128 + -labsl_throw_delegate + -labsl_raw_logging_internal + -labsl_log_severity + -labsl_time_zone +].freeze + +def add_static_ldflags(flags) + static_flags = flags.split + + if MiniPortile.windows? + static_flags.each { |flag| append_ldflags(flag) unless ABSL_LDFLAGS.include?(flag) } + ABSL_LDFLAGS.each { |flag| append_ldflags(flag) } + else + static_flags.each { |flag| append_ldflags(flag) } + end +end + def build_with_vendored_libraries message "Building re2 using packaged libraries.\n" @@ -338,7 +340,7 @@ def build_with_vendored_libraries url: "https://github.com/abseil/abseil-cpp/archive/refs/tags/#{recipe.version}.tar.gz", sha256: dependencies['abseil']['sha256'] }] - recipe.configure_options += ['-DABSL_PROPAGATE_CXX_STD=ON'] + recipe.configure_options += ['-DABSL_PROPAGATE_CXX_STD=ON', '-DCMAKE_CXX_VISIBILITY_PRESET=hidden'] end re2_recipe = process_recipe('libre2', dependencies['libre2']['version']) do |recipe| @@ -346,7 +348,7 @@ def build_with_vendored_libraries url: "https://github.com/google/re2/releases/download/#{recipe.version}/re2-#{recipe.version}.tar.gz", sha256: dependencies['libre2']['sha256'] }] - recipe.configure_options += ["-DCMAKE_PREFIX_PATH=#{abseil_recipe.path}", '-DCMAKE_CXX_FLAGS=-DNDEBUG'] + recipe.configure_options += ["-DCMAKE_PREFIX_PATH=#{abseil_recipe.path}", '-DCMAKE_CXX_FLAGS=-DNDEBUG', '-DCMAKE_CXX_VISIBILITY_PRESET=hidden'] end dir_config("re2", File.join(re2_recipe.path, 'include'), File.join(re2_recipe.path, 'lib')) @@ -366,8 +368,10 @@ def build_with_vendored_libraries # See https://bugs.ruby-lang.org/issues/18490, broken in Ruby 3.1 but fixed in Ruby 3.2. flags = xpopen(['pkg-config', '--libs', '--static', pc_file], err: %i[child out], &:read) - flags.split.each { |flag| append_ldflags(flag) } if $?.success? + raise 'Unable to run pkg-config --libs --static' unless $?.success? + + add_static_ldflags(flags) build_extension end diff --git a/lib/re2.rb b/lib/re2.rb index 9c903ff..76d8fce 100644 --- a/lib/re2.rb +++ b/lib/re2.rb @@ -11,3 +11,4 @@ end require "re2/scanner" +require "re2/version" diff --git a/lib/re2/version.rb b/lib/re2/version.rb new file mode 100644 index 0000000..a96ca94 --- /dev/null +++ b/lib/re2/version.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module RE2 + VERSION = "1.7.0" +end diff --git a/re2.gemspec b/re2.gemspec index 9248960..9bdbca9 100644 --- a/re2.gemspec +++ b/re2.gemspec @@ -1,23 +1,29 @@ +require_relative 'lib/re2/version' + Gem::Specification.new do |s| s.name = "re2" s.summary = "Ruby bindings to re2." s.description = 'Ruby bindings to re2, "an efficient, principled regular expression library".' - s.version = "1.7.0" + s.version = RE2::VERSION s.authors = ["Paul Mucur"] s.homepage = "https://github.com/mudge/re2" s.extensions = ["ext/re2/extconf.rb"] s.license = "BSD-3-Clause" s.required_ruby_version = ">= 2.7.0" s.files = [ + ".rspec", "dependencies.yml", "ext/re2/extconf.rb", "ext/re2/re2.cc", + "Gemfile", "lib/re2.rb", "lib/re2/scanner.rb", "lib/re2/string.rb", + "lib/re2/version.rb", "LICENSE.txt", "README.md", - "Rakefile" + "Rakefile", + "re2.gemspec" ] s.test_files = [ "spec/spec_helper.rb", @@ -32,5 +38,5 @@ Gem::Specification.new do |s| s.add_development_dependency "rake-compiler", "~> 1.2.1" s.add_development_dependency "rake-compiler-dock", "~> 1.3.0" s.add_development_dependency("rspec", "~> 3.2") - s.add_runtime_dependency("mini_portile2", "~> 2.8.2") # keep version in sync with extconf.rb + s.add_runtime_dependency("mini_portile2", "~> 2.8.4") # keep version in sync with extconf.rb end diff --git a/scripts/test-gem-build b/scripts/test-gem-build new file mode 100755 index 0000000..0bdd772 --- /dev/null +++ b/scripts/test-gem-build @@ -0,0 +1,32 @@ +#! /usr/bin/env bash +# +# run as part of CI +# +if [[ $# -lt 2 ]] ; then + echo "usage: $(basename $0) " + exit 1 +fi + +set -e -u + +OUTPUT_DIR=$1 +BUILD_NATIVE_GEM=$2 + +test -e /etc/os-release && cat /etc/os-release + +set -x + +bundle install --local || bundle install +bundle exec rake set-version-to-timestamp + +if [[ "${BUILD_NATIVE_GEM}" == "ruby" ]] ; then + # TODO we're only compiling so that we retrieve tarballs, we can do better. + bundle exec rake clean compile + bundle exec rake gem +else + bundle exec rake gem:${BUILD_NATIVE_GEM}:builder +fi + +mkdir -p ${OUTPUT_DIR} +cp -v pkg/*.gem ${OUTPUT_DIR} +ls -l ${OUTPUT_DIR}/* diff --git a/scripts/test-gem-install b/scripts/test-gem-install new file mode 100755 index 0000000..3783967 --- /dev/null +++ b/scripts/test-gem-install @@ -0,0 +1,33 @@ +#! /usr/bin/env bash +# +# run as part of CI +# +if [[ $# -lt 1 ]] ; then + echo "usage: $(basename $0) [install_flags]" + exit 1 +fi + +GEMS_DIR=$1 +shift +INSTALL_FLAGS=$* + +test -e /etc/os-release && cat /etc/os-release + +set -e -x -u + +pushd $GEMS_DIR + gemfile=$(ls *.gem | head -n1) + ls -l ${gemfile} + gem install --no-document ${gemfile} -- ${INSTALL_FLAGS} + gem list -d re2 +popd + +if [ -n "${BUNDLE_APP_CONFIG:-}" ] ; then + export BUNDLE_CACHE_PATH="${BUNDLE_APP_CONFIG}/cache" +fi + +bundle install --local || bundle install + +cd $(dirname `gem which re2`) +cd .. +rake spec