Skip to content

Commit aa7790f

Browse files
stanhustanhu
committed
Vendor re2 and abseil
If `--disable-system-libraries` is specified, this change will download and build abseil and re2. `cmake` and a C++17 compiler is required for this to work. This makes it possible to ensure all the required dependencies are contained within the C extension rather than depend on system libraries, which may break the extension when updated. The building of these libraries uses mini_portile2 and techniques borrowed from the nokogiri and ruby-magic gems. Closes mudge#61
1 parent 7ffc9e3 commit aa7790f

File tree

4 files changed

+243
-58
lines changed

4 files changed

+243
-58
lines changed

‎Rakefile

+11
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,17 @@
11
require 'rake/extensiontask'
22
require 'rspec/core/rake_task'
33

4+
CLEAN.include FileList['**/*{.o,.so,.dylib,.bundle}'],
5+
FileList['**/extconf.h'],
6+
FileList['**/Makefile'],
7+
FileList['pkg/']
8+
9+
CLOBBER.include FileList['**/tmp'],
10+
FileList['**/*.log'],
11+
FileList['doc/**'],
12+
FileList['tmp/']
13+
CLOBBER.add("ports/*").exclude(%r{ports/archives$})
14+
415
Rake::ExtensionTask.new('re2')
516

617
RSpec::Core::RakeTask.new(:spec)

‎dependencies.yml

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
libre2:
2+
version: "2023-07-01"
3+
sha256: "18cf85922e27fad3ed9c96a27733037da445f35eb1a2744c306a37c6d11e95c4"
4+
# sha-256 hash provided in https://github.com/google/re2/releases/download/2023-07-01/re2-2023-07-01.tar.gz
5+
6+
abseil:
7+
version: "20230125.3"
8+
sha256: 5366d7e7fa7ba0d915014d387b66d0d002c03236448e1ba9ef98122c13b35c36
9+
# sha-256 hash provided in https://github.com/abseil/abseil-cpp/archive/refs/tags/20230125.3.tar.gz

‎ext/re2/extconf.rb

+221-58
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,70 @@
66

77
require 'mkmf'
88

9+
PACKAGE_ROOT_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
10+
11+
REQUIRED_MINI_PORTILE_VERSION = "~> 2.8.2" # keep this version in sync with the one in the gemspec
12+
13+
RE2_HELP_MESSAGE = <<~HELP
14+
USAGE: ruby #{$0} [options]
15+
16+
Flags that are always valid:
17+
18+
--use-system-libraries
19+
--enable-system-libraries
20+
Use system libraries instead of building and using the packaged libraries. This is the default.
21+
22+
--disable-system-libraries
23+
Use the packaged libraries, and ignore the system libraries. This overrides `--use-system-libraries`.
24+
25+
Flags only used when using system libraries:
26+
27+
Related to re2 library:
28+
29+
--with-re2-dir=DIRECTORY
30+
Look for re2 headers and library in DIRECTORY.
31+
32+
Environment variables used:
33+
34+
CC
35+
Use this path to invoke the compiler instead of `RbConfig::CONFIG['CC']`
36+
37+
CPPFLAGS
38+
If this string is accepted by the C preprocessor, add it to the flags passed to the C preprocessor
39+
40+
CFLAGS
41+
If this string is accepted by the compiler, add it to the flags passed to the compiler
42+
43+
LDFLAGS
44+
If this string is accepted by the linker, add it to the flags passed to the linker
45+
46+
LIBS
47+
Add this string to the flags passed to the linker
48+
HELP
49+
50+
#
51+
# utility functions
52+
#
53+
def config_system_libraries?
54+
enable_config("system-libraries", true) do |_, default|
55+
arg_config("--use-system-libraries", default)
56+
end
57+
end
58+
59+
def concat_flags(*args)
60+
args.compact.join(" ")
61+
end
62+
63+
def do_help
64+
print(RE2_HELP_MESSAGE)
65+
exit!(0)
66+
end
67+
68+
#
69+
# main
70+
#
71+
do_help if arg_config('--help')
72+
973
if ENV["CC"]
1074
RbConfig::MAKEFILE_CONFIG["CC"] = ENV["CC"]
1175
RbConfig::CONFIG["CC"] = ENV["CC"]
@@ -16,70 +80,57 @@
1680
RbConfig::CONFIG["CXX"] = ENV["CXX"]
1781
end
1882

19-
header_dirs = [
20-
"/usr/local/include",
21-
"/opt/homebrew/include",
22-
"/usr/include"
23-
]
24-
25-
lib_dirs = [
26-
"/usr/local/lib",
27-
"/opt/homebrew/lib",
28-
"/usr/lib"
29-
]
83+
def build_extension
84+
$CFLAGS << " -Wall -Wextra -funroll-loops"
3085

31-
dir_config("re2", header_dirs, lib_dirs)
86+
# Pass -x c++ to force gcc to compile the test program
87+
# as C++ (as it will end in .c by default).
88+
compile_options = "-x c++"
3289

33-
$CFLAGS << " -Wall -Wextra -funroll-loops"
90+
have_library("stdc++")
91+
have_header("stdint.h")
92+
have_func("rb_str_sublen")
3493

35-
# Pass -x c++ to force gcc to compile the test program
36-
# as C++ (as it will end in .c by default).
37-
compile_options = "-x c++"
38-
39-
have_library("stdc++")
40-
have_header("stdint.h")
41-
have_func("rb_str_sublen")
42-
43-
unless have_library("re2")
44-
abort "You must have re2 installed and specified with --with-re2-dir, please see https://github.com/google/re2/wiki/Install"
45-
end
94+
unless have_library("re2")
95+
abort "You must have re2 installed and specified with --with-re2-dir, please see https://github.com/google/re2/wiki/Install"
96+
end
4697

47-
minimal_program = <<SRC
98+
minimal_program = <<SRC
4899
#include <re2/re2.h>
49100
int main() { return 0; }
50101
SRC
51102

52-
re2_requires_version_flag = checking_for("re2 that requires explicit C++ version flag") do
53-
!try_compile(minimal_program, compile_options)
54-
end
103+
re2_requires_version_flag = checking_for("re2 that requires explicit C++ version flag") do
104+
!try_compile(minimal_program, compile_options)
105+
end
106+
107+
if re2_requires_version_flag
108+
# Recent versions of re2 depend directly on abseil, which requires a
109+
# compiler with C++14 support (see
110+
# https://github.com/abseil/abseil-cpp/issues/1127 and
111+
# https://github.com/abseil/abseil-cpp/issues/1431). However, the
112+
# `std=c++14` flag doesn't appear to suffice; we need at least
113+
# `std=c++17`.
114+
abort "Cannot compile re2 with your compiler: recent versions require C++14 support." unless %w[c++20 c++17 c++11 c++0x].any? do |std|
115+
checking_for("re2 that compiles with #{std} standard") do
116+
if try_compile(minimal_program, compile_options + " -std=#{std}")
117+
compile_options << " -std=#{std}"
118+
$CPPFLAGS << " -std=#{std}"
55119

56-
if re2_requires_version_flag
57-
# Recent versions of re2 depend directly on abseil, which requires a
58-
# compiler with C++14 support (see
59-
# https://github.com/abseil/abseil-cpp/issues/1127 and
60-
# https://github.com/abseil/abseil-cpp/issues/1431). However, the
61-
# `std=c++14` flag doesn't appear to suffice; we need at least
62-
# `std=c++17`.
63-
abort "Cannot compile re2 with your compiler: recent versions require C++14 support." unless %w[c++20 c++17 c++11 c++0x].any? do |std|
64-
checking_for("re2 that compiles with #{std} standard") do
65-
if try_compile(minimal_program, compile_options + " -std=#{std}")
66-
compile_options << " -std=#{std}"
67-
$CPPFLAGS << " -std=#{std}"
68-
69-
true
120+
true
121+
end
70122
end
71123
end
72124
end
73-
end
74125

75-
# Determine which version of re2 the user has installed.
76-
# Revision d9f8806c004d added an `endpos` argument to the
77-
# generic Match() function.
78-
#
79-
# To test for this, try to compile a simple program that uses
80-
# the newer form of Match() and set a flag if it is successful.
81-
checking_for("RE2::Match() with endpos argument") do
82-
test_re2_match_signature = <<SRC
126+
# Determine which version of re2 the user has installed.
127+
# Revision d9f8806c004d added an `endpos` argument to the
128+
# generic Match() function.
129+
#
130+
# To test for this, try to compile a simple program that uses
131+
# the newer form of Match() and set a flag if it is successful.
132+
checking_for("RE2::Match() with endpos argument") do
133+
test_re2_match_signature = <<SRC
83134
#include <re2/re2.h>
84135
85136
int main() {
@@ -91,13 +142,13 @@
91142
}
92143
SRC
93144

94-
if try_compile(test_re2_match_signature, compile_options)
95-
$defs.push("-DHAVE_ENDPOS_ARGUMENT")
145+
if try_compile(test_re2_match_signature, compile_options)
146+
$defs.push("-DHAVE_ENDPOS_ARGUMENT")
147+
end
96148
end
97-
end
98149

99-
checking_for("RE2::Set::Match() with error information") do
100-
test_re2_set_match_signature = <<SRC
150+
checking_for("RE2::Set::Match() with error information") do
151+
test_re2_set_match_signature = <<SRC
101152
#include <vector>
102153
#include <re2/re2.h>
103154
#include <re2/set.h>
@@ -115,9 +166,121 @@
115166
}
116167
SRC
117168

118-
if try_compile(test_re2_set_match_signature, compile_options)
119-
$defs.push("-DHAVE_ERROR_INFO_ARGUMENT")
169+
if try_compile(test_re2_set_match_signature, compile_options)
170+
$defs.push("-DHAVE_ERROR_INFO_ARGUMENT")
171+
end
120172
end
121173
end
122174

175+
def process_recipe(name, version)
176+
require "rubygems"
177+
gem("mini_portile2", REQUIRED_MINI_PORTILE_VERSION) # gemspec is not respected at install time
178+
require "mini_portile2"
179+
message("Using mini_portile version #{MiniPortile::VERSION}\n")
180+
181+
MiniPortileCMake.new(name, version).tap do |recipe|
182+
recipe.target = File.join(PACKAGE_ROOT_DIR, "ports")
183+
recipe.configure_options += ['-DCMAKE_CXX_STANDARD=17', '-DCMAKE_POSITION_INDEPENDENT_CODE=ON']
184+
185+
yield recipe
186+
187+
checkpoint = "#{recipe.target}/#{recipe.name}-#{recipe.version}-#{recipe.host}.installed"
188+
189+
if File.exist?(checkpoint)
190+
message("Building re2 with a packaged version of #{name}-#{version}.\n")
191+
else
192+
message(<<~EOM)
193+
---------- IMPORTANT NOTICE ----------
194+
Building re2 with a packaged version of #{name}-#{version}.
195+
Configuration options: #{recipe.configure_options.shelljoin}
196+
EOM
197+
198+
unless recipe.patch_files.empty?
199+
message("The following patches are being applied:\n")
200+
201+
recipe.patch_files.each do |patch|
202+
message(" - %s\n" % File.basename(patch))
203+
end
204+
end
205+
206+
recipe.cook
207+
208+
FileUtils.touch(checkpoint)
209+
end
210+
211+
recipe.activate
212+
end
213+
end
214+
215+
def build_with_system_libraries
216+
header_dirs = [
217+
"/usr/local/include",
218+
"/opt/homebrew/include",
219+
"/usr/include"
220+
]
221+
222+
lib_dirs = [
223+
"/usr/local/lib",
224+
"/opt/homebrew/lib",
225+
"/usr/lib"
226+
]
227+
228+
dir_config("re2", header_dirs, lib_dirs)
229+
230+
build_extension
231+
end
232+
233+
def build_with_vendored_libraries
234+
message "Building re2 using packaged libraries.\n"
235+
236+
require 'yaml'
237+
dependencies = YAML.load_file(File.join(PACKAGE_ROOT_DIR, 'dependencies.yml'))
238+
239+
abseil_recipe = process_recipe('abseil', dependencies['abseil']['version']) do |recipe|
240+
recipe.files = [{
241+
url: "https://github.com/abseil/abseil-cpp/archive/refs/tags/#{recipe.version}.tar.gz",
242+
sha256: dependencies['abseil']['sha256']
243+
}]
244+
recipe.configure_options += ['-DABSL_PROPAGATE_CXX_STD=ON']
245+
end
246+
247+
re2_recipe = process_recipe('libre2', dependencies['libre2']['version']) do |recipe|
248+
recipe.files = [{
249+
url: "https://github.com/google/re2/releases/download/#{recipe.version}/re2-#{recipe.version}.tar.gz",
250+
sha256: dependencies['libre2']['sha256']
251+
}]
252+
recipe.configure_options += ["-DCMAKE_PREFIX_PATH=#{abseil_recipe.path}", '-DCMAKE_CXX_FLAGS=-DNDEBUG']
253+
end
254+
255+
recipes = [abseil_recipe, re2_recipe]
256+
include_dirs = recipes.map { |recipe| File.join(recipe.path, 'include') }
257+
lib_dirs = recipes.map { |recipe| File.join(recipe.path, 'lib') }
258+
dir_config('re2', include_dirs, lib_dirs)
259+
260+
pkg_config_paths = [
261+
"#{abseil_recipe.path}/lib/pkgconfig",
262+
"#{re2_recipe.path}/lib/pkgconfig"
263+
].join(':')
264+
265+
pkg_config_paths = "#{ENV['PKG_CONFIG_PATH']}:#{pkg_config_paths}" if ENV['PKG_CONFIG_PATH']
266+
267+
ENV['PKG_CONFIG_PATH'] = pkg_config_paths
268+
pc_file = File.join(re2_recipe.path, 'lib', 'pkgconfig', 're2.pc')
269+
if pkg_config(pc_file)
270+
# See https://bugs.ruby-lang.org/issues/18490, broken in Ruby 3.1 but fixed in Ruby 3.2.
271+
flags = xpopen(['pkg-config', '--libs', '--static', pc_file], err: %i[child out], &:read)
272+
flags.split.each { |flag| append_ldflags(flag) } if $?.success?
273+
else
274+
raise 'Please install the `pkg-config` utility!'
275+
end
276+
277+
build_extension
278+
end
279+
280+
if config_system_libraries?
281+
build_with_system_libraries
282+
else
283+
build_with_vendored_libraries
284+
end
285+
123286
create_makefile("re2")

‎re2.gemspec

+2
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ Gem::Specification.new do |s|
1313
"lib/re2.rb",
1414
"lib/re2/scanner.rb",
1515
"lib/re2/string.rb",
16+
"dependencies.yml",
1617
"LICENSE.txt",
1718
"README.md",
1819
"Rakefile"
@@ -29,4 +30,5 @@ Gem::Specification.new do |s|
2930
]
3031
s.add_development_dependency("rake-compiler", "~> 0.9")
3132
s.add_development_dependency("rspec", "~> 3.2")
33+
s.add_runtime_dependency("mini_portile2", "~> 2.8.2") # keep version in sync with extconf.rb
3234
end

0 commit comments

Comments
 (0)