Build Ruby 3.1

Author: adfoster-r7

.github/workflows/verify.yml

@@ -300,12 +300,14 @@ jobs:
         run: |
           cd metasploit-omnibus
           xz -d local/cache/*.xz
+          ls local/cache
 
       - name: Run omnibus
         run: |
           # mkdir -p metasploit-omnibus/local/cache/git_cache/c
           # mv metasploit-framework/ metasploit-omnibus/local/cache/git_cache/c
           # dir metasploit-omnibus/local/cache/git_cache/c
           # dir metasploit-omnibus/local/cache/git_cache/c/metasploit-framework
+          mkdir C:/metasploit-framework
           cd metasploit-omnibus
           make

.gitmodules

@@ -1,3 +1,4 @@
 [submodule "local/cache"]
 	path = local/cache
-	url = https://github.com/rapid7/metasploit-omnibus-cache.git
+	url = https://github.com/adfoster-r7/metasploit-omnibus-cache.git
+	branch = prepare-for-ruby-3.1-update

Makefile

@@ -3,7 +3,6 @@
 .PHONY: all
 all: certs/ca-certificates.crt
 	# export SSL_CERT_FILE=${PWD}/certs/ca-certificates.crt
-
 	# Ensure consistent bundler versions
 	gem install bundler -v 2.2.3
 

config/patches/ruby/ruby-fast-load_31.patch

@@ -0,0 +1,16 @@
+diff --git a/load.c b/load.c
+index a2b9da4..05ea96e 100644
+--- a/load.c
++++ b/load.c
+@@ -981,6 +981,11 @@ search_required(rb_vm_t *vm, VALUE fname, volatile VALUE *path, feature_func rb_
+        if (loading) *path = rb_filesystem_str_new_cstr(loading);
+        return 'r';
+     }
++    else if ((ft = rb_feature_p(vm, ftptr, 0, FALSE, FALSE, &loading)) == 's') {
++       if (loading) *path = rb_filesystem_str_new_cstr(loading);
++       return 's';
++    }
++
+     tmp = fname;
+     type = rb_find_file_ext(&tmp, ft == 's' ? ruby_ext : loadable_ext);
+     switch (type) {

config/software/metasploit-framework.rb

@@ -19,7 +19,7 @@
   dependency "postgresql"
 end
 
-ruby_abi_version = "3.0.0"
+ruby_abi_version = "3.1.0"
 # This depends on extra system libraries on OS X
 whitelist_file "#{install_dir}//embedded/framework/data/isight.bundle"
 

config/software/openssl.rb

@@ -23,29 +23,50 @@
 dependency "cacerts"
 dependency "openssl-fips" if fips_mode?
 
-default_version "1.1.1m"
+default_version "1.1.1t" # # do not remove - Rapid7 custom - do not remove
 
 # Openssl builds engines as libraries into a special directory. We need to include
 # that directory in lib_dirs so omnibus can sign them during macOS deep signing.
 lib_dirs lib_dirs.concat(["#{install_dir}/embedded/lib/engines"])
 lib_dirs lib_dirs.concat(["#{install_dir}/embedded/lib/engines-1.1"]) if version.start_with?("1.1")
+if version.start_with?("3.")
+  lib_dirs lib_dirs.concat(["#{install_dir}/embedded/lib/engines-3"])
+  lib_dirs lib_dirs.concat(["#{install_dir}/embedded/lib/ossl-modules"])
+end
 
 # 1.0.2u was the last public release of 1.0.2. Subsequent releases come from a support contract with OpenSSL Software Services
 if version.satisfies?("< 1.1.0")
   source url: "https://s3.amazonaws.com/chef-releng/openssl/openssl-#{version}.tar.gz", extract: :lax_tar
+  internal_source url: "#{ENV["ARTIFACTORY_REPO_URL"]}/#{name}/#{name}-#{version}.tar.gz", extract: :lax_tar,
+                  authorization: "X-JFrog-Art-Api:#{ENV["ARTIFACTORY_TOKEN"]}"
 else
   # As of 2020-09-09 even openssl-1.0.0.tar.gz can be downloaded from /source/openssl-VERSION.tar.gz
   # However, the latest releases are not in /source/old/VERSION/openssl-VERSION.tar.gz.
   # Let's stick with the simpler one for now.
   source url: "https://www.openssl.org/source/openssl-#{version}.tar.gz", extract: :lax_tar
+  internal_source url: "#{ENV["ARTIFACTORY_REPO_URL"]}/#{name}/#{name}-#{version}.tar.gz", extract: :lax_tar,
+                  authorization: "X-JFrog-Art-Api:#{ENV["ARTIFACTORY_TOKEN"]}"
 end
 
-version("1.1.1m") { source sha256: "f89199be8b23ca45fc7cb9f1d8d3ee67312318286ad030f5316aca6462db6c96" }
-version("1.1.1l") { source sha256: "0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1" }
-version("1.1.1f") { source sha256: "186c6bfe6ecfba7a5b48c47f8a1673d0f3b0e5ba2e25602dd23b629975da3f35" }
+version("3.0.5")   { source sha256: "aa7d8d9bef71ad6525c55ba11e5f4397889ce49c2c9349dcea6d3e4f0b024a7a" }
+version("3.0.4")   { source sha256: "2831843e9a668a0ab478e7020ad63d2d65e51f72977472dc73efcefbafc0c00f" }
+version("3.0.3")   { source sha256: "ee0078adcef1de5f003c62c80cc96527721609c6f3bb42b7795df31f8b558c0b" }
+version("3.0.1")   { source sha256: "c311ad853353bce796edad01a862c50a8a587f62e7e2100ef465ab53ec9b06d1" } # only ruby 3.1 supports openssl-3.0.1
+
+version("1.1.1t")  { source sha256: "8dee9b24bdb1dcbf0c3d1e9b02fb8f6bf22165e807f45adeb7c9677536859d3b" }
+version("1.1.1q")  { source sha256: "d7939ce614029cdff0b6c20f0e2e5703158a489a72b2507b8bd51bf8c8fd10ca" }
+version("1.1.1p")  { source sha256: "bf61b62aaa66c7c7639942a94de4c9ae8280c08f17d4eac2e44644d9fc8ace6f" }
+version("1.1.1o")  { source sha256: "9384a2b0570dd80358841464677115df785edb941c71211f75076d72fe6b438f" }
+version("1.1.1m")  { source sha256: "f89199be8b23ca45fc7cb9f1d8d3ee67312318286ad030f5316aca6462db6c96" }
+version("1.1.1l")  { source sha256: "0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1" }
+version("1.1.1w")  { source sha256: "cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8" }
 
+version("1.0.2zg") { source sha256: "09f8372eaede77ec8e6945e2d2d8eeb1b91662980cf23fe95f627b377162296c" }
 version("1.0.2zb") { source sha256: "b7d8f8c895279caa651e7f3de9a7b87b8dd01a452ca3d9327f45a9ef31d0c518" }
 version("1.0.2za") { source sha256: "86ec5d2ecb53839e9ec999db7f8715d0eb7e534d8a1d8688ef25280fbeee2ff8" }
+version("1.0.2ze") { source sha256: "796624c593c361c695bd16314bc6f944184f5d2ff87efcf0bfa0545aa84c4d88" }
+version("1.0.2zf") { source sha256: "85d2242b7d11a33d5f239f1f34a1ff7eb37431a554b7df99c52c646b70b14b2e" }
+version("1.0.2zi") { source sha256: "80b6c07995fc92456e31c61cf1b2a18f75e314063189bb183af6ae66d0261d84" }
 
 relative_path "openssl-#{version}"
 
@@ -55,9 +76,6 @@
     env["M4"] = "/opt/freeware/bin/m4"
   elsif mac_os_x? && arm?
     env["CFLAGS"] << " -Qunused-arguments"
-  elsif freebsd?
-    # Should this just be in standard_compiler_flags?
-    env["LDFLAGS"] += " -Wl,-rpath,#{install_dir}/embedded/lib"
   elsif windows?
     # XXX: OpenSSL explicitly sets -march=i486 and expects that to be honored.
     # It has OPENSSL_IA32_SSE2 controlling whether it emits optimized SSE2 code
@@ -77,15 +95,21 @@
     "no-mdc2",
     "no-rc5",
     "no-ssl2",
-    "enable-ssl3",
+    "no-ssl3",
     "no-zlib",
     "shared",
   ]
 
+  configure_args += ["--libdir=#{install_dir}/embedded/lib"] if version.satisfies?(">=3.0.1")
+
   # https://www.openssl.org/blog/blog/2021/09/13/LetsEncryptRootCertExpire/
   configure_args += [ "-DOPENSSL_TRUSTED_FIRST_DEFAULT" ] if version.satisfies?(">= 1.0.2zb") && version.satisfies?("< 1.1.0")
 
-  configure_args += ["--with-fipsdir=#{install_dir}/embedded", "fips"] if fips_mode?
+  if version.satisfies?("< 3.0.0")
+    configure_args += ["--with-fipsdir=#{install_dir}/embedded", "fips"] if fips_mode?
+  else
+    configure_args += ["-enable-fips"] if fips_mode?
+  end
 
   configure_cmd =
     if aix?
@@ -135,6 +159,8 @@
     patch source: "openssl-1.0.1f-do-not-build-docs.patch", env: patch_env
   elsif version.start_with? "1.1"
     patch source: "openssl-1.1.0f-do-not-install-docs.patch", env: patch_env
+  elsif version.start_with? "3.0"
+    patch source: "openssl-3.0.1-do-not-install-docs.patch", env: patch_env
   end
 
   if version.start_with?("1.0.2") && mac_os_x? && arm?
@@ -148,7 +174,7 @@
 
   # Out of abundance of caution, we put the feature flags first and then
   # the crazy platform specific compiler flags at the end.
-  configure_args << env["CFLAGS"] << env["LDFLAGS"]
+  configure_args << env["CFLAGS"]
 
   configure_command = configure_args.unshift(configure_cmd).join(" ")
 

config/software/ruby-windows.rb

@@ -15,7 +15,7 @@
 #
 
 name "ruby-windows"
-default_version "3.0.2-1"
+default_version "3.1.4-1"
 
 if windows_arch_i386?
   relative_path "rubyinstaller-#{version}-x86"
@@ -54,6 +54,10 @@
     source sha256: "92894c0488ec7eab02b2ffc61a8945c4bf98d69561e170927ec30d60bee57898"
   end
 
+  version "3.1.4-1" do
+    source sha256: "6701088607ea4b587a31af76d75cb3fe9f7bcd75fc175cffcca22369ebb6331d"
+  end
+
   source url: "https://github.com/oneclick/rubyinstaller2/releases/download/rubyinstaller-#{version}/rubyinstaller-#{version}-x64.7z"
 end
 

config/software/ruby.rb

@@ -26,29 +26,50 @@
 # the default versions should always be the latest release of ruby
 # if you consume this definition it is your responsibility to pin
 # to the desired version of ruby. don't count on this not changing.
-default_version "3.0.2"
+default_version "3.1.4"
 
 dependency "zlib"
 dependency "openssl"
 dependency "libffi"
 dependency "libyaml"
 
+# we build omnibus packages on freebsd 11 and use the packages on freebsd 11, 12 and 13.
+# the ruby executable has been linking to freebsds system's ncurses library files.
+# freebsd 13 system's ncurses library files have a different name than freebsd 11 and 12
+# which causes the ruby executable to fail.
+# adding ncurses as a dependency for freebsd prevents the ruby executable from linking to the
+# system's ncurses library files thereby allowing the package built on freebsd 11 to work on freebsd 13.
+dependency "ncurses" if freebsd?
+
 # version_list: url=https://cache.ruby-lang.org/pub/ruby/ filter=*.tar.gz
+version("3.2.2")      { source sha256: "96c57558871a6748de5bc9f274e93f4b5aad06cd8f37befa0e8d94e7b8a423bc" }
+version("3.2.0")      { source sha256: "daaa78e1360b2783f98deeceb677ad900f3a36c0ffa6e2b6b19090be77abc272" }
+version("3.1.4")      { source sha256: "a3d55879a0dfab1d7141fdf10d22a07dbf8e5cdc4415da1bde06127d5cc3c7b6" }
+version("3.1.3")      { source sha256: "5ea498a35f4cd15875200a52dde42b6eb179e1264e17d78732c3a57cd1c6ab9e" }
+version("3.1.2")      { source sha256: "61843112389f02b735428b53bb64cf988ad9fb81858b8248e22e57336f24a83e" }
+version("3.1.1")      { source sha256: "fe6e4782de97443978ddba8ba4be38d222aa24dc3e3f02a6a8e7701c0eeb619d" }
+
+version("3.0.6")      { source sha256: "6e6cbd490030d7910c0ff20edefab4294dfcd1046f0f8f47f78b597987ac683e" }
+version("3.0.5")      { source sha256: "9afc6380a027a4fe1ae1a3e2eccb6b497b9c5ac0631c12ca56f9b7beb4848776" }
+version("3.0.4")      { source sha256: "70b47c207af04bce9acea262308fb42893d3e244f39a4abc586920a1c723722b" }
+version("3.0.3")      { source sha256: "3586861cb2df56970287f0fd83f274bd92058872d830d15570b36def7f1a92ac" }
 version("3.0.2")      { source sha256: "5085dee0ad9f06996a8acec7ebea4a8735e6fac22f22e2d98c3f2bc3bef7e6f1" }
 version("3.0.1")      { source sha256: "369825db2199f6aeef16b408df6a04ebaddb664fb9af0ec8c686b0ce7ab77727" }
-version("3.0.0")      { source sha256: "a13ed141a1c18eb967aac1e33f4d6ad5f21be1ac543c344e0d6feeee54af8e28" }
 
+version("2.7.7")      { source sha256: "e10127db691d7ff36402cfe88f418c8d025a3f1eea92044b162dd72f0b8c7b90" }
+version("2.7.6")      { source sha256: "e7203b0cc09442ed2c08936d483f8ac140ec1c72e37bb5c401646b7866cb5d10" }
+version("2.7.5")      { source sha256: "2755b900a21235b443bb16dadd9032f784d4a88f143d852bc5d154f22b8781f1" }
 version("2.7.4")      { source sha256: "3043099089608859fc8cce7f9fdccaa1f53a462457e3838ec3b25a7d609fbc5b" }
 version("2.7.3")      { source sha256: "8925a95e31d8f2c81749025a52a544ea1d05dad18794e6828709268b92e55338" }
-version("2.7.2")      { source sha256: "6e5706d0d4ee4e1e2f883db9d768586b4d06567debea353c796ec45e8321c3d4" }
-version("2.7.1")      { source sha256: "d418483bdd0000576c1370571121a6eb24582116db0b7bb2005e90e250eae418" }
 
+version("2.6.10")     { source sha256: "0dc609f263d49c4176d5725deefc337273676395985b5e017789373e8cadf16e" }
+version("2.6.9")      { source sha256: "eb7bae7aac64bf9eb2153710a4cafae450ccbb62ae6f63d573e1786178b0efbb" }
 version("2.6.8")      { source sha256: "1807b78577bc08596a390e8a41aede37b8512190e05c133b17d0501791a8ca6d" }
 version("2.6.7")      { source sha256: "e4227e8b7f65485ecb73397a83e0d09dcd39f25efd411c782b69424e55c7a99e" }
-version("2.6.6")      { source sha256: "364b143def360bac1b74eb56ed60b1a0dca6439b00157ae11ff77d5cd2e92291" }
-version("2.6.5")      { source sha256: "66976b716ecc1fd34f9b7c3c2b07bbd37631815377a2e3e85a5b194cfdcbed7d" }
 
 source url: "https://cache.ruby-lang.org/pub/ruby/#{version.match(/^(\d+\.\d+)/)[0]}/ruby-#{version}.tar.gz"
+internal_source url: "#{ENV["ARTIFACTORY_REPO_URL"]}/#{name}/#{name}-#{version}.tar.gz",
+                authorization: "X-JFrog-Art-Api:#{ENV["ARTIFACTORY_TOKEN"]}"
 
 # In order to pass notarization we need to sign any binaries and libraries included in the package.
 # This makes sure we include and bins and libs that are brought in by gems.
@@ -63,6 +84,7 @@
 
 env = with_standard_compiler_flags(with_embedded_path)
 
+# do not remove - Rapid7 custom - do not remove:
 jemalloc_required = linux? || mac_os_x?
 if jemalloc_required
   dependency "jemalloc"
@@ -121,10 +143,12 @@
   patch_env = env.dup
   patch_env["PATH"] = "/opt/freeware/bin:#{env["PATH"]}" if aix?
 
-  if version.satisfies?("~> 3.0")
+  if version.satisfies?("~> 3.0.0")
     case version
-    when "3.0.0", "3.0.1"
+    when "3.0.1"
       patch source: "ruby-3.0.1-configure.patch", plevel: 1, env: patch_env
+    when "3.0.5", "3.0.6"
+      patch source: "ruby-3.0.5-configure.patch", plevel: 1, env: patch_env
     else
       patch source: "ruby-3.0.2-configure.patch", plevel: 1, env: patch_env
     end
@@ -153,7 +177,11 @@
   # over the top of it.  AFAIK no sane ruby code should need to do that, and the
   # cost of this behavior in core ruby is enormous.
   #
-  patch source: "ruby-fast-load_26.patch", plevel: 1, env: patch_env
+  if version.satisfies?("< 3.1")
+    patch source: "ruby-fast-load_26.patch", plevel: 1, env: patch_env
+  else
+    patch source: "ruby-fast-load_31.patch", plevel: 1, env: patch_env
+  end
 
   # this removes a checks for windows nano in the win32-ole files.
   # windows nano is a dead platform and not supported by chef so we can avoid
@@ -178,13 +206,6 @@
     patch source: "ruby-faster-load_27.patch", plevel: 1, env: patch_env
   end
 
-  # rubygems 3.1.x perf improvements
-  # this is part of ruby 2.7.3 so skip it
-  #
-  if version.satisfies?("~> 2.7") && version.satisfies?("< 2.7.3")
-    patch source: "ruby-2.7.1-rubygemsperf.patch", plevel: 1, env: patch_env
-  end
-
   # disable libpath in mkmf across all platforms, it trolls omnibus and
   # breaks the postgresql cookbook.  i'm not sure why ruby authors decided
   # this was a good idea, but it breaks our use case hard.  AIX cannot even
@@ -198,6 +219,7 @@
 
   configure_command = ["--with-out-ext=dbm,readline",
                        "--enable-shared",
+                       # do not remote - Rapid7 custom - do not remove:
                        "--enable-load-relative",
                        "--disable-install-doc",
                        "--without-gmp",
@@ -206,19 +228,16 @@
                        "--disable-dtrace",
                        "--disable-jit-support"]
   configure_command << "--with-bundled-md5" if fips_mode?
+  # do not remote - Rapid7 custom - do not remove:
   configure_command << "--with-jemalloc" if jemalloc_required
 
   # resolve C99 code accidentally introduced in Ruby 2.6.7 and it's still in 2.6.8 :(
-  patch source: "ruby-2.6.7_c99.patch", plevel: 1, env: patch_env if version.satisfies?("~> 2.6.7")
+  patch source: "ruby-2.6.7_c99.patch", plevel: 1, env: patch_env if version.satisfies?("~> 2.6.7", "< 2.6.10")
 
   if aix?
     # need to patch ruby's configure file so it knows how to find shared libraries
     patch source: "ruby-aix-configure_26_and_later.patch", plevel: 1, env: patch_env
 
-    if version.satisfies?("~> 2.6.4")
-      patch source: "ruby-2.6.4-bug14834.patch", plevel: 1, env: patch_env
-    end
-
     # have ruby use zlib on AIX correctly
     patch source: "ruby_aix_openssl.patch", plevel: 1, env: patch_env
     # AIX has issues with ssl retries, need to patch to have it retry

config/software/winpcap-devpack.rb

@@ -29,7 +29,7 @@
 source url: "https://www.winpcap.org/install/bin/WpdPack_4_1_2.zip"
 
 build do
-  ruby_api_version = "3.0.0"
+  ruby_api_version = "3.1.0"
 
   mkdir "#{install_dir}/embedded/lib"
   if windows_arch_i386?

omnibus.rb

@@ -27,7 +27,7 @@
 
 # Disable git caching
 # ------------------------------
-# use_git_caching false
+use_git_caching false
 
 # Enable S3 asset caching
 # ------------------------------