WIP

Signed-off-by: Juan Cruz Viotti <[email protected]>

Author: jviotti

CMakeLists.txt

@@ -82,6 +82,7 @@ endif()
 
 if(SOURCEMETA_CORE_REGEX)
   find_package(BoostRegex REQUIRED)
+  find_package(ICU REQUIRED)
   add_subdirectory(src/core/regex)
 endif()
 

cmake/FindICU.cmake

@@ -0,0 +1,54 @@
+if(NOT ICU_FOUND)
+  set(ICU_ROOT "/opt/homebrew/opt/icu4c")
+
+  find_path(ICU_INCLUDE_DIR
+    NAMES unicode/unistr.h
+    PATHS ${ICU_ROOT}/include
+    NO_DEFAULT_PATH)
+
+  find_library(ICU_UC_LIBRARY
+    NAMES icuuc
+    PATHS ${ICU_ROOT}/lib
+    NO_DEFAULT_PATH)
+
+  find_library(ICU_I18N_LIBRARY
+    NAMES icui18n
+    PATHS ${ICU_ROOT}/lib
+    NO_DEFAULT_PATH)
+
+  find_library(ICU_DATA_LIBRARY
+    NAMES icudata
+    PATHS ${ICU_ROOT}/lib
+    NO_DEFAULT_PATH)
+
+  if(ICU_INCLUDE_DIR AND ICU_UC_LIBRARY AND ICU_I18N_LIBRARY AND ICU_DATA_LIBRARY)
+    set(ICU_FOUND ON)
+    set(ICU_INCLUDE_DIRS ${ICU_INCLUDE_DIR})
+    set(ICU_LIBRARIES ${ICU_UC_LIBRARY} ${ICU_I18N_LIBRARY} ${ICU_DATA_LIBRARY})
+
+    if(NOT TARGET ICU::uc)
+      add_library(ICU::uc UNKNOWN IMPORTED)
+      set_target_properties(ICU::uc PROPERTIES
+        IMPORTED_LOCATION "${ICU_UC_LIBRARY}"
+        INTERFACE_INCLUDE_DIRECTORIES "${ICU_INCLUDE_DIR}")
+    endif()
+
+    if(NOT TARGET ICU::i18n)
+      add_library(ICU::i18n UNKNOWN IMPORTED)
+      set_target_properties(ICU::i18n PROPERTIES
+        IMPORTED_LOCATION "${ICU_I18N_LIBRARY}"
+        INTERFACE_INCLUDE_DIRECTORIES "${ICU_INCLUDE_DIR}")
+    endif()
+
+    if(NOT TARGET ICU::data)
+      add_library(ICU::data UNKNOWN IMPORTED)
+      set_target_properties(ICU::data PROPERTIES
+        IMPORTED_LOCATION "${ICU_DATA_LIBRARY}"
+        INTERFACE_INCLUDE_DIRECTORIES "${ICU_INCLUDE_DIR}")
+    endif()
+
+    message(STATUS "ICU found")
+  else()
+    message(FATAL_ERROR "ICU not found. Please install ICU or set ICU_ROOT.")
+  endif()
+endif()

src/core/regex/CMakeLists.txt

@@ -4,4 +4,4 @@ if(SOURCEMETA_CORE_INSTALL)
   sourcemeta_library_install(NAMESPACE sourcemeta PROJECT core NAME regex)
 endif()
 
-target_link_libraries(sourcemeta_core_regex INTERFACE Boost::regex)
+target_link_libraries(sourcemeta_core_regex INTERFACE Boost::regex ICU::uc ICU::i18n ICU::data)

src/core/regex/include/sourcemeta/core/regex.h

@@ -12,6 +12,9 @@
 #pragma GCC diagnostic ignored "-Wconversion"
 #endif
 #include <boost/regex.hpp>
+#include <memory> // std::unique_ptr
+#include <unicode/regex.h>
+#include <unicode/unistr.h>
 #if defined(__clang__)
 #pragma clang diagnostic pop
 #elif defined(__GNUC__)
@@ -20,6 +23,7 @@
 
 #include <cassert>  // assert
 #include <cstdint>  // std::uint8_t, std::uint64_t
+#include <iostream> // std::cerr
 #include <optional> // std::optional
 #include <regex>    // std::regex
 #include <string>   // std::stoull
@@ -59,12 +63,15 @@ struct RegexTypeNoop {
   auto operator==(const RegexTypeNoop &) const noexcept -> bool = default;
 };
 
+/// @ingroup regex
+using RegexTypeICU = std::shared_ptr<icu::RegexPattern>;
+
 /// @ingroup regex
 template <typename T>
-using Regex =
-    std::variant<RegexTypeBoost<typename T::value_type>, RegexTypePrefix<T>,
-                 RegexTypeNonEmpty, RegexTypeRange,
-                 RegexTypeStd<typename T::value_type>, RegexTypeNoop>;
+using Regex = std::variant<RegexTypeBoost<typename T::value_type>,
+                           RegexTypePrefix<T>, RegexTypeNonEmpty,
+                           RegexTypeRange, RegexTypeStd<typename T::value_type>,
+                           RegexTypeNoop, RegexTypeICU>;
 #if !defined(DOXYGEN)
 // For fast internal dispatching. It must stay in sync with the variant above
 enum class RegexIndex : std::uint8_t {
@@ -73,10 +80,31 @@ enum class RegexIndex : std::uint8_t {
   NonEmpty,
   Range,
   Std,
-  Noop
+  Noop,
+  ICU
 };
 #endif
 
+// Helper to detect if a pattern requires Unicode support
+template <typename T> auto requires_unicode(const T &pattern) -> bool {
+  if (pattern.find("\\p{") != T::npos || pattern.find("\\u{") != T::npos ||
+      pattern.find("\\u") != T::npos) {
+    return true;
+  }
+
+  for (const auto &ch : pattern) {
+    if (static_cast<unsigned char>(ch) > 127) {
+      return true;
+    }
+  }
+
+  if (pattern.find(".") != T::npos) {
+    return true;
+  }
+
+  return false;
+}
+
 /// @ingroup regex
 ///
 /// Compile a regular expression from a string. If the regular expression is
@@ -119,6 +147,32 @@ auto to_regex(const T &pattern) -> std::optional<Regex<T>> {
     return RegexTypeRange{minimum, maximum};
   }
 
+  if (requires_unicode(pattern)) {
+    T icu_compatible_pattern{pattern};
+    std::size_t pos{0};
+    while ((pos = icu_compatible_pattern.find("\\u{", pos)) != T::npos) {
+      icu_compatible_pattern.replace(pos, 3, "\\x{");
+      pos += 3;
+    }
+
+    UErrorCode status{U_ZERO_ERROR};
+    UParseError parse_error;
+    icu::UnicodeString icu_pattern{
+        icu::UnicodeString::fromUTF8(icu_compatible_pattern)};
+
+    auto *regex_pattern{icu::RegexPattern::compile(icu_pattern, UREGEX_DOTALL,
+                                                   parse_error, status)};
+
+    if (U_FAILURE(status) || regex_pattern == nullptr) {
+      if (regex_pattern != nullptr) {
+        delete regex_pattern;
+      }
+      return std::nullopt;
+    }
+
+    return std::shared_ptr<icu::RegexPattern>(regex_pattern);
+  }
+
   RegexTypeBoost<typename T::value_type> result{
       pattern,
       boost::regex::no_except |
@@ -194,6 +248,32 @@ auto matches(const Regex<T> &regex, const T &value) -> bool {
           value, *std::get_if<RegexTypeStd<typename T::value_type>>(&regex));
     case RegexIndex::Noop:
       return true;
+    case RegexIndex::ICU: {
+      const auto *icu_regex{std::get_if<RegexTypeICU>(&regex)};
+      if (!icu_regex || !(*icu_regex)) {
+        return false;
+      }
+
+      icu::UnicodeString icu_input{icu::UnicodeString::fromUTF8(value)};
+      UErrorCode status{U_ZERO_ERROR};
+      auto *matcher{(*icu_regex)->matcher(icu_input, status)};
+
+      if (U_FAILURE(status) || matcher == nullptr) {
+        if (matcher != nullptr) {
+          delete matcher;
+        }
+        return false;
+      }
+
+      auto result{matcher->find(status)};
+      delete matcher;
+
+      if (U_FAILURE(status)) {
+        return false;
+      }
+
+      return result;
+    }
   }
 
     // See https://en.cppreference.com/w/cpp/utility/unreachable

test/regex/regex_matches_test.cc

@@ -161,9 +161,6 @@ TEST(Regex_matches, match_false_6) {
 }
 
 TEST(Regex_matches, unicode_range_arabic_indic_digit) {
-  // U+0660-U+0669 are Arabic-Indic digits
-  // With /u flag: [\u0660-\u0669] would match any Arabic-Indic digit
-  // Without /u flag: treated as UTF-8 bytes, requires complex pattern
   const auto regex{
       sourcemeta::core::to_regex<std::string>("[\\u0660-\\u0669]")};
   EXPECT_TRUE(regex.has_value());
@@ -173,9 +170,6 @@ TEST(Regex_matches, unicode_range_arabic_indic_digit) {
 }
 
 TEST(Regex_matches, unicode_range_4byte_deseret) {
-  // U+10400-U+1044F are Deseret letters (4-byte UTF-8)
-  // With /u flag: [\u{10400}-\u{1044F}] would work
-  // Without /u flag: cannot express this range simply
   const auto regex{
       sourcemeta::core::to_regex<std::string>("[\\u{10400}-\\u{1044F}]")};
   EXPECT_TRUE(regex.has_value());
@@ -187,8 +181,6 @@ TEST(Regex_matches, unicode_range_4byte_deseret) {
 }
 
 TEST(Regex_matches, unicode_property_letter) {
-  // With /u flag and Unicode property escapes: \p{Letter} matches any letter
-  // This would drastically simplify XML Name validation
   const auto regex{sourcemeta::core::to_regex<std::string>(
       "^\\p{Letter}[\\p{Letter}\\p{Number}]*$")};
   EXPECT_TRUE(regex.has_value());
@@ -204,8 +196,6 @@ TEST(Regex_matches, unicode_property_letter) {
 }
 
 TEST(Regex_matches, unicode_property_exclude_digit) {
-  // With Unicode properties, we could exclude digits from start position
-  // while allowing them in subsequent positions - exactly what XML NCName needs
   const auto regex{sourcemeta::core::to_regex<std::string>(
       "^(?!\\p{Number})\\p{Letter}[\\p{Letter}\\p{Number}-_.]*$")};
   EXPECT_TRUE(regex.has_value());
@@ -220,8 +210,6 @@ TEST(Regex_matches, unicode_property_exclude_digit) {
 }
 
 TEST(Regex_matches, unicode_dot_matches_codepoint) {
-  // With /u flag, . matches one Unicode codepoint (including 4-byte chars)
-  // Without /u flag, . matches one byte
   const auto regex{sourcemeta::core::to_regex<std::string>("^.$")};
   EXPECT_TRUE(regex.has_value());
   EXPECT_TRUE(sourcemeta::core::matches<std::string>(regex.value(), "A"));
@@ -232,8 +220,6 @@ TEST(Regex_matches, unicode_dot_matches_codepoint) {
 }
 
 TEST(Regex_matches, unicode_quantifier_on_codepoints) {
-  // With /u flag, quantifiers work on Unicode codepoints
-  // Without /u flag, quantifiers work on bytes
   const auto regex{sourcemeta::core::to_regex<std::string>("^.{3}$")};
   EXPECT_TRUE(regex.has_value());
   EXPECT_TRUE(sourcemeta::core::matches<std::string>(regex.value(), "ABC"));
@@ -245,7 +231,6 @@ TEST(Regex_matches, unicode_quantifier_on_codepoints) {
 
 TEST(Regex_matches, digit_ascii_only) {
   // \d should only match ASCII digits 0-9, not Unicode digits
-  // From: https://github.com/json-schema-org/JSON-Schema-Test-Suite
   const auto regex{sourcemeta::core::to_regex<std::string>("^\\d$")};
   EXPECT_TRUE(regex.has_value());
   EXPECT_TRUE(sourcemeta::core::matches<std::string>(regex.value(), "0"));
@@ -289,14 +274,10 @@ TEST(Regex_matches, nonbmp_literal_match) {
 }
 
 TEST(Regex_matches, xml_ncname_simplified) {
-  // Current: 722KB exhaustive byte pattern
-  // With Unicode: ~50 bytes using property escapes
-  // NCName = letter (not colon) followed by letters/digits/punctuation
   const auto regex{sourcemeta::core::to_regex<std::string>(
       "^(?![:\\p{Nd}])[\\p{L}_][\\p{L}\\p{Nd}\\-._·]*$")};
   EXPECT_TRUE(regex.has_value());
 
-  // Valid NCNames
   EXPECT_TRUE(sourcemeta::core::matches<std::string>(regex.value(), "element"));
   EXPECT_TRUE(
       sourcemeta::core::matches<std::string>(regex.value(), "_element"));
@@ -307,8 +288,6 @@ TEST(Regex_matches, xml_ncname_simplified) {
       sourcemeta::core::matches<std::string>(regex.value(), "element٠"));
   EXPECT_TRUE(sourcemeta::core::matches<std::string>(regex.value(),
                                                      "\U00010400element"));
-
-  // Invalid NCNames
   EXPECT_FALSE(
       sourcemeta::core::matches<std::string>(regex.value(), ":element"));
   EXPECT_FALSE(