feat: add pointer compare/subtract option + smart san support detection

aminya · Sep 11, 2024 · 886f87b · 886f87b
1 parent f9ba484
commit 886f87b
Show file tree

Hide file tree

Showing 6 changed files with 156 additions and 114 deletions.
diff --git a/README.md b/README.md
@@ -100,11 +100,8 @@ if(FEATURE_TESTS)
   set(ENABLE_CPPCHECK "ENABLE_CPPCHECK")
   set(ENABLE_COVERAGE "ENABLE_COVERAGE")
 
-  check_sanitizers_support(ENABLE_SANITIZER_ADDRESS
-                           ENABLE_SANITIZER_UNDEFINED_BEHAVIOR
-                           ENABLE_SANITIZER_LEAK
-                           ENABLE_SANITIZER_THREAD
-                           ENABLE_SANITIZER_MEMORY)
+  set(ENABLE_SANITIZER_ADDRESS "ENABLE_SANITIZER_ADDRESS")
+  set(ENABLE_SANITIZER_UNDEFINED_BEHAVIOR "ENABLE_SANITIZER_UNDEFINED_BEHAVIOR")
 endif()
 
 # Enable doxgen for the docs
@@ -129,6 +126,8 @@ project_options(
       ${ENABLE_SANITIZER_UNDEFINED_BEHAVIOR}
       # ${ENABLE_SANITIZER_THREAD}
       # ${ENABLE_SANITIZER_MEMORY}
+      # ENABLE_SANITIZER_POINTER_COMPARE
+      # ENABLE_SANITIZER_POINTER_SUBTRACT
       # ENABLE_CONTROL_FLOW_PROTECTION
       # ENABLE_STACK_PROTECTION
       # ENABLE_OVERFLOW_PROTECTION

diff --git a/docs/src/project_options_example.md b/docs/src/project_options_example.md
@@ -54,11 +54,8 @@ if(FEATURE_TESTS)
   set(ENABLE_CPPCHECK "ENABLE_CPPCHECK")
   set(ENABLE_COVERAGE "ENABLE_COVERAGE")
 
-  check_sanitizers_support(ENABLE_SANITIZER_ADDRESS
-                           ENABLE_SANITIZER_UNDEFINED_BEHAVIOR
-                           ENABLE_SANITIZER_LEAK
-                           ENABLE_SANITIZER_THREAD
-                           ENABLE_SANITIZER_MEMORY)
+  set(ENABLE_SANITIZER_ADDRESS "ENABLE_SANITIZER_ADDRESS")
+  set(ENABLE_SANITIZER_UNDEFINED_BEHAVIOR "ENABLE_SANITIZER_UNDEFINED_BEHAVIOR")
 endif()
 
 # Enable doxgen for the docs
@@ -83,6 +80,8 @@ project_options(
       ${ENABLE_SANITIZER_UNDEFINED_BEHAVIOR}
       # ${ENABLE_SANITIZER_THREAD}
       # ${ENABLE_SANITIZER_MEMORY}
+      # ENABLE_SANITIZER_POINTER_COMPARE
+      # ENABLE_SANITIZER_POINTER_SUBTRACT
       # ENABLE_CONTROL_FLOW_PROTECTION
       # ENABLE_STACK_PROTECTION
       # ENABLE_OVERFLOW_PROTECTION

diff --git a/src/DynamicProjectOptions.cmake b/src/DynamicProjectOptions.cmake
@@ -104,8 +104,13 @@ macro(dynamic_project_options)
   endif()
 
   check_sanitizers_support(
-    ENABLE_SANITIZER_ADDRESS ENABLE_SANITIZER_UNDEFINED_BEHAVIOR ENABLE_SANITIZER_LEAK
-    ENABLE_SANITIZER_THREAD ENABLE_SANITIZER_MEMORY
+    ENABLE_SANITIZER_ADDRESS
+    ENABLE_SANITIZER_UNDEFINED_BEHAVIOR
+    ENABLE_SANITIZER_LEAK
+    ENABLE_SANITIZER_THREAD
+    ENABLE_SANITIZER_MEMORY
+    ENABLE_SANITIZER_POINTER_COMPARE
+    ENABLE_SANITIZER_POINTER_SUBTRACT
   )
 
   if(ENABLE_SANITIZER_ADDRESS)

diff --git a/src/Sanitizers.cmake b/src/Sanitizers.cmake
@@ -11,89 +11,108 @@ function(
   ENABLE_SANITIZER_UNDEFINED_BEHAVIOR
   ENABLE_SANITIZER_THREAD
   ENABLE_SANITIZER_MEMORY
+  ENABLE_SANITIZER_POINTER_COMPARE
+  ENABLE_SANITIZER_POINTER_SUBTRACT
 )
 
-  if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES ".*Clang")
-    set(SANITIZERS "")
-
-    if(${ENABLE_SANITIZER_ADDRESS})
-      list(APPEND SANITIZERS "address")
-      if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 8)
-        list(APPEND SANITIZERS "pointer-compare" "pointer-subtract")
-        message(
-          STATUS
-            "To enable invalid pointer pairs detection, add detect_invalid_pointer_pairs=2 to the environment variable ASAN_OPTIONS."
-        )
+  # check if the sanitizers are supported
+  check_sanitizers_support(
+    SUPPORTS_SANITIZER_ADDRESS
+    SUPPORTS_SANITIZER_UNDEFINED_BEHAVIOR
+    SUPPORTS_SANITIZER_LEAK
+    SUPPORTS_SANITIZER_THREAD
+    SUPPORTS_SANITIZER_MEMORY
+    SUPPORTS_SANITIZER_POINTER_COMPARE
+    SUPPORTS_SANITIZER_POINTER_SUBTRACT
+  )
+
+  # for each sanitizer, check if it is supported and enabled
+  set(SANITIZERS "")
+  foreach(
+    SANITIZER IN
+    ITEMS "address"
+          "leak"
+          "undefined"
+          "thread"
+          "memory"
+          "pointer-compare"
+          "pointer-subtract"
+  )
+    if(${ENABLE_SANITIZER_${SANITIZER}})
+      if(${SUPPORTS_SANITIZER_${SANITIZER}})
+        list(APPEND SANITIZERS ${SANITIZER})
+      else()
+        # do not enable the sanitizer if it is not supported
+        message(STATUS "${SANITIZER} sanitizer is not supported. Not enabling it.")
       endif()
     endif()
+  endforeach()
 
-    if(${ENABLE_SANITIZER_LEAK})
-      list(APPEND SANITIZERS "leak")
-    endif()
+  # Info on special cases
 
-    if(${ENABLE_SANITIZER_UNDEFINED_BEHAVIOR})
-      list(APPEND SANITIZERS "undefined")
-    endif()
-
-    if(${ENABLE_SANITIZER_THREAD})
-      if("address" IN_LIST SANITIZERS OR "leak" IN_LIST SANITIZERS)
-        message(WARNING "Thread sanitizer does not work with Address and Leak sanitizer enabled")
-      else()
-        list(APPEND SANITIZERS "thread")
-      endif()
+  # Address sanitizer requires Leak sanitizer to be disabled
+  if(${ENABLE_SANITIZER_THREAD} AND ${SUPPORTS_SANITIZER_THREAD})
+    if("address" IN_LIST SANITIZERS OR "leak" IN_LIST SANITIZERS)
+      message(
+        WARNING
+          "Thread sanitizer does not work with Address or Leak sanitizer enabled. Disabling the thread sanitizer."
+      )
+      # remove thread sanitizer from the list
+      list(REMOVE_ITEM SANITIZERS "thread")
     endif()
+  endif()
 
-    if(${ENABLE_SANITIZER_MEMORY} AND CMAKE_CXX_COMPILER_ID MATCHES ".*Clang")
+  # Memory sanitizer requires all the code (including libc++) to be MSan-instrumented otherwise it reports false positives
+  if(${ENABLE_SANITIZER_MEMORY} AND ${SUPPORTS_SANITIZER_MEMORY} AND CMAKE_CXX_COMPILER_ID MATCHES ".*Clang")
+    message(
+      STATUS
+        "Memory sanitizer requires all the code (including libc++) to be MSan-instrumented otherwise it reports false positives"
+    )
+    if("address" IN_LIST SANITIZERS OR "thread" IN_LIST SANITIZERS OR "leak" IN_LIST SANITIZERS)
       message(
         WARNING
-          "Memory sanitizer requires all the code (including libc++) to be MSan-instrumented otherwise it reports false positives"
+          "Memory sanitizer does not work with Address, Thread and Leak sanitizer enabled. Disabling the memory sanitizer."
       )
-      if("address" IN_LIST SANITIZERS OR "thread" IN_LIST SANITIZERS OR "leak" IN_LIST SANITIZERS)
-        message(WARNING "Memory sanitizer does not work with Address, Thread and Leak sanitizer enabled")
-      else()
-        list(APPEND SANITIZERS "memory")
-      endif()
-    endif()
-  elseif(MSVC)
-    if(${ENABLE_SANITIZER_ADDRESS})
-      list(APPEND SANITIZERS "address")
+      # remove memory sanitizer from the list
+      list(REMOVE_ITEM SANITIZERS "memory")
     endif()
-    if(${ENABLE_SANITIZER_LEAK}
-       OR ${ENABLE_SANITIZER_UNDEFINED_BEHAVIOR}
-       OR ${ENABLE_SANITIZER_THREAD}
-       OR ${ENABLE_SANITIZER_MEMORY}
+  endif()
+
+  if((${ENABLE_SANITIZER_POINTER_COMPARE} AND ${SUPPORTS_SANITIZER_POINTER_COMPARE})
+     OR (${ENABLE_SANITIZER_POINTER_SUBTRACT} AND ${SUPPORTS_SANITIZER_POINTER_SUBTRACT})
+  )
+    message(
+      STATUS
+        "To enable invalid pointer pairs detection, add detect_invalid_pointer_pairs=2 to the environment variable ASAN_OPTIONS."
     )
-      message(WARNING "MSVC only supports address sanitizer")
-    endif()
   endif()
 
+  # Join the sanitizers
   list(JOIN SANITIZERS "," LIST_OF_SANITIZERS)
 
-  if(LIST_OF_SANITIZERS)
-    if(NOT "${LIST_OF_SANITIZERS}" STREQUAL "")
-      if(NOT MSVC)
-        target_compile_options(${_project_name} INTERFACE -fsanitize=${LIST_OF_SANITIZERS})
-        target_link_options(${_project_name} INTERFACE -fsanitize=${LIST_OF_SANITIZERS})
-      else()
-        string(FIND "$ENV{PATH}" "$ENV{VSINSTALLDIR}" index_of_vs_install_dir)
-        if("${index_of_vs_install_dir}" STREQUAL "-1")
-          message(
-            SEND_ERROR
-              "Using MSVC sanitizers requires setting the MSVC environment before building the project. Please manually open the MSVC command prompt and rebuild the project."
-          )
-        endif()
-        if(POLICY CMP0141)
-          if("${CMAKE_MSVC_DEBUG_INFORMATION_FORMAT}" STREQUAL "" OR "${CMAKE_MSVC_DEBUG_INFORMATION_FORMAT}"
-                                                                     STREQUAL "EditAndContinue"
-          )
-            set_target_properties(${_project_name} PROPERTIES MSVC_DEBUG_INFORMATION_FORMAT ProgramDatabase)
-          endif()
-        else()
-          target_compile_options(${_project_name} INTERFACE /Zi)
+  if(LIST_OF_SANITIZERS AND NOT "${LIST_OF_SANITIZERS}" STREQUAL "")
+    if(NOT MSVC)
+      target_compile_options(${_project_name} INTERFACE -fsanitize=${LIST_OF_SANITIZERS})
+      target_link_options(${_project_name} INTERFACE -fsanitize=${LIST_OF_SANITIZERS})
+    else()
+      string(FIND "$ENV{PATH}" "$ENV{VSINSTALLDIR}" index_of_vs_install_dir)
+      if("${index_of_vs_install_dir}" STREQUAL "-1")
+        message(
+          SEND_ERROR
+            "Using MSVC sanitizers requires setting the MSVC environment before building the project. Please manually open the MSVC command prompt and rebuild the project."
+        )
+      endif()
+      if(POLICY CMP0141)
+        if("${CMAKE_MSVC_DEBUG_INFORMATION_FORMAT}" STREQUAL "" OR "${CMAKE_MSVC_DEBUG_INFORMATION_FORMAT}"
+                                                                   STREQUAL "EditAndContinue"
+        )
+          set_target_properties(${_project_name} PROPERTIES MSVC_DEBUG_INFORMATION_FORMAT ProgramDatabase)
         endif()
-        target_compile_options(${_project_name} INTERFACE /fsanitize=${LIST_OF_SANITIZERS} /INCREMENTAL:NO)
-        target_link_options(${_project_name} INTERFACE /INCREMENTAL:NO)
+      else()
+        target_compile_options(${_project_name} INTERFACE /Zi)
       endif()
+      target_compile_options(${_project_name} INTERFACE /fsanitize=${LIST_OF_SANITIZERS} /INCREMENTAL:NO)
+      target_link_options(${_project_name} INTERFACE /INCREMENTAL:NO)
     endif()
   endif()
 
@@ -115,6 +134,8 @@ Output variables:
 - ``ENABLE_SANITIZER_LEAK``: Leak sanitizer is supported
 - ``ENABLE_SANITIZER_THREAD``: Thread sanitizer is supported
 - ``ENABLE_SANITIZER_MEMORY``: Memory sanitizer is supported
+- ``ENABLE_SANITIZER_POINTER_COMPARE``: Pointer compare sanitizer is supported
+- ``ENABLE_SANITIZER_POINTER_SUBTRACT``: Pointer subtract sanitizer is supported
 
 
 .. code:: cmake
@@ -123,7 +144,9 @@ Output variables:
                            ENABLE_SANITIZER_UNDEFINED_BEHAVIOR
                            ENABLE_SANITIZER_LEAK
                            ENABLE_SANITIZER_THREAD
-                           ENABLE_SANITIZER_MEMORY)
+                           ENABLE_SANITIZER_MEMORY
+                           ENABLE_SANITIZER_POINTER_COMPARE
+                           ENABLE_SANITIZER_POINTER_SUBTRACT)
 
   # then pass the sanitizers (e.g. ${ENABLE_SANITIZER_ADDRESS}) to project_options(... ${ENABLE_SANITIZER_ADDRESS} ...)
 
@@ -135,9 +158,13 @@ function(
   ENABLE_SANITIZER_LEAK
   ENABLE_SANITIZER_THREAD
   ENABLE_SANITIZER_MEMORY
+  ENABLE_SANITIZER_POINTER_COMPARE
+  ENABLE_SANITIZER_POINTER_SUBTRACT
 )
-  set(SANITIZERS "")
-  if(NOT "${CMAKE_SYSTEM_NAME}" STREQUAL "Windows")
+  set(SUPPORTED_SANITIZERS "")
+  if(NOT "${CMAKE_SYSTEM_NAME}" STREQUAL "Windows" AND (CMAKE_CXX_COMPILER_ID STREQUAL "GNU"
+                                                        OR CMAKE_CXX_COMPILER_ID MATCHES ".*Clang")
+  )
     set(HAS_SANITIZER_SUPPORT ON)
 
     # Disable gcc sanitizer on some macos according to https://github.com/orgs/Homebrew/discussions/3384#discussioncomment-6264292
@@ -154,39 +181,52 @@ function(
     endif()
 
     if(HAS_SANITIZER_SUPPORT)
-      list(APPEND SANITIZERS "address")
-      list(APPEND SANITIZERS "undefined")
-      list(APPEND SANITIZERS "leak")
-      list(APPEND SANITIZERS "thread")
-      list(APPEND SANITIZERS "memory")
+      list(APPEND SUPPORTED_SANITIZERS "address")
+      list(APPEND SUPPORTED_SANITIZERS "undefined")
+      list(APPEND SUPPORTED_SANITIZERS "leak")
+      list(APPEND SUPPORTED_SANITIZERS "thread")
+      list(APPEND SUPPORTED_SANITIZERS "memory")
+
+      if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 8)
+        list(APPEND SUPPORTED_SANITIZERS "pointer-compare")
+      endif()
+
+      if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 8)
+        list(APPEND SUPPORTED_SANITIZERS "pointer-subtract")
+      endif()
     endif()
   elseif(MSVC)
     # or it is MSVC and has run vcvarsall
     string(FIND "$ENV{PATH}" "$ENV{VSINSTALLDIR}" index_of_vs_install_dir)
     if(NOT "${index_of_vs_install_dir}" STREQUAL "-1")
-      list(APPEND SANITIZERS "address")
+      list(APPEND SUPPORTED_SANITIZERS "address")
     endif()
   endif()
 
-  list(JOIN SANITIZERS "," LIST_OF_SANITIZERS)
+  if(NOT SUPPORTED_SANITIZERS OR "${SUPPORTED_SANITIZERS}" STREQUAL "")
+    message(STATUS "No sanitizer is supported for the current platform/compiler")
+    return()
+  endif()
 
-  if(LIST_OF_SANITIZERS)
-    if(NOT "${LIST_OF_SANITIZERS}" STREQUAL "")
-      if("address" IN_LIST SANITIZERS)
-        set(${ENABLE_SANITIZER_ADDRESS} "ENABLE_SANITIZER_ADDRESS" PARENT_SCOPE)
-      endif()
-      if("undefined" IN_LIST SANITIZERS)
-        set(${ENABLE_SANITIZER_UNDEFINED_BEHAVIOR} "ENABLE_SANITIZER_UNDEFINED_BEHAVIOR" PARENT_SCOPE)
-      endif()
-      if("leak" IN_LIST SANITIZERS)
-        set(${ENABLE_SANITIZER_LEAK} "ENABLE_SANITIZER_LEAK" PARENT_SCOPE)
-      endif()
-      if("thread" IN_LIST SANITIZERS)
-        set(${ENABLE_SANITIZER_THREAD} "ENABLE_SANITIZER_THREAD" PARENT_SCOPE)
-      endif()
-      if("memory" IN_LIST SANITIZERS)
-        set(${ENABLE_SANITIZER_MEMORY} "ENABLE_SANITIZER_MEMORY" PARENT_SCOPE)
-      endif()
-    endif()
+  if("address" IN_LIST SUPPORTED_SANITIZERS)
+    set(${ENABLE_SANITIZER_ADDRESS} "ENABLE_SANITIZER_ADDRESS" PARENT_SCOPE)
+  endif()
+  if("pointer-compare" IN_LIST SUPPORTED_SANITIZERS)
+    set(${ENABLE_SANITIZER_POINTER_COMPARE} "ENABLE_SANITIZER_POINTER_COMPARE" PARENT_SCOPE)
+  endif()
+  if("pointer-subtract" IN_LIST SUPPORTED_SANITIZERS)
+    set(${ENABLE_SANITIZER_POINTER_SUBTRACT} "ENABLE_SANITIZER_POINTER_SUBTRACT" PARENT_SCOPE)
+  endif()
+  if("undefined" IN_LIST SUPPORTED_SANITIZERS)
+    set(${ENABLE_SANITIZER_UNDEFINED_BEHAVIOR} "ENABLE_SANITIZER_UNDEFINED_BEHAVIOR" PARENT_SCOPE)
+  endif()
+  if("leak" IN_LIST SUPPORTED_SANITIZERS)
+    set(${ENABLE_SANITIZER_LEAK} "ENABLE_SANITIZER_LEAK" PARENT_SCOPE)
+  endif()
+  if("thread" IN_LIST SUPPORTED_SANITIZERS)
+    set(${ENABLE_SANITIZER_THREAD} "ENABLE_SANITIZER_THREAD" PARENT_SCOPE)
+  endif()
+  if("memory" IN_LIST SUPPORTED_SANITIZERS)
+    set(${ENABLE_SANITIZER_MEMORY} "ENABLE_SANITIZER_MEMORY" PARENT_SCOPE)
   endif()
 endfunction()
diff --git a/tests/install/CMakeLists.txt b/tests/install/CMakeLists.txt
@@ -10,11 +10,8 @@ project(anotherproj VERSION 0.1.0 LANGUAGES CXX C)
 
 option(FEATURE_TESTS "Enable the tests" ON)
 if(FEATURE_TESTS)
-  # Enable sanitizers and static analyzers when running the tests
-  check_sanitizers_support(
-    ENABLE_SANITIZER_ADDRESS ENABLE_SANITIZER_UNDEFINED_BEHAVIOR ENABLE_SANITIZER_LEAK
-    ENABLE_SANITIZER_THREAD ENABLE_SANITIZER_MEMORY
-  )
+  set(ENABLE_SANITIZER_ADDRESS "ENABLE_SANITIZER_ADDRESS")
+  set(ENABLE_SANITIZER_UNDEFINED_BEHAVIOR "ENABLE_SANITIZER_UNDEFINED_BEHAVIOR")
 endif()
 
 # Initialize project_options
@@ -36,8 +33,8 @@ project_options(
   # ENABLE_BUILD_WITH_TIME_TRACE
   # ENABLE_UNITY
   ${ENABLE_SANITIZER_ADDRESS}
-  # ${ENABLE_SANITIZER_LEAK}
   ${ENABLE_SANITIZER_UNDEFINED_BEHAVIOR}
+  # ${ENABLE_SANITIZER_LEAK}
   # ${ENABLE_SANITIZER_THREAD}
   # ${ENABLE_SANITIZER_MEMORY}
 )

diff --git a/tests/myproj/CMakeLists.txt b/tests/myproj/CMakeLists.txt
@@ -37,10 +37,10 @@ option(FEATURE_TESTS "Enable the tests" ON)
 
 if(FEATURE_TESTS)
   # Enable sanitizers and static analyzers when running the tests
-  check_sanitizers_support(
-    ENABLE_SANITIZER_ADDRESS ENABLE_SANITIZER_UNDEFINED_BEHAVIOR ENABLE_SANITIZER_LEAK
-    ENABLE_SANITIZER_THREAD ENABLE_SANITIZER_MEMORY
-  )
+  set(ENABLE_SANITIZER_ADDRESS "ENABLE_SANITIZER_ADDRESS")
+  set(ENABLE_SANITIZER_UNDEFINED_BEHAVIOR "ENABLE_SANITIZER_UNDEFINED_BEHAVIOR")
+  set(ENABLE_SANITIZER_POINTER_COMPARE "ENABLE_SANITIZER_POINTER_COMPARE")
+  set(ENABLE_SANITIZER_POINTER_SUBTRACT "ENABLE_SANITIZER_POINTER_SUBTRACT")
 endif()
 
 # Detect custom linker
@@ -72,6 +72,8 @@ project_options(
   ${ENABLE_SANITIZER_ADDRESS}
   # ${ENABLE_SANITIZER_LEAK}
   ${ENABLE_SANITIZER_UNDEFINED_BEHAVIOR}
+  ${ENABLE_SANITIZER_POINTER_COMPARE}
+  ${ENABLE_SANITIZER_POINTER_SUBTRACT}
   # ${ENABLE_SANITIZER_THREAD}
   # ${ENABLE_SANITIZER_MEMORY}
   # CLANG_WARNINGS "-Weverything"