Custom fuzz options (#1077)

add mechanics for the fuzzer to add custom options.

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/tests.yml

@@ -390,3 +390,15 @@ jobs:
           cmake-version: "3.28.X"
           args: -DCLI11_SANITIZERS=ON -DCLI11_BUILD_EXAMPLES_JSON=ON
         if: success() || failure()
+
+      - name: Check CMake 3.29
+        uses: ./.github/actions/quick_cmake
+        with:
+          cmake-version: "3.29"
+        if: success() || failure()
+
+      - name: Check CMake 3.30
+        uses: ./.github/actions/quick_cmake
+        with:
+          cmake-version: "3.30"
+        if: success() || failure()

CMakeLists.txt

@@ -2,14 +2,14 @@ cmake_minimum_required(VERSION 3.5)
 # Note: this is a header only library. If you have an older CMake than 3.5,
 # just add the CLI11/include directory and that's all you need to do.
 
-# Make sure users don't get warnings on a tested (3.5 to 3.28) version
+# Make sure users don't get warnings on a tested (3.5 to 3.30) version
 # of CMake. For most of the policies, the new version is better (hence the change).
-# We don't use the 3.5...3.28 syntax because of a bug in an older MSVC's
+# We don't use the 3.5...3.30 syntax because of a bug in an older MSVC's
 # built-in and modified CMake 3.11
-if(${CMAKE_VERSION} VERSION_LESS 3.28)
+if(${CMAKE_VERSION} VERSION_LESS 3.30)
   cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
 else()
-  cmake_policy(VERSION 3.28)
+  cmake_policy(VERSION 3.30)
 endif()
 
 set(VERSION_REGEX "#define CLI11_VERSION[ \t]+\"(.+)\"")

fuzz/CMakeLists.txt

@@ -29,7 +29,7 @@ if(CMAKE_CXX_STANDARD GREATER 16)
       QUICK_CLI11_APP_FUZZ
       COMMAND ${CMAKE_COMMAND} -E make_directory corp
       COMMAND
-        cli11_app_fuzzer corp -max_total_time=${CLI11_FUZZ_TIME_APP} -max_len=2148
+        cli11_app_fuzzer corp -max_total_time=${CLI11_FUZZ_TIME_APP} -max_len=4096
         -dict=${CMAKE_CURRENT_SOURCE_DIR}/fuzz_dictionary1.txt
         -exact_artifact_path=${CLI11_FUZZ_ARTIFACT_PATH}/cli11_app_roundtrip_fail_artifact.txt)
 

fuzz/cli11_app_fuzz.cpp

@@ -15,35 +15,24 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
         return 0;
     }
     std::string parseString(reinterpret_cast<const char *>(Data), Size);
-    std::string optionString;
-    std::string flagString;
-    if(parseString.size() > 25) {
-        optionString = parseString.substr(0, 25);
-        parseString.erase(0, 25);
-    }
-    if(parseString.size() > 25) {
-        flagString = parseString.substr(0, 25);
-        parseString.erase(0, 25);
-    }
+
     CLI::FuzzApp fuzzdata;
     CLI::FuzzApp fuzzdata2;
     auto app = fuzzdata.generateApp();
     auto app2 = fuzzdata2.generateApp();
+    std::size_t pstring_start{0};
     try {
-        if(!optionString.empty()) {
-            app->add_option(optionString, fuzzdata.buffer);
-            app2->add_option(optionString, fuzzdata2.buffer);
-        }
-        if(!flagString.empty()) {
-            app->add_flag(flagString, fuzzdata.intbuffer);
-            app2->add_flag(flagString, fuzzdata2.intbuffer);
-        }
+        pstring_start = fuzzdata.add_custom_options(app.get(), parseString);
     } catch(const CLI::ConstructionError &e) {
         return 0;  // Non-zero return values are reserved for future use.
     }
 
     try {
-        app->parse(parseString);
+        if(pstring_start > 0) {
+            app->parse(parseString.substr(pstring_start, std::string::npos));
+        } else {
+            app->parse(parseString);
+        }
 
     } catch(const CLI::ParseError &e) {
         //(app)->exit(e);
@@ -54,6 +43,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
     std::string configOut = app->config_to_str();
     app->clear();
     std::stringstream out(configOut);
+    if(pstring_start > 0) {
+        fuzzdata2.add_custom_options(app2.get(), parseString);
+    }
     app2->parse_from_stream(out);
     auto result = fuzzdata2.compare(fuzzdata);
     if(!result) {

fuzz/fuzzApp.cpp

@@ -291,6 +291,82 @@ bool FuzzApp::compare(const FuzzApp &other) const {
     if(validator_strings != other.validator_strings) {
         return false;
     }
+    // now test custom string_options
+    if(custom_string_options.size() != other.custom_string_options.size()) {
+        return false;
+    }
+    for(std::size_t ii = 0; ii < custom_string_options.size(); ++ii) {
+        if(*custom_string_options[ii] != *other.custom_string_options[ii]) {
+            return false;
+        }
+    }
+    // now test custom vector_options
+    if(custom_vector_options.size() != other.custom_vector_options.size()) {
+        return false;
+    }
+    for(std::size_t ii = 0; ii < custom_vector_options.size(); ++ii) {
+        if(*custom_vector_options[ii] != *other.custom_vector_options[ii]) {
+            return false;
+        }
+    }
     return true;
 }
+
+//<option>name_string</option>
+//<vector>name_string</vector>
+//<flag>name_string</flag>
+/** generate additional options based on a string config*/
+std::size_t FuzzApp::add_custom_options(CLI::App *app, std::string &description_string) {
+    std::size_t current_index{0};
+    while(description_string.size() - 5 > current_index && description_string[current_index] == '<') {
+        if(description_string.compare(current_index, 7, "<option") == 0) {
+            auto end_option = description_string.find("</option>", current_index + 8);
+            if(end_option == std::string::npos) {
+                break;
+            }
+            auto header_close = description_string.find_last_of('>', end_option);
+            if(header_close == std::string::npos || header_close < current_index) {
+                break;
+            }
+            std::string name = description_string.substr(header_close + 1, end_option - header_close - 1);
+            custom_string_options.push_back(std::make_shared<std::string>());
+            app->add_option(name, *(custom_string_options.back()));
+            current_index = end_option + 9;
+        } else if(description_string.compare(current_index, 5, "<flag") == 0) {
+            auto end_option = description_string.find("</flag>", current_index + 6);
+            if(end_option == std::string::npos) {
+                break;
+            }
+            auto header_close = description_string.find_last_of('>', end_option);
+            if(header_close == std::string::npos || header_close < current_index) {
+                break;
+            }
+            std::string name = description_string.substr(header_close + 1, end_option - header_close - 1);
+            custom_string_options.push_back(std::make_shared<std::string>());
+            app->add_option(name, *(custom_string_options.back()));
+            current_index = end_option + 7;
+        } else if(description_string.compare(current_index, 7, "<vector") == 0) {
+            auto end_option = description_string.find("</vector>", current_index + 8);
+            if(end_option == std::string::npos) {
+                break;
+            }
+            auto header_close = description_string.find_last_of('>', end_option);
+            if(header_close == std::string::npos || header_close < current_index) {
+                break;
+            }
+            std::string name = description_string.substr(header_close + 1, end_option - header_close - 1);
+            custom_vector_options.push_back(std::make_shared<std::vector<std::string>>());
+            app->add_option(name, *(custom_string_options.back()));
+            current_index = end_option + 9;
+        } else {
+            if(isspace(description_string[current_index]) != 0) {
+                ++current_index;
+            } else {
+                break;
+            }
+        }
+    }
+    return current_index;
+}
+
 }  // namespace CLI

fuzz/fuzzApp.hpp

@@ -58,7 +58,8 @@ class FuzzApp {
     std::shared_ptr<CLI::App> generateApp();
     /** compare two fuzz apps for equality*/
     CLI11_NODISCARD bool compare(const FuzzApp &other) const;
-
+    /** generate additional options based on a string config*/
+    std::size_t add_custom_options(CLI::App *app, std::string &description_string);
     int32_t val32{0};
     int16_t val16{0};
     int8_t val8{0};
@@ -116,5 +117,7 @@ class FuzzApp {
     std::vector<std::string> vstrF{};
     std::string mergeBuffer{};
     std::vector<std::string> validator_strings{};
+    std::vector<std::shared_ptr<std::string>> custom_string_options{};
+    std::vector<std::shared_ptr<std::vector<std::string>>> custom_vector_options{};
 };
 }  // namespace CLI

fuzz/fuzz_dictionary1.txt

@@ -172,3 +172,14 @@
 "vS"
 "vM"
 "vE"
+"<option>"
+"</option>"
+"<vector>"
+"</vector>"
+"<option "
+"<vector "
+"<flag "
+"<flag>"
+"</flag>"
+">-"
+">--"

tests/FuzzFailTest.cpp

@@ -245,3 +245,16 @@ TEST_CASE("app_roundtrip_single") {
     */
     CHECK(result);
 }
+
+TEST_CASE("fuzz_config_test1") {
+    CLI::FuzzApp fuzzdata;
+    auto app = fuzzdata.generateApp();
+
+    std::string config_string = "<option>--new_option</option><flag>--new_flag</flag><vector>--new_vector</vector>";
+    auto loc = fuzzdata.add_custom_options(app.get(), config_string);
+    config_string = config_string.substr(loc);
+    CHECK(config_string.empty());
+    CHECK(app->get_option_no_throw("--new_option") != nullptr);
+    CHECK(app->get_option_no_throw("--new_flag") != nullptr);
+    CHECK(app->get_option_no_throw("--new_vector") != nullptr);
+}

tests/SubcommandTest.cpp

@@ -828,6 +828,22 @@ TEST_CASE_METHOD(TApp, "RequiredPosInSubcommand", "[subcom]") {
     CHECK_THROWS_AS(run(), CLI::RequiredError);
 }
 
+// from  https://github.com/CLIUtils/CLI11/issues/1002
+TEST_CASE_METHOD(TApp, "ForcedSubcommandExclude", "[subcom]") {
+    auto *subcommand_1 = app.add_subcommand("sub_1");
+    std::string forced;
+    subcommand_1->add_flag_function("-f", [&forced](bool f) { forced = f ? "got true" : "got false"; })
+        ->force_callback();
+
+    auto *subcommand_2 = app.add_subcommand("sub2");
+
+    subcommand_1->excludes(subcommand_2);
+
+    args = {"sub2"};
+    CHECK_NOTHROW(run());
+    CHECK(forced == "got false");
+}
+
 TEST_CASE_METHOD(TApp, "invalidSubcommandName", "[subcom]") {
 
     bool gotError{false};