diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3dc6f6941..3c4565673 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -748,30 +748,103 @@ jobs: export LAZYLLM_K8S_CONFIG_PATH="/mnt/nfs_share/k8s_config.yaml" export LAZYLLM_HOME="${{ env.K8S_CI_PATH }}/${{ github.run_id }}-${{ github.job }}" mkdir -p $LAZYLLM_HOME - source /mnt/nfs_share/env.sh + source /mnt/nfs_share/env.sh pytest --lf --last-failed-no-failures=all --durations=0 --reruns=2 -v tests/k8s_tests cpp_ext_test: - name: C++ Extension Test (${{ matrix.os }}) + name: C++ Build + Python Regression (${{ matrix.os }}) needs: [ clone ] + if: always() runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] + timeout-minutes: 120 + defaults: + run: + shell: bash + env: + PYTHONNOUSERSITE: "1" + PYTHONPATH: ${{ github.workspace }} + LAZYLLM_ENABLE_CPP_OVERRIDE: "1" + LAZYLLM_EXPECTED_LOG_MODULES: "all" + LAZYLLM_DEFAULT_LAUNCHER: "empty" + LAZYLLM_OPENAI_API_KEY: ${{ secrets.LAZYLLM_OPENAI_API_KEY }} + LAZYLLM_KIMI_API_KEY: ${{ secrets.LAZYLLM_KIMI_API_KEY }} + LAZYLLM_AIPING_API_KEY: ${{ secrets.LAZYLLM_AIPING_API_KEY }} + LAZYLLM_GLM_API_KEY: ${{ secrets.LAZYLLM_GLM_API_KEY }} + LAZYLLM_GLM_MODEL_NAME: ${{ secrets.LAZYLLM_GLM_MODEL_NAME }} + LAZYLLM_QWEN_API_KEY: ${{ secrets.LAZYLLM_QWEN_API_KEY }} + LAZYLLM_QWEN_MODEL_NAME: ${{ secrets.LAZYLLM_QWEN_MODEL_NAME }} + LAZYLLM_QWEN_TEXT2IMAGE_MODEL_NAME: ${{ secrets.LAZYLLM_QWEN_TEXT2IMAGE_MODEL_NAME }} + LAZYLLM_SENSENOVA_API_KEY: ${{ secrets.LAZYLLM_SENSENOVA_API_KEY }} + LAZYLLM_SENSENOVA_SECRET_KEY: ${{ secrets.LAZYLLM_SENSENOVA_SECRET_KEY }} + LAZYLLM_DOUBAO_API_KEY: ${{ secrets.LAZYLLM_DOUBAO_API_KEY }} + LAZYLLM_DOUBAO_MODEL_NAME: ${{ secrets.LAZYLLM_DOUBAO_MODEL_NAME }} + LAZYLLM_SILICONFLOW_API_KEY: ${{ secrets.LAZYLLM_SILICONFLOW_API_KEY }} + LAZYLLM_SILICONFLOW_MODEL_NAME: ${{ secrets.LAZYLLM_SILICONFLOW_MODEL_NAME }} + LAZYLLM_MINIMAX_API_KEY: ${{ secrets.LAZYLLM_MINIMAX_API_KEY }} + LAZYLLM_MINIMAX_MODEL_NAME: ${{ secrets.LAZYLLM_MINIMAX_MODEL_NAME }} + LAZYLLM_PPOP_API_KEY: ${{ secrets.LAZYLLM_PPOP_API_KEY }} steps: - name: Checkout uses: actions/checkout@v4 with: - submodules: false + submodules: false - - name: Set up python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} + - name: Setup + uses: ./.github/actions/setup + + - name: Install test requirements + run: | + pip install -r tests/requirements.txt + if [[ "${{ runner.os }}" == "Linux" ]]; then + pip install -r tests/requirements_linux.txt + elif [[ "${{ runner.os }}" == "macOS" ]]; then + pip install -r tests/requirements_mac.txt + fi + + - name: Download test dataset + run: | + set -euo pipefail + DATA_DIR="$GITHUB_WORKSPACE/.ci_data" + rm -rf "$DATA_DIR" + export GIT_TERMINAL_PROMPT=0 + git clone --depth 1 https://$GITHUB_TOKEN@github.com/LazyAGI/LazyLLM-Data.git "$DATA_DIR" + echo "LAZYLLM_DATA_PATH=$DATA_DIR" >> "$GITHUB_ENV" + env: + GITHUB_TOKEN: ${{ secrets.PERSONAL_GITHUB_TOKEN || github.token }} - - name: Test + - name: Build and run C++ tests run: | pip install pybind11 bash csrc/scripts/build_test.sh + + - name: Install C++ extension artifacts into workspace + run: | + cmake --install build --prefix . --component lazyllm_cpp + ls -al lazyllm | rg "lazyllm_cpp|cpp_lib" || true + + - name: Run basic tests + run: | + if [[ "${{ runner.os }}" == "Linux" ]]; then + MARKER="not skip_on_linux" + elif [[ "${{ runner.os }}" == "macOS" ]]; then + MARKER="not skip_on_mac" + else + MARKER="not skip_on_win" + fi + pytest -v --order-scope=class -m "$MARKER" tests/basic_tests + + - name: Run advanced tests + run: | + if [[ "${{ runner.os }}" == "Linux" ]]; then + MARKER="not skip_on_linux" + elif [[ "${{ runner.os }}" == "macOS" ]]; then + MARKER="not skip_on_mac" + else + MARKER="not skip_on_win" + fi + pytest -v --order-scope=class -m "$MARKER" tests/advanced_tests diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml index 5d23326c4..ae7ecfb86 100644 --- a/.github/workflows/publish_release.yml +++ b/.github/workflows/publish_release.yml @@ -191,12 +191,6 @@ jobs: name: repo-with-docs path: ./repo_artifact - - name: Install Python dev headers (Ubuntu only) - if: startsWith(matrix.os, 'ubuntu') - run: | - sudo apt-get update - sudo apt-get install -y python3-dev - - name: Extract repo-with-docs run: | set -ex diff --git a/.gitignore b/.gitignore index 0e34cdd67..938aaaa6b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ test/ dist/ tmp/ build +.cache/ *.lock *.db mkdocs.yml @@ -64,3 +65,4 @@ docs/zh/assets build* lazyllm_cpp.egg-info/ !build*.sh +lazyllm/cpp_lib/ diff --git a/csrc/CMakeLists.txt b/csrc/CMakeLists.txt index 055fd5cbf..5906158aa 100644 --- a/csrc/CMakeLists.txt +++ b/csrc/CMakeLists.txt @@ -1,23 +1,81 @@ cmake_minimum_required(VERSION 3.16) project(LazyLLMCPP LANGUAGES CXX) -set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_POSITION_INDEPENDENT_CODE ON) -find_package(Python3 COMPONENTS Interpreter Development.Module REQUIRED) -find_package(pybind11 CONFIG REQUIRED) +function(lazyllm_enable_strict_warnings target_name) + if (MSVC) + target_compile_options(${target_name} PRIVATE /W4 /WX) + else () + target_compile_options(${target_name} PRIVATE -Werror -Wshadow) + endif () +endfunction() + +# Third party libs +include(cmake/third_party.cmake) # Config lazyllm_core lib with pure cpp code. -file(GLOB_RECURSE LAZYLLM_CORE_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp") +file(GLOB_RECURSE LAZYLLM_CORE_SOURCES CONFIGURE_DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/core/src/*.cpp") add_library(lazyllm_core STATIC ${LAZYLLM_CORE_SOURCES}) -target_include_directories(lazyllm_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) +target_include_directories(lazyllm_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/core/include) +target_link_libraries(lazyllm_core PUBLIC xxhash) +target_link_libraries(lazyllm_core PUBLIC tiktoken) +target_link_libraries(lazyllm_core PUBLIC utf8proc) +lazyllm_enable_strict_warnings(lazyllm_core) -# Config lazyllm_cpp lib with binding infomations. -set(LAZYLLM_BINDING_SOURCES binding/lazyllm.cpp binding/doc.cpp) +# Config lazyllm_cpp lib with binding informations. +file(GLOB_RECURSE LAZYLLM_BINDING_SOURCES CONFIGURE_DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/binding/*.cpp") set(INTERFACE_TARGET_NAME lazyllm_cpp) pybind11_add_module(${INTERFACE_TARGET_NAME} ${LAZYLLM_BINDING_SOURCES}) +target_include_directories(${INTERFACE_TARGET_NAME} PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/binding + ${CMAKE_CURRENT_SOURCE_DIR}/core/include +) target_link_libraries(${INTERFACE_TARGET_NAME} PRIVATE lazyllm_core) +lazyllm_enable_strict_warnings(${INTERFACE_TARGET_NAME}) + +# Runtime loader configuration per platform. +set(_lazyllm_cpp_rpath "") +set(LAZYLLM_TEST_RUNTIME_ENV "" CACHE INTERNAL "Runtime env for LazyLLM C++ tests" FORCE) +if (WIN32) + # Windows has no ELF rpath; loader resolution is driven by PATH and DLL search order. + # Keep test runtime env empty by default. +elseif (APPLE) + # Ensure lazyllm_cpp can find third-party dylibs under lazyllm/cpp_lib. + list(APPEND _lazyllm_cpp_rpath "@loader_path/cpp_lib") +else () + # Ensure lazyllm_cpp can find third-party shared libraries under lazyllm/cpp_lib. + list(APPEND _lazyllm_cpp_rpath "$ORIGIN/cpp_lib") + # Use DT_RPATH (instead of DT_RUNPATH) so the extension's own runtime + # search path can take precedence over host interpreter bundled libs. + target_link_options(${INTERFACE_TARGET_NAME} PRIVATE -Wl,--disable-new-dtags) + + # Resolve libstdc++ from the active C++ compiler and include it in rpath. + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} -print-file-name=libstdc++.so.6 + OUTPUT_VARIABLE LIBSTDCPP_PATH + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if (LIBSTDCPP_PATH AND NOT LIBSTDCPP_PATH STREQUAL "libstdc++.so.6") + get_filename_component(LIBSTDCPP_DIR "${LIBSTDCPP_PATH}" DIRECTORY) + if (LIBSTDCPP_DIR) + list(APPEND _lazyllm_cpp_rpath "${LIBSTDCPP_DIR}") + set(LAZYLLM_TEST_RUNTIME_ENV "LD_LIBRARY_PATH=${LIBSTDCPP_DIR}:$ENV{LD_LIBRARY_PATH}" + CACHE INTERNAL "Runtime env for LazyLLM C++ tests" FORCE) + endif () + endif () +endif () + +if (_lazyllm_cpp_rpath) + set_target_properties(${INTERFACE_TARGET_NAME} PROPERTIES + BUILD_RPATH "${_lazyllm_cpp_rpath}" + INSTALL_RPATH "${_lazyllm_cpp_rpath}" + ) +endif () if (CMAKE_BUILD_TYPE STREQUAL "Debug") # SHOW_SYMBOL @@ -26,7 +84,14 @@ if (CMAKE_BUILD_TYPE STREQUAL "Debug") endif() # Install -install(TARGETS ${INTERFACE_TARGET_NAME} LIBRARY DESTINATION lazyllm) +install(TARGETS ${INTERFACE_TARGET_NAME} + LIBRARY DESTINATION lazyllm COMPONENT lazyllm_cpp + RUNTIME DESTINATION lazyllm COMPONENT lazyllm_cpp +) +install(TARGETS tiktoken utf8proc + LIBRARY DESTINATION lazyllm/cpp_lib COMPONENT lazyllm_cpp + RUNTIME DESTINATION lazyllm/cpp_lib COMPONENT lazyllm_cpp +) # TESTS diff --git a/csrc/include/README.md b/csrc/README.md similarity index 100% rename from csrc/include/README.md rename to csrc/README.md diff --git a/csrc/binding/binding_utils.cpp b/csrc/binding/binding_utils.cpp new file mode 100644 index 000000000..112f5f5e1 --- /dev/null +++ b/csrc/binding/binding_utils.cpp @@ -0,0 +1,138 @@ +#include "binding_utils.hpp" + +#include + +namespace lazyllm::pybind_utils { + +std::string DumpJson(const py::object& obj) { + py::object json = py::module_::import("json"); + py::object dumps = json.attr("dumps"); + py::object dumped = dumps(obj, py::arg("ensure_ascii") = false); + return dumped.cast(); +} + +py::object LoadJson(const std::string& text) { + py::object json = py::module_::import("json"); + py::object loads = json.attr("loads"); + return loads(py::str(text)); +} + +bool ExtractStringSequence(const py::object& obj, std::vector* out) { + if (!py::isinstance(obj) || py::isinstance(obj)) return false; + py::sequence seq = obj.cast(); + out->clear(); + out->reserve(seq.size()); + for (py::handle item : seq) { + if (!py::isinstance(item)) { + out->clear(); + return false; + } + out->push_back(py::cast(item)); + } + return true; +} + +lazyllm::MetadataMode ParseMetadataMode(const py::object& mode) { + if (mode.is_none()) return lazyllm::MetadataMode::NONE; + try { + if (py::hasattr(mode, "name")) { + const auto name = py::cast(mode.attr("name")); + if (name == "ALL") return lazyllm::MetadataMode::ALL; + if (name == "EMBED") return lazyllm::MetadataMode::EMBED; + if (name == "LLM") return lazyllm::MetadataMode::LLM; + if (name == "NONE") return lazyllm::MetadataMode::NONE; + } + } catch (const py::error_already_set&) { + } + if (py::isinstance(mode)) { + const auto name = mode.cast(); + if (name == "ALL") return lazyllm::MetadataMode::ALL; + if (name == "EMBED") return lazyllm::MetadataMode::EMBED; + if (name == "LLM") return lazyllm::MetadataMode::LLM; + if (name == "NONE") return lazyllm::MetadataMode::NONE; + } + if (py::isinstance(mode)) { + const auto value = mode.cast(); + switch (value) { + case 0: return lazyllm::MetadataMode::ALL; + case 1: return lazyllm::MetadataMode::EMBED; + case 2: return lazyllm::MetadataMode::LLM; + case 3: return lazyllm::MetadataMode::NONE; + default: break; + } + } + return lazyllm::MetadataMode::NONE; +} + +lazyllm::MetadataVType PyToMetadataValue(const py::handle& value) { + if (value.is_none()) { + throw py::type_error( + "Unsupported metadata value type: None. " + "Only str/int/float and list[str]/list[int]/list[float] are allowed." + ); + } + if (py::isinstance(value)) return static_cast(value.cast()); + if (py::isinstance(value)) return value.cast(); + if (py::isinstance(value)) return value.cast(); + if (py::isinstance(value)) return value.cast(); + + if (py::isinstance(value) && !py::isinstance(value)) { + py::sequence seq = value.cast(); + if (seq.empty()) return std::vector{}; + + bool all_str = true; + bool all_int = true; + bool all_numeric = true; + + for (py::handle item : seq) { + const bool is_str = py::isinstance(item); + const bool is_int = py::isinstance(item) && !py::isinstance(item); + const bool is_numeric = is_int || py::isinstance(item) || py::isinstance(item); + all_str = all_str && is_str; + all_int = all_int && is_int; + all_numeric = all_numeric && is_numeric; + } + + if (all_str) { + std::vector out; + out.reserve(seq.size()); + for (py::handle item : seq) out.push_back(py::cast(item)); + return out; + } + if (all_int) { + std::vector out; + out.reserve(seq.size()); + for (py::handle item : seq) out.push_back(py::cast(item)); + return out; + } + if (all_numeric) { + std::vector out; + out.reserve(seq.size()); + for (py::handle item : seq) out.push_back(py::cast(item)); + return out; + } + throw py::type_error( + "Unsupported metadata sequence element type. " + "Only str/int/float (and homogeneous lists of them) are allowed." + ); + } + throw py::type_error( + "Unsupported metadata value type. " + "Only str/int/float and list[str]/list[int]/list[float] are allowed." + ); +} + +py::object MetadataValueToPy(const lazyllm::MetadataVType& value) { + return std::visit([](const auto& v) -> py::object { + using T = std::decay_t; + if constexpr (std::is_same_v) return py::str(v); + if constexpr (std::is_same_v) return py::int_(v); + if constexpr (std::is_same_v) return py::float_(v); + if constexpr (std::is_same_v>) return py::cast(v); + if constexpr (std::is_same_v>) return py::cast(v); + if constexpr (std::is_same_v>) return py::cast(v); + return py::none(); + }, value); +} + +} // namespace lazyllm::pybind_utils diff --git a/csrc/binding/binding_utils.hpp b/csrc/binding/binding_utils.hpp new file mode 100644 index 000000000..81e012261 --- /dev/null +++ b/csrc/binding/binding_utils.hpp @@ -0,0 +1,47 @@ +#pragma once + +#include +#include +#include +#include + +#include "lazyllm.hpp" +#include "doc_node.hpp" + +namespace lazyllm::pybind_utils { + +std::string DumpJson(const py::object& obj); +py::object LoadJson(const std::string& text); +bool ExtractStringSequence(const py::object& obj, std::vector* out); +lazyllm::MetadataMode ParseMetadataMode(const py::object& mode); +lazyllm::MetadataVType PyToMetadataValue(const py::handle& value); +py::object MetadataValueToPy(const lazyllm::MetadataVType& value); + +} // namespace lazyllm::pybind_utils + +namespace pybind11::detail { + +template <> +struct type_caster { +public: + PYBIND11_TYPE_CASTER(lazyllm::MetadataVType, _("MetadataVType")); + + bool load(handle src, bool) { + try { + value = lazyllm::pybind_utils::PyToMetadataValue(src); + return true; + } catch (const pybind11::error_already_set&) { + PyErr_Clear(); + return false; + } catch (...) { + return false; + } + } + + static handle cast(const lazyllm::MetadataVType& src, return_value_policy, handle) { + pybind11::object obj = lazyllm::pybind_utils::MetadataValueToPy(src); + return obj.release(); + } +}; + +} // namespace pybind11::detail diff --git a/csrc/binding/doc.cpp b/csrc/binding/export_add_doc_str.cpp similarity index 97% rename from csrc/binding/doc.cpp rename to csrc/binding/export_add_doc_str.cpp index 587315a2a..4ef0fd53b 100644 --- a/csrc/binding/doc.cpp +++ b/csrc/binding/export_add_doc_str.cpp @@ -28,6 +28,6 @@ void addDocStr(py::object obj, std::string docs) { } } -void exportDoc(py::module& m) { +void exportAddDocStr(py::module& m) { m.def("add_doc", &addDocStr, "Add docstring to a function or method", py::arg("obj"), py::arg("docs")); } diff --git a/csrc/binding/export_doc_node.cpp b/csrc/binding/export_doc_node.cpp new file mode 100644 index 000000000..c353e926b --- /dev/null +++ b/csrc/binding/export_doc_node.cpp @@ -0,0 +1,198 @@ +#include +#include +#include + +#include "binding_utils.hpp" +#include "doc_node.hpp" +#include "lazyllm.hpp" + +#include + +PYBIND11_MAKE_OPAQUE(lazyllm::DocNodeCore::Metadata); + +namespace { + +namespace pyu = lazyllm::pybind_utils; + +struct PyDocNodeCore : lazyllm::DocNodeCore { + using lazyllm::DocNodeCore::DocNodeCore; + + std::string get_metadata_string(lazyllm::MetadataMode mode) const override { + PYBIND11_OVERRIDE( + std::string, + lazyllm::DocNodeCore, + get_metadata_string, + mode + ); + } + + std::string get_text(lazyllm::MetadataMode mode) const override { + PYBIND11_OVERRIDE( + std::string, + lazyllm::DocNodeCore, + get_text, + mode + ); + } +}; + +lazyllm::DocNodeCore::Metadata MetadataFromPy(const py::object& obj) { + lazyllm::DocNodeCore::Metadata out; + if (obj.is_none()) return out; + py::dict d = py::dict(obj); + out.reserve(d.size()); + for (auto item : d) { + const std::string key = py::cast(item.first); + out.emplace(key, pyu::PyToMetadataValue(item.second)); + } + return out; +} + +std::set StringSetFromPy(const py::object& obj) { + std::set keys; + if (obj.is_none()) return keys; + for (auto item : obj) keys.insert(py::str(item).cast()); + return keys; +} + +lazyllm::MetadataMode ParseMode(const py::object& mode, lazyllm::MetadataMode default_mode) { + if (mode.is_none()) return default_mode; + return pyu::ParseMetadataMode(mode); +} + +} // namespace + +void exportDocNode(py::module& m) { + py::enum_(m, "MetadataMode") + .value("ALL", lazyllm::MetadataMode::ALL) + .value("EMBED", lazyllm::MetadataMode::EMBED) + .value("LLM", lazyllm::MetadataMode::LLM) + .value("NONE", lazyllm::MetadataMode::NONE); + + auto metadata_cls = py::bind_map(m, "MetadataMap"); + metadata_cls + .def("get", + [](const lazyllm::DocNodeCore::Metadata& self, const std::string& key, const py::object& default_value) { + auto it = self.find(key); + if (it == self.end()) return default_value; + return py::cast(it->second); + }, + py::arg("key"), py::arg("default") = py::none() + ) + .def("pop", + [](lazyllm::DocNodeCore::Metadata& self, const std::string& key) { + auto it = self.find(key); + if (it == self.end()) throw py::key_error(key); + py::object value = py::cast(it->second); + self.erase(it); + return value; + }, + py::arg("key") + ) + .def("pop", + [](lazyllm::DocNodeCore::Metadata& self, const std::string& key, const py::object& default_value) { + auto it = self.find(key); + if (it == self.end()) return default_value; + py::object value = py::cast(it->second); + self.erase(it); + return value; + }, + py::arg("key"), py::arg("default") + ) + .def("copy", + [](const lazyllm::DocNodeCore::Metadata& self) { + py::dict out; + for (const auto& [k, v] : self) out[py::str(k)] = py::cast(v); + return out; + } + ) + .def("update", + [](lazyllm::DocNodeCore::Metadata& self, const py::object& other) { + py::dict d = py::dict(other); + for (auto item : d) { + const std::string key = py::cast(item.first); + self[key] = pyu::PyToMetadataValue(item.second); + } + }, + py::arg("other") + ) + .def("__eq__", + [](const lazyllm::DocNodeCore::Metadata& self, const py::object& other) { + py::dict out; + for (const auto& [k, v] : self) out[py::str(k)] = py::cast(v); + const int cmp = PyObject_RichCompareBool(out.ptr(), other.ptr(), Py_EQ); + if (cmp < 0) throw py::error_already_set(); + return cmp == 1; + }, + py::is_operator() + ) + .def("__repr__", + [](const lazyllm::DocNodeCore::Metadata& self) { + py::dict out; + for (const auto& [k, v] : self) out[py::str(k)] = py::cast(v); + return py::repr(out).cast(); + } + ) + .def("__deepcopy__", + [](const lazyllm::DocNodeCore::Metadata& self, const py::dict&) { + py::dict out; + for (const auto& [k, v] : self) out[py::str(k)] = py::cast(v); + return out; + }, + py::arg("memo") + ); + + py::class_>( + m, "DocNodeCore", py::dynamic_attr() + ) + .def(py::init([](const py::object& text, const py::object& metadata, const py::object& uid) { + return std::make_shared( + text.is_none() ? std::string() : py::str(text).cast(), + MetadataFromPy(metadata), + uid.is_none() ? std::string() : py::cast(uid) + ); + }), + py::arg("text") = py::none(), + py::arg("metadata") = py::none(), + py::arg("uid") = py::none() + ) + .def_readwrite("_uid", &lazyllm::DocNodeCore::_uid) + .def_readwrite("_text", &lazyllm::DocNodeCore::_text) + .def_property("_metadata", + [](lazyllm::DocNodeCore& node) -> lazyllm::DocNodeCore::Metadata& { + return node._metadata; + }, + [](lazyllm::DocNodeCore& node, const py::object& metadata) { + node._metadata = MetadataFromPy(metadata); + }, + py::return_value_policy::reference_internal + ) + .def_property("_excluded_embed_metadata_keys", + [](const lazyllm::DocNodeCore& node) { + return std::vector( + node._excluded_embed_metadata_keys.begin(), + node._excluded_embed_metadata_keys.end() + ); + }, + [](lazyllm::DocNodeCore& node, const py::object& keys_obj) { + node._excluded_embed_metadata_keys = StringSetFromPy(keys_obj); + } + ) + .def_property("_excluded_llm_metadata_keys", + [](const lazyllm::DocNodeCore& node) { + return std::vector( + node._excluded_llm_metadata_keys.begin(), + node._excluded_llm_metadata_keys.end() + ); + }, + [](lazyllm::DocNodeCore& node, const py::object& keys_obj) { + node._excluded_llm_metadata_keys = StringSetFromPy(keys_obj); + } + ) + .def("get_metadata_str", [](const lazyllm::DocNodeCore& node, const py::object& mode) { + return node.get_metadata_string(ParseMode(mode, lazyllm::MetadataMode::ALL)); + }, py::arg("mode") = py::none()) + .def("get_text", [](const lazyllm::DocNodeCore& node, const py::object& metadata_mode) { + return node.get_text(ParseMode(metadata_mode, lazyllm::MetadataMode::NONE)); + }, py::arg("metadata_mode") = py::none()); +} diff --git a/csrc/binding/export_sentence_splitter.cpp b/csrc/binding/export_sentence_splitter.cpp new file mode 100644 index 000000000..32fb08650 --- /dev/null +++ b/csrc/binding/export_sentence_splitter.cpp @@ -0,0 +1,86 @@ +#include "lazyllm.hpp" + +#include "sentence_splitter.hpp" + +#include +#include + +#include +#include +#include +#include + +namespace { + +class SentenceSplitterCPPImpl : public lazyllm::SentenceSplitter { +public: + SentenceSplitterCPPImpl( + unsigned chunk_size, + unsigned chunk_overlap, + const std::string& encoding_name = "gpt2") + : lazyllm::SentenceSplitter(chunk_size, chunk_overlap, encoding_name) {} + + int chunk_size() const { return _chunk_size; } + void set_chunk_size(int value) { _chunk_size = value; } + + int overlap() const { return _overlap; } + void set_overlap(int value) { _overlap = value; } + + + std::vector merge_chunks_impl(py::list splits, int chunk_size) const { + std::vector owned; + owned.reserve(py::len(splits)); + for (auto item : splits) { + py::object split = py::reinterpret_borrow(item); + owned.push_back(lazyllm::Chunk{ + split.attr("text").cast(), + split.attr("is_sentence").cast(), + split.attr("token_size").cast() + }); + } + + std::vector chunks; + { + py::gil_scoped_release release; + chunks = lazyllm::SentenceSplitter::merge_chunks(owned, chunk_size); + } + return chunks; + } + + py::list split_text_impl(const std::string& text, int metadata_size) const { + std::vector chunks; + { + py::gil_scoped_release release; + chunks = lazyllm::SentenceSplitter::split_text(text, metadata_size); + } + + py::list out; + for (const auto& chunk : chunks) { + PyObject* decoded = PyUnicode_DecodeUTF8( + chunk.data(), + static_cast(chunk.size()), + "replace" + ); + if (decoded == nullptr) throw py::error_already_set(); + out.append(py::reinterpret_steal(decoded)); + } + return out; + } +}; + +} // namespace + +void exportSentenceSplitter(py::module& m) { + auto cls = py::class_(m, "SentenceSplitterCPPImpl", py::dynamic_attr()) + .def(py::init(), + py::arg("chunk_size") = 1024, + py::arg("chunk_overlap") = 200, + py::arg("encoding_name") = "gpt2" + ) + .def_property("_chunk_size", &SentenceSplitterCPPImpl::chunk_size, &SentenceSplitterCPPImpl::set_chunk_size) + .def_property("_overlap", &SentenceSplitterCPPImpl::overlap, &SentenceSplitterCPPImpl::set_overlap) + .def("split_text", &SentenceSplitterCPPImpl::split_text_impl, py::arg("text"), py::arg("metadata_size")) + .def("_merge", &SentenceSplitterCPPImpl::merge_chunks_impl, py::arg("splits"), py::arg("chunk_size")); + + (void)cls; +} diff --git a/csrc/binding/export_text_splitter_base.cpp b/csrc/binding/export_text_splitter_base.cpp new file mode 100644 index 000000000..8d3cbf8f1 --- /dev/null +++ b/csrc/binding/export_text_splitter_base.cpp @@ -0,0 +1,90 @@ +#include "lazyllm.hpp" + +#include "text_splitter_base.hpp" + +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +class TextSplitterBaseCPPImpl : public lazyllm::TextSplitterBase { +public: + TextSplitterBaseCPPImpl( + unsigned chunk_size, + unsigned overlap, + const std::string& encoding_name = "gpt2") + : lazyllm::TextSplitterBase( + static_cast(chunk_size), + static_cast(overlap), + encoding_name) {} + + int chunk_size() const { return _chunk_size; } + void set_chunk_size(int value) { _chunk_size = value; } + + int overlap() const { return _overlap; } + void set_overlap(int value) { _overlap = value; } + + + std::vector merge_chunks_impl(py::list splits, int chunk_size) const { + std::vector owned; + owned.reserve(py::len(splits)); + for (auto item : splits) { + py::object split = py::reinterpret_borrow(item); + owned.push_back(lazyllm::Chunk{ + split.attr("text").cast(), + split.attr("is_sentence").cast(), + split.attr("token_size").cast() + }); + } + + std::vector chunks; + { + py::gil_scoped_release release; + chunks = lazyllm::TextSplitterBase::merge_chunks(owned, chunk_size); + } + return chunks; + } + + py::list split_text_impl(const std::string& text, int metadata_size) const { + std::vector chunks; + { + py::gil_scoped_release release; + chunks = lazyllm::TextSplitterBase::split_text(text, metadata_size); + } + + py::list out; + for (const auto& chunk : chunks) { + PyObject* decoded = PyUnicode_DecodeUTF8( + chunk.data(), + static_cast(chunk.size()), + "replace" + ); + if (decoded == nullptr) throw py::error_already_set(); + out.append(py::reinterpret_steal(decoded)); + } + return out; + } +}; + +} // namespace + +void exportTextSplitterBase(py::module& m) { + auto cls = py::class_(m, "_TextSplitterBaseCPPImpl", py::dynamic_attr()) + .def(py::init(), + py::arg("chunk_size") = 1024, + py::arg("overlap") = 200, + py::arg("encoding_name") = "gpt2" + ) + .def_property("_chunk_size", &TextSplitterBaseCPPImpl::chunk_size, &TextSplitterBaseCPPImpl::set_chunk_size) + .def_property("_overlap", &TextSplitterBaseCPPImpl::overlap, &TextSplitterBaseCPPImpl::set_overlap) + .def("split_text", &TextSplitterBaseCPPImpl::split_text_impl, py::arg("text"), py::arg("metadata_size")) + .def("_merge", &TextSplitterBaseCPPImpl::merge_chunks_impl, py::arg("splits"), py::arg("chunk_size")); + + (void)cls; +} diff --git a/csrc/binding/lazyllm.cpp b/csrc/binding/lazyllm.cpp index 2789e8e1c..7f7a5156a 100644 --- a/csrc/binding/lazyllm.cpp +++ b/csrc/binding/lazyllm.cpp @@ -1,21 +1,21 @@ #include "lazyllm.hpp" -#include "doc_node.h" +#include "doc_node.hpp" + +#include namespace py = pybind11; PYBIND11_MODULE(lazyllm_cpp, m) { m.doc() = "LazyLLM CPP Module."; - exportDoc(m); + exportAddDocStr(m); - // prevent document generation + // Prevent document generation py::options options; options.disable_function_signatures(); - // DocNode - py::class_(m, "DocNode") - .def(py::init<>()) - .def(py::init(), py::arg("text")) - .def("set_text", &lazyllm::DocNode::set_text, py::arg("text")) - .def("get_text", &lazyllm::DocNode::get_text); + // Export classes + exportDocNode(m); + exportTextSplitterBase(m); + exportSentenceSplitter(m); } diff --git a/csrc/binding/lazyllm.hpp b/csrc/binding/lazyllm.hpp index d248f6677..df29afc88 100644 --- a/csrc/binding/lazyllm.hpp +++ b/csrc/binding/lazyllm.hpp @@ -1,6 +1,15 @@ #pragma once +#include +#include + +#include #include #include -void exportDoc(pybind11::module& m); +namespace py = pybind11; + +void exportAddDocStr(pybind11::module& m); +void exportDocNode(pybind11::module& m); +void exportTextSplitterBase(pybind11::module& m); +void exportSentenceSplitter(pybind11::module& m); diff --git a/csrc/binding/map_binding_helper.hpp b/csrc/binding/map_binding_helper.hpp new file mode 100644 index 000000000..83f00c9e7 --- /dev/null +++ b/csrc/binding/map_binding_helper.hpp @@ -0,0 +1,106 @@ +#pragma once + +#include +#include + +#include "lazyllm.hpp" + +namespace lazyllm::pybind_utils { + +template +void RegisterMapAsMutableMapping(ClassT& map_cls) { + py::object abc = py::module_::import("collections.abc"); + abc.attr("Mapping").attr("register")(map_cls); + abc.attr("MutableMapping").attr("register")(map_cls); +} + +template +void BindDictLikeMethods( + ClassT& map_cls, + ToPyValueFn to_py_value, + FromPyValueFn from_py_value, + ToPyDictFn to_py_dict, + py::object setdefault_default = py::none() +) { + map_cls + .def("get", + [to_py_value](MapType& self, const std::string& key, py::object default_value) { + auto it = self.find(key); + if (it == self.end()) return default_value; + return to_py_value(it->second); + }, + py::arg("key"), + py::arg("default") = py::none()) + .def("pop", + [to_py_value](MapType& self, const std::string& key) { + auto it = self.find(key); + if (it == self.end()) throw py::key_error(py::str(key)); + py::object value = to_py_value(it->second); + self.erase(it); + return value; + }, + py::arg("key")) + .def("pop", + [to_py_value](MapType& self, const std::string& key, py::object default_value) { + auto it = self.find(key); + if (it == self.end()) return default_value; + py::object value = to_py_value(it->second); + self.erase(it); + return value; + }, + py::arg("key"), + py::arg("default")) + .def("setdefault", + [to_py_value, from_py_value](MapType& self, const std::string& key, py::object default_value) { + auto it = self.find(key); + if (it != self.end()) return to_py_value(it->second); + auto [inserted, ok] = self.emplace(key, from_py_value(default_value)); + (void)ok; + return to_py_value(inserted->second); + }, + py::arg("key"), + py::arg("default") = setdefault_default) + .def("copy", + [to_py_dict](const MapType& self) { + return to_py_dict(self); + }) + .def("__copy__", + [to_py_dict](const MapType& self) { + return to_py_dict(self); + }) + .def("__deepcopy__", + [to_py_dict](const MapType& self, const py::dict& memo) { + py::object copy = py::module_::import("copy"); + return copy.attr("deepcopy")(to_py_dict(self), memo); + }, + py::arg("memo")) + .def("update", + [from_py_value](MapType& self, py::object other, py::kwargs kwargs) { + if (!other.is_none()) { + py::dict d = py::dict(other); + for (auto item : d) { + const std::string key = py::cast(item.first); + py::object value = py::reinterpret_borrow(item.second); + self[key] = from_py_value(value); + } + } + for (auto item : kwargs) { + const std::string key = py::cast(item.first); + py::object value = py::reinterpret_borrow(item.second); + self[key] = from_py_value(value); + } + }, + py::arg("other") = py::none()) + .def("__eq__", + [to_py_dict](const MapType& self, py::object other) { + py::dict lhs = to_py_dict(self); + if (py::isinstance(other) || py::hasattr(other, "items")) { + return py::bool_(lhs.equal(py::dict(other))); + } + return py::bool_(false); + }, + py::is_operator()); +} + +} // namespace lazyllm::pybind_utils + diff --git a/csrc/cmake/tests.cmake b/csrc/cmake/tests.cmake index 3eb114d9e..63e0f4e2e 100644 --- a/csrc/cmake/tests.cmake +++ b/csrc/cmake/tests.cmake @@ -2,7 +2,7 @@ include(FetchContent) FetchContent_Declare( googletest - URL https://codeload.github.com/google/googletest/zip/refs/tags/release-1.12.1 + URL https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip ) # Fix gtest version to maintain C++11 compatibility. set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) @@ -29,5 +29,15 @@ foreach (test_src ${LAZYLLM_TEST_SOURCES}) pybind11::headers Python3::Python ) - gtest_discover_tests(${test_name}) + gtest_add_tests( + TARGET ${test_name} + TEST_LIST discovered_tests + ) + + # Attach runtime env per discovered case so each test gets the same loader path. + if (LAZYLLM_TEST_RUNTIME_ENV AND discovered_tests) + set_tests_properties(${discovered_tests} PROPERTIES + ENVIRONMENT "${LAZYLLM_TEST_RUNTIME_ENV}" + ) + endif () endforeach () diff --git a/csrc/cmake/third_party.cmake b/csrc/cmake/third_party.cmake new file mode 100644 index 000000000..794f2d51e --- /dev/null +++ b/csrc/cmake/third_party.cmake @@ -0,0 +1,40 @@ +include(FetchContent) + +find_package(Python3 COMPONENTS Interpreter Development Development.Module REQUIRED) +find_package(pybind11 CONFIG REQUIRED) + +find_package(xxHash QUIET) +if (NOT TARGET xxhash) + FetchContent_Declare( + xxhash + GIT_REPOSITORY https://github.com/Cyan4973/xxHash.git + GIT_TAG v0.8.2 + ) + FetchContent_Populate(xxhash) + add_subdirectory(${xxhash_SOURCE_DIR}/cmake_unofficial ${xxhash_BINARY_DIR}) +endif() + +find_package(cpp_tiktoken QUIET) +if (NOT TARGET cpp_tiktoken) + # We only need cpp_tiktoken for in-tree usage; avoid exporting/installing it. + set(CPP_TIKTOKEN_INSTALL OFF CACHE BOOL "" FORCE) + set(CPP_TIKTOKEN_TESTING OFF CACHE BOOL "" FORCE) + FetchContent_Declare( + cpp_tiktoken + GIT_REPOSITORY https://github.com/gh-markt/cpp-tiktoken.git + GIT_TAG master + ) + FetchContent_MakeAvailable(cpp_tiktoken) +endif() + +find_package(utf8proc QUIET) +if (NOT TARGET utf8proc) + # We only need utf8proc for in-tree usage; avoid exporting/installing it. + set(UTF8PROC_INSTALL OFF CACHE BOOL "" FORCE) + FetchContent_Declare( + utf8proc + GIT_REPOSITORY https://github.com/JuliaStrings/utf8proc.git + GIT_TAG v2.9.0 + ) + FetchContent_MakeAvailable(utf8proc) +endif() diff --git a/csrc/core/include/adaptor_base.hpp b/csrc/core/include/adaptor_base.hpp new file mode 100644 index 000000000..602dcb2d3 --- /dev/null +++ b/csrc/core/include/adaptor_base.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include +#include +#include + +#if defined(__GNUC__) || defined(__clang__) +#define LAZYLLM_HIDDEN __attribute__((visibility("hidden"))) +#else +#define LAZYLLM_HIDDEN +#endif + +namespace lazyllm { + +class AdaptorBase { +public: + virtual ~AdaptorBase() = default; + virtual std::any call( + const std::string& func_name, + const std::unordered_map& args) const = 0; +}; + +} // namespace lazyllm diff --git a/csrc/core/include/doc_node.hpp b/csrc/core/include/doc_node.hpp new file mode 100644 index 000000000..fd905ebe6 --- /dev/null +++ b/csrc/core/include/doc_node.hpp @@ -0,0 +1,66 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "utils.hpp" + +namespace lazyllm { + +enum class MetadataMode { ALL, EMBED, LLM, NONE }; + +struct DocNodeCore { + using Metadata = std::unordered_map; + + Metadata _metadata; + std::string _text; + std::string _uid; + std::set _excluded_embed_metadata_keys; + std::set _excluded_llm_metadata_keys; + + explicit DocNodeCore( + const std::string& text, + const Metadata& metadata = {}, + const std::string& uid = "" + ) : _text(text), + _uid(uid.empty() ? GenerateUUID() : uid), + _metadata(metadata) {} + explicit DocNodeCore(const char* text, const Metadata& metadata = {}, const std::string& uid = "") + : DocNodeCore(std::string(text == nullptr ? "" : text), metadata, uid) {} + + DocNodeCore(const DocNodeCore&) = default; + DocNodeCore& operator=(const DocNodeCore&) = default; + virtual ~DocNodeCore() = default; + + virtual std::string get_metadata_string(MetadataMode mode = MetadataMode::ALL) const { + if (mode == MetadataMode::NONE) return ""; + + std::set valid_keys; + for (const auto& [key, _] : _metadata) valid_keys.insert(key); + + if (mode == MetadataMode::LLM) + valid_keys = SetDiff(valid_keys, _excluded_llm_metadata_keys); + else if (mode == MetadataMode::EMBED) + valid_keys = SetDiff(valid_keys, _excluded_embed_metadata_keys); + + std::vector kv_strings; + for (const std::string& key : valid_keys) + kv_strings.emplace_back(key + ": " + any_to_string(_metadata.at(key))); + + return JoinLines(kv_strings); + } + + virtual std::string get_text(MetadataMode mode = MetadataMode::NONE) const { + if (mode == MetadataMode::NONE) return _text; + const auto& metadata_string = get_metadata_string(mode); + if (metadata_string.empty()) return _text; + return metadata_string + "\n\n" + _text; + } +}; + +} // namespace lazyllm diff --git a/csrc/core/include/sentence_splitter.hpp b/csrc/core/include/sentence_splitter.hpp new file mode 100644 index 000000000..a1020fb15 --- /dev/null +++ b/csrc/core/include/sentence_splitter.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "text_splitter_base.hpp" + +namespace lazyllm { + +class SentenceSplitter : public TextSplitterBase { +public: + explicit SentenceSplitter( + unsigned chunk_size, + unsigned chunk_overlap, + const std::string& encoding_name = "gpt2") + : TextSplitterBase(chunk_size, chunk_overlap, encoding_name) {} + +protected: + std::vector merge_chunks(std::vector splits, int chunk_size) const override; +}; + +} // namespace lazyllm diff --git a/csrc/core/include/text_splitter_base.hpp b/csrc/core/include/text_splitter_base.hpp new file mode 100644 index 000000000..9ecc331ff --- /dev/null +++ b/csrc/core/include/text_splitter_base.hpp @@ -0,0 +1,52 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tokenizer.hpp" +#include "utils.hpp" + +namespace lazyllm { + +class TextSplitterBase { +public: + TextSplitterBase(unsigned chunk_size, unsigned overlap, const std::string& encoding_name = "gpt2") : + _chunk_size(chunk_size), + _overlap(overlap), + _tokenizer(std::make_shared(encoding_name)) {} + + std::vector split_text(const std::string_view& view, int metadata_size) const; + static std::vector split_text_while_keeping_separator( + const std::string_view& text, + const std::string_view& separator); + +protected: + virtual std::vector split_recursive(const std::string_view& view, const int chunk_size) const; + virtual std::vector merge_chunks(std::vector splits, int chunk_size) const; + +private: + std::tuple, bool> split_by_functions(const std::string_view& text) const; + + int get_token_size(const std::string_view& view) const { + if (view.empty()) return 0; + return static_cast(_tokenizer->encode(view).size()); + } + +protected: + std::shared_ptr _tokenizer = nullptr; + unsigned _overlap = 0; + unsigned _chunk_size = 0; +}; + +} // namespace lazyllm diff --git a/csrc/core/include/thread_pool.hpp b/csrc/core/include/thread_pool.hpp new file mode 100644 index 000000000..ca9c2a9c8 --- /dev/null +++ b/csrc/core/include/thread_pool.hpp @@ -0,0 +1,99 @@ +// https://github.com/progschj/ThreadPool + +#ifndef THREAD_POOL_H +#define THREAD_POOL_H +#include +#include +#include +#include +#include +#include +#include +#include +#include + +class ThreadPool { +public: + ThreadPool(size_t); + template + auto enqueue(F&& f, Args&&... args) + -> std::future::type>; + ~ThreadPool(); +private: + // need to keep track of threads so we can join them + std::vector< std::thread > workers; + // the task queue + std::queue< std::function > tasks; + + // synchronization + std::mutex queue_mutex; + std::condition_variable condition; + bool stop; +}; + +// the constructor just launches some amount of workers +inline ThreadPool::ThreadPool(size_t threads) + : stop(false) +{ + for(size_t i = 0;i task; + + { + std::unique_lock lock(this->queue_mutex); + this->condition.wait(lock, + [this]{ return this->stop || !this->tasks.empty(); }); + if(this->stop && this->tasks.empty()) + return; + task = std::move(this->tasks.front()); + this->tasks.pop(); + } + + task(); + } + } + ); +} + +// add new work item to the pool +template +auto ThreadPool::enqueue(F&& f, Args&&... args) + -> std::future::type> +{ + using return_type = typename std::result_of::type; + + auto task = std::make_shared< std::packaged_task >( + std::bind(std::forward(f), std::forward(args)...) + ); + + std::future res = task->get_future(); + { + std::unique_lock lock(queue_mutex); + + // don't allow enqueueing after stopping the pool + if(stop) + throw std::runtime_error("enqueue on stopped ThreadPool"); + + tasks.emplace([task](){ (*task)(); }); + } + condition.notify_one(); + return res; +} + +// the destructor joins all threads +inline ThreadPool::~ThreadPool() +{ + { + std::unique_lock lock(queue_mutex); + stop = true; + } + condition.notify_all(); + for(std::thread &worker: workers) + worker.join(); +} + +#endif diff --git a/csrc/core/include/tokenizer.hpp b/csrc/core/include/tokenizer.hpp new file mode 100644 index 000000000..a7f2aada8 --- /dev/null +++ b/csrc/core/include/tokenizer.hpp @@ -0,0 +1,157 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +class Tokenizer { +public: + virtual ~Tokenizer() = default; + virtual std::vector encode(const std::string_view& view) const = 0; + virtual std::string decode(const std::vector& token_ids) const = 0; +}; + +class FallbackByteTokenizer final : public Tokenizer { +public: + FallbackByteTokenizer() = default; + ~FallbackByteTokenizer() override = default; + + std::vector encode(const std::string_view& view) const override { + std::vector token_ids; + token_ids.reserve(view.size()); + for (unsigned char ch : view) { + token_ids.push_back(static_cast(ch)); + } + return token_ids; + } + + std::string decode(const std::vector& token_ids) const override { + std::string text; + text.reserve(token_ids.size()); + for (int id : token_ids) { + text.push_back(static_cast(id & 0xFF)); + } + return text; + } +}; + +class TiktokenTokenizer final : public Tokenizer { +public: + TiktokenTokenizer() = delete; + explicit TiktokenTokenizer(LanguageModel model) + : _encoding(load_encoding(model)) {} + + explicit TiktokenTokenizer(std::string_view encoding_name) + : TiktokenTokenizer(parse_tiktoken_model(encoding_name)) {} + + ~TiktokenTokenizer() override = default; + + std::vector encode(const std::string_view& view) const override { + return _encoding->encode(std::string(view)); // TODO refactor to string_view + } + + std::string decode(const std::vector& token_ids) const override { + return _encoding->decode(token_ids); + } + +private: + class FilePathResourceReader final : public IResourceReader { + public: + explicit FilePathResourceReader(std::filesystem::path resource_path) + : resource_path_(std::move(resource_path)) {} + + std::vector readLines() override { + std::ifstream file(resource_path_); + if (!file.is_open()) { + throw std::runtime_error("Embedded resource '" + resource_path_.string() + "' not found."); + } + std::string line; + std::vector lines; + while (std::getline(file, line)) lines.push_back(line); + return lines; + } + + private: + std::filesystem::path resource_path_; + }; + + static std::string resource_name(LanguageModel model) { + switch (model) { + case LanguageModel::R50K_BASE: return "r50k_base.tiktoken"; + case LanguageModel::P50K_BASE: return "p50k_base.tiktoken"; + case LanguageModel::P50K_EDIT: return "p50k_base.tiktoken"; + case LanguageModel::CL100K_BASE: return "cl100k_base.tiktoken"; + case LanguageModel::O200K_BASE: return "o200k_base.tiktoken"; + case LanguageModel::QWEN_BASE: return "qwen.tiktoken"; + } + throw std::runtime_error("Unknown language model"); + } + + static std::shared_ptr load_encoding(LanguageModel model) { + try { + return GptEncoding::get_encoding(model); + } catch (const std::exception&) { + const std::filesystem::path repo_root = + std::filesystem::path(__FILE__).parent_path().parent_path().parent_path().parent_path(); + const std::string file_name = resource_name(model); + const std::vector candidates = { + repo_root / "build" / "tokenizers" / file_name, + repo_root / "tokenizers" / file_name, + std::filesystem::current_path() / "build" / "tokenizers" / file_name, + std::filesystem::current_path() / "tokenizers" / file_name + }; + for (const auto& path : candidates) { + if (!std::filesystem::exists(path)) continue; + FilePathResourceReader reader(path); + return GptEncoding::get_encoding(model, &reader); + } + throw; + } + } + + static bool has_prefix(std::string_view value, std::string_view prefix) { + return value.size() >= prefix.size() && value.compare(0, prefix.size(), prefix) == 0; + } + + static LanguageModel parse_tiktoken_model(std::string_view name) { + if (name.empty()) return LanguageModel::R50K_BASE; + + // Model-name aliases used by Python tiktoken.encoding_for_model. + if (name == "gpt-3.5-turbo" || has_prefix(name, "gpt-3.5-turbo-")) return LanguageModel::CL100K_BASE; + if (name == "gpt-4" || has_prefix(name, "gpt-4-")) return LanguageModel::CL100K_BASE; + if (name == "text-embedding-ada-002") return LanguageModel::CL100K_BASE; + if (name == "text-embedding-3-small" || name == "text-embedding-3-large") return LanguageModel::CL100K_BASE; + if (name == "gpt-4o" || has_prefix(name, "gpt-4o-")) return LanguageModel::O200K_BASE; + if (name == "gpt-4.1" || has_prefix(name, "gpt-4.1-")) return LanguageModel::O200K_BASE; + if (name == "gpt-4.5" || has_prefix(name, "gpt-4.5-")) return LanguageModel::O200K_BASE; + if (name == "o1" || has_prefix(name, "o1-")) return LanguageModel::O200K_BASE; + if (name == "o3" || has_prefix(name, "o3-")) return LanguageModel::O200K_BASE; + if (name == "o4-mini" || has_prefix(name, "o4-mini-")) return LanguageModel::O200K_BASE; + + if (name == "gpt2" || name == "r50k_base" || name == "r50k") return LanguageModel::R50K_BASE; + if (name == "p50k_base" || name == "p50k") return LanguageModel::P50K_BASE; + if (name == "p50k_edit") return LanguageModel::P50K_EDIT; + if (name == "cl100k_base" || name == "cl100k") return LanguageModel::CL100K_BASE; + if (name == "o200k_base" || name == "o200k") return LanguageModel::O200K_BASE; + if (name == "qwen_base" || name == "qwen") return LanguageModel::QWEN_BASE; + + throw std::runtime_error( + "Unknown tiktoken encoding/model name: " + std::string(name) + + ". Expected one of: gpt2, r50k_base, p50k_base, p50k_edit, cl100k_base, o200k_base, qwen_base." + + "(Case sensitive)"); + } + +private: + std::shared_ptr _encoding; +}; diff --git a/csrc/core/include/unicode_processor.hpp b/csrc/core/include/unicode_processor.hpp new file mode 100644 index 000000000..4e5fbc125 --- /dev/null +++ b/csrc/core/include/unicode_processor.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace lazyllm { + +class UnicodeProcessor { +public: + explicit UnicodeProcessor(std::string_view text) : _text(text) {} + UnicodeProcessor(std::string&&) = delete; + UnicodeProcessor(const std::string&&) = delete; + + std::vector split_to_chars() const; + std::vector split_by_sentence_endings() const; + std::vector split_by_punctuation() const; + +private: + using Utf8Visitor = std::function; + + void for_each_utf8_unit(const Utf8Visitor& visitor) const; + static bool is_sentence_ending_punctuation(char32_t codepoint) { + return std::find(kSentenceEndingCodepoints.begin(), kSentenceEndingCodepoints.end(), + codepoint) != kSentenceEndingCodepoints.end(); + } + + static bool is_sub_sentence_punctuation(char32_t codepoint) { + return std::find(kSubSentencePunctuationCodepoints.begin(), kSubSentencePunctuationCodepoints.end(), + codepoint) != kSubSentencePunctuationCodepoints.end(); + } + + static const std::array kSentenceEndingCodepoints; + static const std::array kSubSentencePunctuationCodepoints; + std::string_view _text; +}; + +} // namespace lazyllm diff --git a/csrc/core/include/utils.hpp b/csrc/core/include/utils.hpp new file mode 100644 index 000000000..4084c5c29 --- /dev/null +++ b/csrc/core/include/utils.hpp @@ -0,0 +1,163 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace lazyllm { + +struct RAGMetadataKeys { + static constexpr std::string_view KB_ID = "kb_id"; + static constexpr std::string_view DOC_ID = "docid"; + static constexpr std::string_view DOC_PATH = "lazyllm_doc_path"; + static constexpr std::string_view DOC_FILE_NAME = "file_name"; + static constexpr std::string_view DOC_FILE_TYPE = "file_type"; + static constexpr std::string_view DOC_FILE_SIZE = "file_size"; + static constexpr std::string_view DOC_CREATION_DATE = "creation_date"; + static constexpr std::string_view DOC_LAST_MODIFIED_DATE = "last_modified_date"; + static constexpr std::string_view DOC_LAST_ACCESSED_DATE = "last_accessed_date"; +}; + +inline std::string JoinLines(const std::vector& lines, char delim = '\n') { + if (lines.empty()) return {}; + std::string out = lines.front(); + for (size_t i = 1; i < lines.size(); ++i) { + out += delim; + out += lines[i]; + } + return out; +} + +template +std::vector ConcatVector(const std::vector& l, const std::vector& r) { + std::vector out; + out.reserve(l.size() + r.size()); + out.insert(out.end(), l.begin(), l.end()); + out.insert(out.end(), r.begin(), r.end()); + return out; +} + +template +std::set SetUnion(const std::set& l, const std::set& r) { + std::set out; + std::set_union(l.begin(), l.end(), r.begin(), r.end(), std::inserter(out, out.begin())); + return out; +} + +template +std::set SetDiff(const std::set& l, const std::set& r) { + std::set out; + std::set_difference(l.begin(), l.end(), r.begin(), r.end(), std::inserter(out, out.begin())); + return out; +} + +inline std::string to_hex(size_t v) { + std::ostringstream oss; + oss << std::hex << v; + return oss.str(); +} + +inline std::string NumberToString(double v) { + std::ostringstream oss; + oss << v; + return oss.str(); +} + +inline std::string GenerateUUID() { + static const char HEX_CHAR[] = "0123456789abcdef"; + + // Single static generator per thread. + static thread_local std::mt19937 GEN(std::random_device{}()); + static thread_local std::uniform_int_distribution DIST(0, 255); + + std::array bytes{}; + for (auto& b : bytes) b = static_cast(DIST(GEN)); + + // RFC 4122 UUID v4: + // - Version: high 4 bits of byte 6 are 0100b. + // - Variant: high 2 bits of byte 8 are 10b. + bytes[6] = static_cast((bytes[6] & 0x0F) | 0x40); + bytes[8] = static_cast((bytes[8] & 0x3F) | 0x80); + + std::string out; + out.reserve(36); + for (size_t i = 0; i < bytes.size(); ++i) { + if (i == 4 || i == 6 || i == 8 || i == 10) out.push_back('-'); + out.push_back(HEX_CHAR[(bytes[i] >> 4) & 0x0F]); + out.push_back(HEX_CHAR[bytes[i] & 0x0F]); + } + return out; +} + +inline std::string VectorToString(const std::vector& values) { + if (values.empty()) return "[]"; + std::string out = "["; + for (size_t i = 0; i < values.size() - 1; ++i) { + out += values[i]; + out += ","; + } + out += values.back(); + out += "]"; + return out; +} + +inline std::string VectorToString(const std::vector& values) { + if (values.empty()) return "[]"; + std::string out = "["; + for (size_t i = 0; i < values.size() - 1; ++i) { + out += std::to_string(values[i]); + out += ","; + } + out += std::to_string(values.back()); + out += "]"; + return out; +} + +inline std::string VectorToString(const std::vector& values) { + if (values.empty()) return "[]"; + std::string out = "["; + for (size_t i = 0; i < values.size() - 1; ++i) { + out += NumberToString(values[i]); + out += ","; + } + out += NumberToString(values.back()); + out += "]"; + return out; +} +using MetadataVType = std::variant< + std::string, std::vector, + int, std::vector, + double, std::vector +>; +std::string any_to_string(const MetadataVType& value); + +inline bool is_adjacent(const std::string_view& left, const std::string_view& right) { + return left.data() + left.size() == right.data(); +} + +struct ChunkView { + std::string_view view; + bool is_sentence = false; + int token_size = 0; +}; + +struct Chunk { + std::string text; + bool is_sentence = false; + int token_size = 0; + + Chunk& operator+=(const Chunk& r) { + text += r.text; + is_sentence = is_sentence && r.is_sentence; + token_size += r.token_size; + return *this; + } +}; +} // namespace lazyllm diff --git a/csrc/core/src/doc_node.cpp b/csrc/core/src/doc_node.cpp new file mode 100644 index 000000000..031d608c5 --- /dev/null +++ b/csrc/core/src/doc_node.cpp @@ -0,0 +1 @@ +#include "doc_node.hpp" diff --git a/csrc/core/src/sentence_splitter.cpp b/csrc/core/src/sentence_splitter.cpp new file mode 100644 index 000000000..d2c1d5260 --- /dev/null +++ b/csrc/core/src/sentence_splitter.cpp @@ -0,0 +1,70 @@ +#include "sentence_splitter.hpp" + +#include +#include + +namespace lazyllm { + +std::string join_views( + size_t string_size, + std::vector::const_iterator begin, + const std::vector::const_iterator& end +) { + std::string out; + out.reserve(string_size); + while(begin != end) { + out.append(begin->text); + ++begin; + } + return out; +} + +std::vector SentenceSplitter::merge_chunks(std::vector chunks, int chunk_size) const { + std::vector out; + + auto iLeft = chunks.begin(); + auto iRight = chunks.begin(); + const auto& iEnd = chunks.end(); + int window_token_sum = 0; + size_t string_size = 0; + + while (iRight != iEnd) { + if (iRight->token_size > chunk_size) + throw std::runtime_error("Chunk size is too big."); + + // Grow right edge to the largest window under chunk_size. + while (iRight != iEnd && window_token_sum + iRight->token_size <= chunk_size) { + window_token_sum += iRight->token_size; + string_size += iRight->text.size(); + ++iRight; + } + + // Merge chunks witin window. + out.push_back(join_views(string_size, iLeft, iRight)); + + // Shrink left edge to select overlap of next merge. + while (iRight != iEnd && iLeft != iRight && ( + window_token_sum > _overlap || window_token_sum + iRight->token_size > chunk_size + )) { + window_token_sum -= iLeft->token_size; + string_size -= iLeft->text.size(); + ++iLeft; + } + // Now window contains only overlap. + } + + // Keep Python behavior: remove leading/trailing whitespace and drop empty chunks. + std::vector normalized; + normalized.reserve(out.size()); + for (auto& chunk : out) { + size_t begin = 0; + while (begin < chunk.size() && std::isspace(static_cast(chunk[begin]))) ++begin; + size_t end = chunk.size(); + while (end > begin && std::isspace(static_cast(chunk[end - 1]))) --end; + if (end > begin) normalized.emplace_back(chunk.substr(begin, end - begin)); + } + + return normalized; +} + +} // namespace lazyllm diff --git a/csrc/core/src/text_splitter_base.cpp b/csrc/core/src/text_splitter_base.cpp new file mode 100644 index 000000000..d90dc9a12 --- /dev/null +++ b/csrc/core/src/text_splitter_base.cpp @@ -0,0 +1,198 @@ +#include "text_splitter_base.hpp" +#include "unicode_processor.hpp" + +namespace lazyllm { + +/* + * split_text + * ---------- + * Purpose: + * 1) Validate chunk budget after accounting for metadata tokens. + * 2) Recursively split the original text view into token-bounded SplitUnit pieces. + * 3) Merge the pieces into final chunk strings with overlap behavior aligned to Python implementation. + * + * Flow: + * 1) Compute effective_chunk_size = chunk_size - metadata_size. + * 2) Reject invalid/too-small budgets. + * 3) Call split_recursive(...) to produce SplitUnit sequence. + * 4) Call merge_chunks(...) to build final std::string chunks. + * + * Notes: + * - This function returns std::string chunks intentionally because current tokenizer + * encode/decode materializes strings in the merge path. + * - Ownership is explicit here to avoid dangling string_view in downstream DocNodeCore construction. + * + * TODO: + * - After tokenizer supports true string_view encode/decode, migrate this path back to + * std::vector and remove eager string materialization. + */ +std::vector TextSplitterBase::split_text(const std::string_view& view, int metadata_size) const { + if (view.empty()) return {""}; + int effective_chunk_size = _chunk_size - metadata_size; + if (effective_chunk_size <= 0) { + throw std::invalid_argument( + "Metadata length (" + std::to_string(metadata_size) + + ") is longer than chunk size (" + std::to_string(_chunk_size) + + "). Consider increasing the chunk size or decreasing the size of your metadata to avoid this."); + } + else if (effective_chunk_size < 50) { + // Keep Python behavior: this is only a warning there, not an exception. + // We continue splitting with the small effective chunk size. + } + auto split_views = split_recursive(view, effective_chunk_size); + std::vector splits; + splits.reserve(split_views.size()); + for (const auto& split : split_views) { + splits.push_back(Chunk{std::string(split.view), split.is_sentence, split.token_size}); + } + return merge_chunks(std::move(splits), effective_chunk_size); +} + +std::vector TextSplitterBase::split_recursive( + const std::string_view& view, const int chunk_size) const +{ + int token_size = get_token_size(view); + if (token_size <= chunk_size) return {ChunkView{view, true, token_size}}; + + auto [views, is_sentence] = split_by_functions(view); + std::vector splits; + for (const auto& segment_view : views) { + const int seg_token_size = get_token_size(segment_view); + if (seg_token_size == 0) continue; + if (seg_token_size <= chunk_size) { + splits.push_back({segment_view, is_sentence, seg_token_size}); + } else { + auto new_splits = split_recursive(segment_view, chunk_size); + splits.insert(splits.end(), new_splits.begin(), new_splits.end()); + } + } + return splits; +} + +std::tuple, bool> TextSplitterBase::split_by_functions( + const std::string_view& text) const +{ + auto views = split_text_while_keeping_separator(text, "\n\n\n"); + if (views.size() > 1) return {views, true}; + + views = UnicodeProcessor(text).split_by_sentence_endings(); + if (views.size() > 1) return {views, true}; + + views = UnicodeProcessor(text).split_by_punctuation(); + if (views.size() > 1) return {views, false}; + + views = split_text_while_keeping_separator(text, " "); + if (views.size() > 1) return {views, false}; + + return {UnicodeProcessor(text).split_to_chars(), false}; +} + +std::vector TextSplitterBase::split_text_while_keeping_separator( + const std::string_view& text, + const std::string_view& separator) +{ + if (text.empty()) return {}; + else if (separator.empty()) return {text}; + + std::vector result; + size_t start = 0; + const size_t sep_len = separator.size(); + while (start < text.size()) { + const size_t idx = text.find(separator, start); + if (idx == std::string_view::npos) { + result.emplace_back(text.substr(start)); + break; + } + + if (idx == start) { + start += sep_len; + continue; + } + + result.emplace_back(text.substr(start, idx + sep_len - start)); + start = idx + sep_len; + } + return result; +} + +/** + * @brief Build final chunks from token-sized split units while preserving overlap semantics. + * + * @details + * 1) Convert input SplitUnit views to owned strings (MergedSplit) for safe concatenation. + * 2) If the tail split exactly matches chunk_size and overlap > 0: + * split it by token-halves via encode/decode, then push both halves back. + * 3) Iterate backward: + * Add previous split, or part of it, to current split as overlap. + * - If the previous split is small enough, prepend it fully. + * - Otherwise, prepend token-based overlap suffix from previous split. + * 4) Emit chunks in original order. + * + * @todo Replace eager string materialization once tokenizer encode/decode supports + * end-to-end zero-copy string_view operations. + */ +std::vector TextSplitterBase::merge_chunks(std::vector splits, int chunk_size) const +{ + if (splits.empty()) return {}; + + if (splits.size() == 1) return {splits.front().text}; + + if (splits.back().token_size == chunk_size && _overlap > 0) { + Chunk end_split = splits.back(); + splits.pop_back(); + + auto text_tokens = _tokenizer->encode(end_split.text); + const size_t half = text_tokens.size() / 2; + const auto split_it = text_tokens.begin() + static_cast::difference_type>(half); + std::vector prefix_tokens(text_tokens.begin(), split_it); + std::vector suffix_tokens(split_it, text_tokens.end()); + + std::string prefix_text = _tokenizer->decode(prefix_tokens); + std::string suffix_text = _tokenizer->decode(suffix_tokens); + splits.push_back( + Chunk{prefix_text, end_split.is_sentence, get_token_size(prefix_text)}); + splits.push_back( + Chunk{suffix_text, end_split.is_sentence, get_token_size(suffix_text)}); + } + + Chunk end_split = splits.back(); + std::vector reversed_result; + reversed_result.reserve(splits.size()); + for (int idx = static_cast(splits.size()) - 2; idx >= 0; --idx) { + const Chunk& start_split = splits[static_cast(idx)]; + if (start_split.token_size <= _overlap && end_split.token_size <= chunk_size - _overlap) { + end_split = Chunk{ + start_split.text + end_split.text, + start_split.is_sentence && end_split.is_sentence, + start_split.token_size + end_split.token_size + }; + continue; + } + + if (end_split.token_size > chunk_size) { + throw std::runtime_error("split token size is greater than chunk size."); + } + + const int remaining_space = chunk_size - end_split.token_size; + const int overlap_len = std::min({static_cast(_overlap), remaining_space, start_split.token_size}); + if (overlap_len > 0) { + auto start_tokens = _tokenizer->encode(start_split.text); + std::vector overlap_tokens(start_tokens.end() - overlap_len, start_tokens.end()); + std::string overlap_text = _tokenizer->decode(overlap_tokens); + + end_split = Chunk{ + overlap_text + end_split.text, + end_split.is_sentence, + end_split.token_size + overlap_len}; + } + + reversed_result.emplace_back(end_split.text); + end_split = start_split; + } + + reversed_result.emplace_back(end_split.text); + std::reverse(reversed_result.begin(), reversed_result.end()); + return reversed_result; +} + +} diff --git a/csrc/core/src/unicode_processor.cpp b/csrc/core/src/unicode_processor.cpp new file mode 100644 index 000000000..3993258fe --- /dev/null +++ b/csrc/core/src/unicode_processor.cpp @@ -0,0 +1,156 @@ +#include "unicode_processor.hpp" +namespace lazyllm { + +const std::array UnicodeProcessor::kSentenceEndingCodepoints = { + U'.', + U'!', + U'?', + U'\u3002', // CJK full stop + U'\uFF1F', // fullwidth question mark + U'\uFF01', // fullwidth exclamation mark +}; + +const std::array UnicodeProcessor::kSubSentencePunctuationCodepoints = { + U',', + U'.', + U';', + U'!', + U'?', + U'\uFF0C', // fullwidth comma + U'\uFF1B', // fullwidth semicolon + U'\u3002', // CJK full stop + U'\uFF1F', // fullwidth question mark + U'\uFF01', // fullwidth exclamation mark +}; + +void UnicodeProcessor::for_each_utf8_unit(const Utf8Visitor& visitor) const { + size_t i = 0; + auto text_size = _text.size(); + while (i < text_size) { + utf8proc_int32_t codepoint = -1; + const utf8proc_ssize_t n = utf8proc_iterate( + reinterpret_cast(_text.data() + i), + static_cast(text_size - i), + &codepoint); + + if (n <= 0) { + i += 1; + continue; + // TODO: when adding stronger logging, collect all invalid UTF-8 + // bytes encountered and report them together. + } + + visitor(i, static_cast(n), codepoint); + i += static_cast(n); + } +} + +/** + * UTF-8 text processing has three distinct layers: + * 1) Byte: the storage unit in std::string_view; one code point uses 1-4 UTF-8 bytes. + * 2) Code point: a Unicode scalar value (for example U+0061, U+4E2D), decoded by utf8proc_iterate. + * 3) Grapheme cluster: one user-perceived character, which may contain multiple code points + * (for example base + combining mark, or emoji + VS/ZWJ sequences). + * + * This function splits by grapheme cluster, not by byte or code point: + * - for_each_utf8_unit() uses utf8proc_iterate to decode UTF-8 and provide + * code point, byte offset, and byte length. + * - utf8proc_grapheme_break_stateful(prev, codepoint, &state) determines whether + * there is a grapheme boundary between prev and the current code point. + * - When a boundary appears, we emit a string_view slice over byte range + * [cluster_start, offset). + * + * This keeps splitting zero-copy (string_view) while following Unicode grapheme-boundary rules. + */ +std::vector UnicodeProcessor::split_to_chars() const { + std::vector out; + if (_text.empty()) return out; + out.reserve(_text.size()); // Grapheme count <= byte length + + size_t cluster_start = std::string_view::npos; + utf8proc_int32_t prev = -1; + utf8proc_int32_t state = 0; + + for_each_utf8_unit([&](size_t offset, size_t byte_len, utf8proc_int32_t codepoint) { + if (cluster_start == std::string_view::npos) { + cluster_start = offset; + } else if (utf8proc_grapheme_break_stateful(prev, codepoint, &state)) { + out.emplace_back(_text.substr(cluster_start, offset - cluster_start)); + cluster_start = offset; + } + prev = codepoint; + }); + + if (cluster_start != std::string_view::npos) { + out.emplace_back(_text.substr(cluster_start)); + } + return out; +} + +std::vector UnicodeProcessor::split_by_sentence_endings() const { + if (_text.empty()) return {}; + + std::vector out; + size_t chunk_start = std::string_view::npos; + bool trim_leading_space = true; + + for_each_utf8_unit([&](size_t offset, size_t byte_len, char32_t codepoint) { + const bool is_space = utf8proc_category(codepoint) == UTF8PROC_CATEGORY_ZS + || codepoint == U'\t' || codepoint == U'\n' || codepoint == U'\r' || codepoint == U'\f'; + if (chunk_start == std::string_view::npos && is_space && trim_leading_space) return; + + if (is_sentence_ending_punctuation(codepoint)) { + if (chunk_start != std::string_view::npos) { + const size_t end = offset + byte_len; + out.push_back(_text.substr(chunk_start, end - chunk_start)); + chunk_start = std::string_view::npos; + trim_leading_space = true; + } + } else if (chunk_start == std::string_view::npos) { + chunk_start = offset; + trim_leading_space = false; + } + }); + + if (chunk_start != std::string_view::npos) { + out.emplace_back(_text.substr(chunk_start)); + } + if (out.empty()) out.emplace_back(_text); + return out; +} + +std::vector UnicodeProcessor::split_by_punctuation() const { + if (_text.empty()) return {}; + + std::vector out; + size_t chunk_start = std::string_view::npos; + bool trim_leading_space = true; + + for_each_utf8_unit([&](size_t offset, size_t byte_len, char32_t codepoint) { + const bool is_space = utf8proc_category(codepoint) == UTF8PROC_CATEGORY_ZS + || codepoint == U'\t' || codepoint == U'\n' || codepoint == U'\r' || codepoint == U'\f'; + if (chunk_start == std::string_view::npos && is_space && trim_leading_space) return; + + if (is_sub_sentence_punctuation(codepoint)) { + if (chunk_start != std::string_view::npos) { + const size_t end = offset + byte_len; + out.push_back(_text.substr(chunk_start, end - chunk_start)); + chunk_start = std::string_view::npos; + trim_leading_space = ( + codepoint == U'.' || codepoint == U'!' || codepoint == U'\uFF1F' + || codepoint == U'\uFF01' || codepoint == U'?' || codepoint == U'\u3002'); + } + } else if (chunk_start == std::string_view::npos) { + chunk_start = offset; + trim_leading_space = false; + } + }); + + if (chunk_start != std::string_view::npos) { + out.emplace_back(_text.substr(chunk_start)); + } + if (out.empty()) out.emplace_back(_text); + return out; +} + +} // namespace lazyllm diff --git a/csrc/core/src/utils.cpp b/csrc/core/src/utils.cpp new file mode 100644 index 000000000..ce198b201 --- /dev/null +++ b/csrc/core/src/utils.cpp @@ -0,0 +1,22 @@ +#include "utils.hpp" + +#include +#include +#include + +namespace lazyllm { + +std::string any_to_string(const MetadataVType& value) { + return std::visit([](const auto& v) -> std::string { + using T = std::decay_t; + if constexpr (std::is_same_v) return v; + else if constexpr (std::is_same_v) return std::to_string(v); + else if constexpr (std::is_same_v) return NumberToString(v); + else if constexpr (std::is_same_v>) return VectorToString(v); + else if constexpr (std::is_same_v>) return VectorToString(v); + else if constexpr (std::is_same_v>) return VectorToString(v); + throw std::runtime_error(std::string("Unsupported Metadata value type: ") + typeid(T).name()); + }, value); +} + +} // namespace lazyllm diff --git a/csrc/include/doc_node.h b/csrc/include/doc_node.h deleted file mode 100644 index 4908d9941..000000000 --- a/csrc/include/doc_node.h +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -#include - -namespace lazyllm { - -class DocNode { -public: - DocNode() = default; - explicit DocNode(const std::string& text); - - void set_text(const std::string& text); - const std::string& get_text() const; - -private: - std::string _text; -}; - -} // namespace lazyllm diff --git a/csrc/scripts/build_debug.sh b/csrc/scripts/build_debug.sh index 08adee4c7..b7b358479 100644 --- a/csrc/scripts/build_debug.sh +++ b/csrc/scripts/build_debug.sh @@ -1,7 +1,15 @@ #!/usr/bin/env bash +# Run at LazyLLM/. set -euo pipefail cmake -S csrc -B build \ -Dpybind11_DIR="$(python -m pybind11 --cmakedir)" \ -DCMAKE_BUILD_TYPE=Debug cmake --build build + +# Install into ./lazyllm (local repo copy). +cmake --install build --prefix . --component lazyllm_cpp + +# Install into active Python site-packages (editable/venv runtime copy). +PY_PLATLIB="$(python -c 'import sysconfig; print(sysconfig.get_path("platlib"))')" +cmake --install build --prefix "$PY_PLATLIB" --component lazyllm_cpp diff --git a/csrc/scripts/build_release.sh b/csrc/scripts/build_release.sh index 865d22dc0..9be330f4d 100644 --- a/csrc/scripts/build_release.sh +++ b/csrc/scripts/build_release.sh @@ -1,7 +1,15 @@ #!/usr/bin/env bash +# Run at LazyLLM/. set -euo pipefail cmake -S csrc -B build-release \ -Dpybind11_DIR="$(python -m pybind11 --cmakedir)" \ -DCMAKE_BUILD_TYPE=Release cmake --build build-release + +# Install into ./lazyllm (local repo copy). +cmake --install build-release --prefix . --component lazyllm_cpp + +# Install into active Python site-packages (editable/venv runtime copy). +PY_PLATLIB="$(python -c 'import sysconfig; print(sysconfig.get_path("platlib"))')" +cmake --install build-release --prefix "$PY_PLATLIB" --component lazyllm_cpp diff --git a/csrc/scripts/build_test.sh b/csrc/scripts/build_test.sh index cd9336823..f571a4e0a 100644 --- a/csrc/scripts/build_test.sh +++ b/csrc/scripts/build_test.sh @@ -4,6 +4,7 @@ set -euo pipefail cmake -S csrc -B build \ -Dpybind11_DIR="$(python -m pybind11 --cmakedir)" \ -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DBUILD_TESTS=ON cmake --build build -ctest --test-dir build +ctest --test-dir build --rerun-failed --output-on-failure diff --git a/csrc/scripts/config_cmake.sh b/csrc/scripts/config_cmake.sh new file mode 100644 index 000000000..a02b81209 --- /dev/null +++ b/csrc/scripts/config_cmake.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +cmake -S csrc -B build \ + -Dpybind11_DIR="$(python -m pybind11 --cmakedir)" \ + -DCMAKE_BUILD_TYPE=Debug diff --git a/csrc/src/README.md b/csrc/src/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/csrc/src/doc_node.cpp b/csrc/src/doc_node.cpp deleted file mode 100644 index f774ec0e5..000000000 --- a/csrc/src/doc_node.cpp +++ /dev/null @@ -1,15 +0,0 @@ -#include "doc_node.h" - -namespace lazyllm { - -DocNode::DocNode(const std::string& text) : _text(text) {} - -void DocNode::set_text(const std::string& text) { - _text = text; -} - -const std::string& DocNode::get_text() const { - return _text; -} - -} // namespace lazyllm diff --git a/csrc/tests/test_adaptor_base.cpp b/csrc/tests/test_adaptor_base.cpp new file mode 100644 index 000000000..3a7cc10e8 --- /dev/null +++ b/csrc/tests/test_adaptor_base.cpp @@ -0,0 +1,37 @@ +#include + +#include +#include +#include + +#include "adaptor_base.hpp" + +namespace { + +class MockAdaptor final : public lazyllm::AdaptorBase { +public: + mutable int call_count = 0; + + std::any call( + const std::string& func_name, + const std::unordered_map& args) const override + { + ++call_count; + if (func_name == "sum") { + return std::any_cast(args.at("left")) + std::any_cast(args.at("right")); + } + return func_name; + } +}; + +} // namespace + +TEST(adaptor_base, derived_call) { + MockAdaptor adaptor; + auto result = adaptor.call("echo_me", {}); + EXPECT_EQ(std::any_cast(result), "echo_me"); + EXPECT_EQ(adaptor.call_count, 1); + + result = adaptor.call("sum", {{"left", 3}, {"right", 4}}); + EXPECT_EQ(std::any_cast(result), 7); +} diff --git a/csrc/tests/test_doc_node.cpp b/csrc/tests/test_doc_node.cpp index ff3dead26..4ffd8888e 100644 --- a/csrc/tests/test_doc_node.cpp +++ b/csrc/tests/test_doc_node.cpp @@ -1,16 +1,33 @@ #include -#include "doc_node.h" +#include -TEST(DocNode, DefaultEmpty) { - lazyllm::DocNode node; - EXPECT_EQ(node.get_text(), ""); +#include "doc_node.hpp" + +TEST(doc_node_core, constructor_sets_text) { + lazyllm::DocNodeCore node("hello", {}, "fixed-uid"); + EXPECT_EQ(node.get_text(lazyllm::MetadataMode::NONE), "hello"); + EXPECT_EQ(node._uid, "fixed-uid"); +} + +TEST(doc_node_core, constructor_generates_uid_when_empty) { + lazyllm::DocNodeCore node("hello"); + EXPECT_FALSE(node._uid.empty()); } -TEST(DocNode, SetGet) { - lazyllm::DocNode node("hello"); - EXPECT_EQ(node.get_text(), "hello"); +TEST(doc_node_core, metadata_string_and_text) { + lazyllm::DocNodeCore node("body"); + node._metadata = lazyllm::DocNodeCore::Metadata{ + {"alpha", std::string("A")}, + {"beta", std::string("B")}, + }; - node.set_text("world"); - EXPECT_EQ(node.get_text(), "world"); + EXPECT_EQ(node.get_metadata_string(lazyllm::MetadataMode::ALL), "alpha: A\nbeta: B"); + node._excluded_llm_metadata_keys = {"beta"}; + node._excluded_embed_metadata_keys = {"alpha"}; + EXPECT_EQ(node.get_metadata_string(lazyllm::MetadataMode::LLM), "alpha: A"); + EXPECT_EQ(node.get_metadata_string(lazyllm::MetadataMode::EMBED), "beta: B"); + EXPECT_EQ(node.get_metadata_string(lazyllm::MetadataMode::NONE), ""); + EXPECT_EQ(node.get_text(lazyllm::MetadataMode::NONE), "body"); + EXPECT_EQ(node.get_text(lazyllm::MetadataMode::ALL), "alpha: A\nbeta: B\n\nbody"); } diff --git a/csrc/tests/test_sentence_splitter.cpp b/csrc/tests/test_sentence_splitter.cpp new file mode 100644 index 000000000..60eeebb9a --- /dev/null +++ b/csrc/tests/test_sentence_splitter.cpp @@ -0,0 +1,96 @@ +#include + +#include +#include +#include + +#include "sentence_splitter.hpp" +#include "tokenizer.hpp" +#include "utils.hpp" + +namespace { + +class ByteTokenizer final : public Tokenizer { +public: + std::vector encode(const std::string_view& view) const override { + std::vector out; + out.reserve(view.size()); + for (unsigned char ch : view) out.push_back(static_cast(ch)); + return out; + } + + std::string decode(const std::vector& token_ids) const override { + std::string out; + out.reserve(token_ids.size()); + for (int token_id : token_ids) out.push_back(static_cast(token_id)); + return out; + } +}; + +class TestSentenceSplitter final : public lazyllm::SentenceSplitter { +public: + TestSentenceSplitter(unsigned chunk_size, unsigned overlap) + : lazyllm::SentenceSplitter(chunk_size, overlap, "gpt2") {} + + using lazyllm::SentenceSplitter::merge_chunks; + + void set_tokenizer_for_test(std::shared_ptr tokenizer) { + _tokenizer = std::move(tokenizer); + } +}; + +} // namespace + +TEST(sentence_splitter, merge_chunks_applies_overlap) { + TestSentenceSplitter splitter(5, 2); + + const std::vector splits{ + {"ab", false, 2}, + {"cd", false, 2}, + {"ef", false, 2}, + }; + + const auto merged = splitter.merge_chunks(splits, 5); + EXPECT_EQ(merged, (std::vector{"abcd", "cdef"})); +} + +TEST(sentence_splitter, merge_chunks_throws_on_oversized_single_split) { + TestSentenceSplitter splitter(3, 1); + const std::vector splits{ + {"abcd", false, 4}, + }; + + EXPECT_THROW((void)splitter.merge_chunks(splits, 3), std::runtime_error); +} + +TEST(sentence_splitter, merge_chunks_shrinks_overlap_to_fit_next_chunk) { + TestSentenceSplitter splitter(5, 4); + + const std::vector splits{ + {"aa", false, 2}, + {"b", false, 1}, + {"cccc", false, 4}, + {"dd", false, 2}, + {"ee", false, 2}, + }; + + const auto merged = splitter.merge_chunks(splits, 5); + EXPECT_EQ(merged, (std::vector{"aab", "bcccc", "ddee"})); +} + +TEST(sentence_splitter, split_text_empty_input_returns_single_empty_chunk) { + lazyllm::SentenceSplitter splitter(100, 10, "gpt2"); + const auto chunks = splitter.split_text("", 0); + ASSERT_EQ(chunks.size(), 1u); + EXPECT_EQ(chunks[0], ""); +} + +TEST(sentence_splitter, split_text_splits_large_text_with_byte_tokenizer) { + TestSentenceSplitter splitter(60, 0); + splitter.set_tokenizer_for_test(std::make_shared()); + + std::string text(130, 'x'); + const auto chunks = splitter.split_text(text, 0); + ASSERT_FALSE(chunks.empty()); + for (const auto& chunk : chunks) EXPECT_LE(chunk.size(), 60u); +} diff --git a/csrc/tests/test_smoke.cpp b/csrc/tests/test_smoke.cpp index 4c291b34b..998974c44 100644 --- a/csrc/tests/test_smoke.cpp +++ b/csrc/tests/test_smoke.cpp @@ -2,6 +2,6 @@ #include "lazyllm.hpp" -TEST(LazyLLM, Smoke) { +TEST(lazyllm, smoke) { EXPECT_GT(PYBIND11_VERSION_MAJOR, 0); } diff --git a/csrc/tests/test_text_splitter_base.cpp b/csrc/tests/test_text_splitter_base.cpp new file mode 100644 index 000000000..4284f9879 --- /dev/null +++ b/csrc/tests/test_text_splitter_base.cpp @@ -0,0 +1,126 @@ +#include + +#include +#include +#include +#include + +#include "text_splitter_base.hpp" +#include "utils.hpp" + +namespace { + +class ByteTokenizer final : public Tokenizer { +public: + std::vector encode(const std::string_view& view) const override { + std::vector out; + out.reserve(view.size()); + for (unsigned char ch : view) { + out.push_back(static_cast(ch)); + } + return out; + } + + std::string decode(const std::vector& token_ids) const override { + std::string out; + out.reserve(token_ids.size()); + for (int token_id : token_ids) { + out.push_back(static_cast(token_id)); + } + return out; + } +}; + +class TestTextSplitter final : public lazyllm::TextSplitterBase { +public: + TestTextSplitter(unsigned chunk_size, unsigned overlap = 0) + : lazyllm::TextSplitterBase(chunk_size, overlap, "gpt2") {} + + using lazyllm::TextSplitterBase::merge_chunks; + using lazyllm::TextSplitterBase::split_recursive; + + void set_tokenizer_for_test(std::shared_ptr tokenizer) { + _tokenizer = std::move(tokenizer); + } +}; + +} // namespace + +TEST(text_splitter_base, split_text_keep_separator_returns_segments) { + const auto parts = lazyllm::TextSplitterBase::split_text_while_keeping_separator("a--b--", "--"); + ASSERT_EQ(parts.size(), 2u); + EXPECT_EQ(parts[0], "a--"); + EXPECT_EQ(parts[1], "b--"); +} + +TEST(text_splitter_base, split_text_keep_separator_skips_leading_separator) { + const auto parts = lazyllm::TextSplitterBase::split_text_while_keeping_separator("--x", "--"); + ASSERT_EQ(parts.size(), 1u); + EXPECT_EQ(parts[0], "x"); +} + +TEST(text_splitter_base, split_text_throws_when_metadata_exceeds_chunk_size) { + lazyllm::TextSplitterBase splitter(60, 0); + EXPECT_THROW((void)splitter.split_text("abc", 60), std::invalid_argument); +} + +TEST(text_splitter_base, split_text_allows_small_metadata_budget) { + lazyllm::TextSplitterBase splitter(60, 0); + EXPECT_NO_THROW((void)splitter.split_text("abc", 11)); +} + +TEST(text_splitter_base, split_recursive_falls_back_to_char_level) { + TestTextSplitter splitter(100, 0); + splitter.set_tokenizer_for_test(std::make_shared()); + + const auto chunks = splitter.split_recursive("abc", 2); + ASSERT_EQ(chunks.size(), 3u); + EXPECT_EQ(chunks[0].view, "a"); + EXPECT_EQ(chunks[1].view, "b"); + EXPECT_EQ(chunks[2].view, "c"); + EXPECT_FALSE(chunks[0].is_sentence); +} + +TEST(text_splitter_base, merge_chunks_uses_overlap) { + TestTextSplitter splitter(100, 1); + splitter.set_tokenizer_for_test(std::make_shared()); + + const std::vector splits{ + {"ab", true, 2}, + {"cd", true, 2}, + {"ef", true, 2}, + }; + + const auto merged = splitter.merge_chunks(splits, 4); + EXPECT_EQ(merged, (std::vector{"ab", "bcd", "def"})); +} + +TEST(text_splitter_base, split_text_returns_single_empty_chunk_for_empty_input) { + lazyllm::TextSplitterBase splitter(100, 0); + const auto chunks = splitter.split_text("", 0); + ASSERT_EQ(chunks.size(), 1u); + EXPECT_EQ(chunks[0], ""); +} + +TEST(text_splitter_base, split_text_uses_current_definition_for_large_inputs) { + TestTextSplitter splitter(60, 0); + splitter.set_tokenizer_for_test(std::make_shared()); + + std::string text(120, 'a'); + const auto chunks = splitter.split_text(text, 0); + + ASSERT_FALSE(chunks.empty()); + for (const auto& chunk : chunks) EXPECT_LE(chunk.size(), 60u); +} + +TEST(text_splitter_base, merge_chunks_throws_on_oversized_end_split) { + TestTextSplitter splitter(100, 1); + splitter.set_tokenizer_for_test(std::make_shared()); + + const std::vector splits{ + {"a", true, 1}, + {"bbbb", true, 4}, + }; + + EXPECT_THROW((void)splitter.merge_chunks(splits, 3), std::runtime_error); +} diff --git a/csrc/tests/test_thread_pool.cpp b/csrc/tests/test_thread_pool.cpp new file mode 100644 index 000000000..b834fe63c --- /dev/null +++ b/csrc/tests/test_thread_pool.cpp @@ -0,0 +1,28 @@ +#include + +#include +#include + +#include "thread_pool.hpp" + +TEST(thread_pool, executes_tasks) { + ThreadPool pool(3); + + auto f1 = pool.enqueue([] { return 1 + 2; }); + EXPECT_EQ(f1.get(), 3); +} + +TEST(thread_pool, returns_values_from_futures) { + ThreadPool pool(3); + auto f2 = pool.enqueue([](int v) { return v * 2; }, 5); + EXPECT_EQ(f2.get(), 10); +} + +TEST(thread_pool, propagates_task_exception_through_future) { + ThreadPool pool(1); + auto failing = pool.enqueue([]() -> int { + throw std::runtime_error("boom"); + }); + + EXPECT_THROW((void)failing.get(), std::runtime_error); +} diff --git a/csrc/tests/test_tokenizer.cpp b/csrc/tests/test_tokenizer.cpp new file mode 100644 index 000000000..346aa4e39 --- /dev/null +++ b/csrc/tests/test_tokenizer.cpp @@ -0,0 +1,56 @@ +#include + +#include +#include +#include + +#include "tokenizer.hpp" + +namespace { + +class IdentityTokenizer final : public Tokenizer { +public: + std::vector encode(const std::string_view& view) const override { + std::vector out; + out.reserve(view.size()); + for (unsigned char ch : view) out.push_back(static_cast(ch)); + return out; + } + + std::string decode(const std::vector& token_ids) const override { + std::string out; + out.reserve(token_ids.size()); + for (int id : token_ids) out.push_back(static_cast(id)); + return out; + } +}; + +} // namespace + +TEST(tokenizer, abstract_interface_via_derived_class) { + std::unique_ptr tokenizer = std::make_unique(); + const auto ids = tokenizer->encode("abc"); + + EXPECT_EQ(ids, (std::vector{97, 98, 99})); + EXPECT_EQ(tokenizer->decode(ids), "abc"); +} + +TEST(tiktoken_tokenizer, round_trip_encoding) { + TiktokenTokenizer tokenizer("gpt2"); + const std::string text = "hello tokenizer"; + + const auto token_ids = tokenizer.encode(text); + EXPECT_FALSE(token_ids.empty()); + EXPECT_EQ(tokenizer.decode(token_ids), text); +} + +TEST(tiktoken_tokenizer, alias_names_map_to_same_encoding) { + TiktokenTokenizer gpt2("gpt2"); + TiktokenTokenizer r50k("r50k_base"); + + EXPECT_EQ(gpt2.encode("same input"), r50k.encode("same input")); +} + +TEST(tiktoken_tokenizer, unknown_encoding_throws) { + EXPECT_THROW((void)TiktokenTokenizer("unknown_model"), std::runtime_error); +} diff --git a/csrc/tests/test_unicode_processor.cpp b/csrc/tests/test_unicode_processor.cpp new file mode 100644 index 000000000..a59b03141 --- /dev/null +++ b/csrc/tests/test_unicode_processor.cpp @@ -0,0 +1,54 @@ +#include + +#include +#include + +#include "unicode_processor.hpp" + +TEST(unicode_processor, split_to_chars_supports_multibyte) { + const std::string text = "a你🙂"; + const lazyllm::UnicodeProcessor processor(text); + + const auto chars = processor.split_to_chars(); + EXPECT_EQ(chars, (std::vector{"a", "你", "🙂"})); +} + +TEST(unicode_processor, split_to_chars_empty_input_returns_empty_vector) { + const std::string text = ""; + const lazyllm::UnicodeProcessor processor(text); + + const auto chars = processor.split_to_chars(); + EXPECT_TRUE(chars.empty()); +} + +TEST(unicode_processor, split_by_punctuation_for_ascii) { + const std::string text = "Hello,world!"; + const lazyllm::UnicodeProcessor processor(text); + + const auto chunks = processor.split_by_punctuation(); + EXPECT_EQ(chunks, (std::vector{"Hello,", "world!"})); +} + +TEST(unicode_processor, split_by_punctuation_empty_input_returns_empty_vector) { + const std::string text = ""; + const lazyllm::UnicodeProcessor processor(text); + + const auto chunks = processor.split_by_punctuation(); + EXPECT_TRUE(chunks.empty()); +} + +TEST(unicode_processor, split_by_punctuation_for_cjk) { + const std::string text = "你好。世界！"; + const lazyllm::UnicodeProcessor processor(text); + + const auto chunks = processor.split_by_punctuation(); + EXPECT_EQ(chunks, (std::vector{"你好。", "世界！"})); +} + +TEST(unicode_processor, split_by_sentence_endings_for_ascii) { + const std::string text = "Hello, world! This is a test."; + const lazyllm::UnicodeProcessor processor(text); + + const auto chunks = processor.split_by_sentence_endings(); + EXPECT_EQ(chunks, (std::vector{"Hello, world!", "This is a test."})); +} diff --git a/csrc/tests/test_utils.cpp b/csrc/tests/test_utils.cpp new file mode 100644 index 000000000..8a4248e84 --- /dev/null +++ b/csrc/tests/test_utils.cpp @@ -0,0 +1,94 @@ +#include + +#include +#include +#include +#include + +#include "utils.hpp" + +TEST(utils, join_lines_returns_empty_for_empty_input) { + EXPECT_EQ(lazyllm::JoinLines({}), ""); +} + +TEST(utils, join_lines_uses_newline_separator_by_default) { + EXPECT_EQ(lazyllm::JoinLines({"a", "b", "c"}), "a\nb\nc"); +} + +TEST(utils, join_lines_supports_custom_delimiter) { + EXPECT_EQ(lazyllm::JoinLines({"a", "b", "c"}, ','), "a,b,c"); +} + +TEST(utils, concat_vector_appends_right_sequence) { + const auto merged = lazyllm::ConcatVector(std::vector{1, 2}, std::vector{3, 4}); + EXPECT_EQ(merged, (std::vector{1, 2, 3, 4})); +} + +TEST(utils, set_union_returns_all_unique_values) { + const std::set left{1, 2, 3}; + const std::set right{3, 4}; + EXPECT_EQ(lazyllm::SetUnion(left, right), (std::set{1, 2, 3, 4})); +} + +TEST(utils, set_diff_returns_only_left_unique_values) { + const std::set left{1, 2, 3}; + const std::set right{3, 4}; + EXPECT_EQ(lazyllm::SetDiff(left, right), (std::set{1, 2})); +} + +TEST(utils, to_hex_returns_lowercase_hex_text) { + EXPECT_EQ(lazyllm::to_hex(255u), "ff"); +} + +TEST(utils, generate_uuid_matches_expected_pattern) { + const std::string uuid = lazyllm::GenerateUUID(); + const std::regex pattern("^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"); + EXPECT_TRUE(std::regex_match(uuid, pattern)); +} + +TEST(utils, is_adjacent_returns_true_for_contiguous_views) { + const std::string text = "abcdef"; + const std::string_view left = std::string_view(text.data(), 3); + const std::string_view right = std::string_view(text.data() + 3, 3); + EXPECT_TRUE(lazyllm::is_adjacent(left, right)); +} + +TEST(utils, is_adjacent_returns_false_for_non_contiguous_views) { + const std::string text = "abcdef"; + const std::string_view left = std::string_view(text.data(), 3); + const std::string_view right = std::string_view(text.data() + 4, 2); + EXPECT_FALSE(lazyllm::is_adjacent(left, right)); +} + +TEST(utils, chunk_operator_plus_equals_accumulates_fields) { + lazyllm::Chunk l{"ab", true, 2}; + lazyllm::Chunk r{"cd", false, 3}; + + l += r; + EXPECT_EQ(l.text, "abcd"); + EXPECT_FALSE(l.is_sentence); + EXPECT_EQ(l.token_size, 5); +} + +TEST(utils, rag_metadata_keys_constants_are_exposed) { + EXPECT_EQ(lazyllm::RAGMetadataKeys::DOC_PATH, "lazyllm_doc_path"); + EXPECT_EQ(lazyllm::RAGMetadataKeys::DOC_ID, "docid"); +} + +TEST(utils, any_to_string_formats_scalar_metadata_values) { + EXPECT_EQ(lazyllm::any_to_string(lazyllm::MetadataVType(std::string("alpha"))), "alpha"); + EXPECT_EQ(lazyllm::any_to_string(lazyllm::MetadataVType(7)), "7"); + EXPECT_EQ(lazyllm::any_to_string(lazyllm::MetadataVType(3.5)), "3.5"); +} + +TEST(utils, any_to_string_formats_vector_metadata_values_with_brackets) { + EXPECT_EQ( + lazyllm::any_to_string(lazyllm::MetadataVType(std::vector{"a", "b"})), + "[a,b]"); + EXPECT_EQ( + lazyllm::any_to_string(lazyllm::MetadataVType(std::vector{1, 2, 3})), + "[1,2,3]"); + EXPECT_EQ( + lazyllm::any_to_string(lazyllm::MetadataVType(std::vector{1.5, 2.0})), + "[1.5,2]"); +} diff --git a/lazyllm/cpp.py b/lazyllm/cpp.py index 66e213d70..0c0f6b522 100644 --- a/lazyllm/cpp.py +++ b/lazyllm/cpp.py @@ -1,4 +1,223 @@ -try: - from .lazyllm_cpp import * # noqa F403 -except ImportError: - pass +import importlib +import inspect +import re +from functools import wraps +from itertools import combinations +from typing import Any, Dict, Optional, Tuple, TypeVar, cast + +from lazyllm import config + +config.add('cpp_switch', bool, False, 'ENABLE_CPP_OVERRIDE') + +_LAZYLLM_CPP_MODULE = None +_C = TypeVar('_C', bound=type) + + +def _load_cpp_module(): + global _LAZYLLM_CPP_MODULE + if _LAZYLLM_CPP_MODULE is None: + _LAZYLLM_CPP_MODULE = importlib.import_module('lazyllm.lazyllm_cpp') + return _LAZYLLM_CPP_MODULE + + +def _normalize_param_names(callable_obj: Any) -> Tuple[str, ...]: + signature = inspect.signature(callable_obj) + names = [] + for index, param in enumerate(signature.parameters.values()): + if index == 0 and param.name in ('self', 'cls'): + continue + names.append(param.name) + return tuple(names) + + +def _build_valid_kwargs(impl_cls: type, kwargs: Dict[str, Any]) -> Dict[str, Any]: + '''Keep only kwargs whose names exist in impl __init__ and whose types match exactly.''' + try: + signature = inspect.signature(impl_cls.__init__) + except (TypeError, ValueError): + return dict(kwargs) + + valid_params: Dict[str, Any] = {} + for name, value in kwargs.items(): + param = signature.parameters.get(name) + if param is None: + continue + + expected_type = param.annotation + if expected_type is inspect._empty or not isinstance(expected_type, type): + continue + + if type(value) is expected_type: + valid_params[name] = value + + return valid_params + + +def _instantiate_impl(impl_cls: type, kwargs: Dict[str, Any]): + '''Instantiate the C++ object; if signature is unavailable, try valid kwargs subsets dynamically.''' + candidate_kwargs = dict(kwargs) + while True: + try: + return impl_cls(**candidate_kwargs) + except TypeError as exc: + message = str(exc) + match = re.search(r"unexpected keyword argument '([^']+)'", message) + if not match: + break + bad_name = match.group(1) + if bad_name not in candidate_kwargs: + break + candidate_kwargs.pop(bad_name) + + # In some pybind build configurations, method signatures are not introspectable and + # only "incompatible constructor arguments" is reported. In that case, try subsets + # from larger to smaller to keep as many accepted kwargs as possible. + last_exc = None + keys = tuple(kwargs.keys()) + for size in range(len(keys), -1, -1): + for subset in combinations(keys, size): + subset_kwargs = {k: kwargs[k] for k in subset} + try: + return impl_cls(**subset_kwargs) + except TypeError as exc: + last_exc = exc + continue + + if last_exc is not None: + raise last_exc + raise TypeError(f'Failed to construct {impl_cls.__name__} with kwargs: {kwargs}') + + +def _scan_proxy_members(py_cls: type, impl_cls: type): + '''Scan exported impl members and collect same-name methods/properties for proxying.''' + proxy_methods = [] + proxy_attrs = [] + + for name, member in impl_cls.__dict__.items(): + if name.startswith('__'): + continue + + if isinstance(member, property): + proxy_attrs.append(name) + continue + + if not callable(member): + continue + if name not in py_cls.__dict__: + continue + + py_member = py_cls.__dict__[name] + if not callable(py_member): + continue + + # Dynamic validation: if both signatures are available, require identical + # parameter names; otherwise skip signature validation. + try: + py_sig = _normalize_param_names(py_member) + except (TypeError, ValueError): + py_sig = None + try: + impl_sig = _normalize_param_names(member) + except (TypeError, ValueError): + impl_sig = None + if py_sig is not None and impl_sig is not None and py_sig != impl_sig: + raise TypeError( + f'Signature mismatch for {py_cls.__name__}.{name}: ' + f'python params={py_sig}, cpp params={impl_sig}' + ) + + proxy_methods.append(name) + + return tuple(proxy_methods), tuple(proxy_attrs) + + +def cpp_class(py_class: Optional[_C] = None, *, cpp_class_name: Optional[str] = None): + def _decorate(cls: _C) -> _C: + if not isinstance(cls, type): + raise TypeError(f'@cpp_class can only decorate classes, got: {type(cls).__name__}') + + if not config.cpp_switch: + return cls + + cpp_module = _load_cpp_module() + export_name = cpp_class_name or cls.__name__ + cpp_export = getattr(cpp_module, export_name) + return cast(_C, cpp_export) + + if py_class is None: + return _decorate + return _decorate(py_class) + + +def cpp_proxy( + py_class: Optional[_C] = None, + *, + method_fallbacks: Optional[Dict[str, Tuple[str, ...]]] = None, + python_methods_for_self: Tuple[str, ...] = (), + cpp_class_name: Optional[str] = None, +): + def _decorate(cls: _C) -> _C: + if not isinstance(cls, type): + raise TypeError(f'@cpp_proxy can only decorate classes, got: {type(cls).__name__}') + + if not config.cpp_switch: + return cls + + cpp_module = _load_cpp_module() + impl_name = cpp_class_name or f'{cls.__name__}CPPImpl' + if not hasattr(cpp_module, impl_name): + raise AttributeError(f'@cpp_proxy cannot find C++ impl: {impl_name}') + + impl_cls = getattr(cpp_module, impl_name) + proxy_methods, proxy_attrs = _scan_proxy_members(cls, impl_cls) + fallback_rules = method_fallbacks or {} + force_python_methods = set(python_methods_for_self) + + impl_holder = '_c_obj' + original_init = cls.__init__ + + @wraps(original_init) + def _proxied_init(self, *args, **kwargs): + original_init(self, *args, **kwargs) + valid_params = _build_valid_kwargs(impl_cls, kwargs) + impl = _instantiate_impl(impl_cls, valid_params) + object.__setattr__(self, impl_holder, impl) + + cls.__init__ = _proxied_init + + def _make_method_proxy(method_name: str, original_method): + @wraps(original_method) + def _proxy(self, *args, **kwargs): + if type(self) is cls and method_name in force_python_methods: + return original_method(self, *args, **kwargs) + if type(self) is not cls: + deps = fallback_rules.get(method_name, ()) + if any(dep_name in type(self).__dict__ for dep_name in deps): + return original_method(self, *args, **kwargs) + + impl = getattr(self, impl_holder) + cpp_method = getattr(impl, method_name) + result = cpp_method(*args, **kwargs) + return self if result is impl else result + + return _proxy + + for method_name in proxy_methods: + original_method = getattr(cls, method_name) + setattr(cls, method_name, _make_method_proxy(method_name, original_method)) + + if proxy_attrs: + original_setattr = cls.__setattr__ + + def _proxied_setattr(self, name, value): + original_setattr(self, name, value) + if name in proxy_attrs and hasattr(self, impl_holder): + setattr(getattr(self, impl_holder), name, value) + + cls.__setattr__ = _proxied_setattr + + return cls + + if py_class is None: + return _decorate + return _decorate(py_class) diff --git a/lazyllm/tools/rag/doc_node.py b/lazyllm/tools/rag/doc_node.py index dd5a73537..15abfd58a 100644 --- a/lazyllm/tools/rag/doc_node.py +++ b/lazyllm/tools/rag/doc_node.py @@ -3,6 +3,7 @@ from collections import defaultdict from lazyllm.thirdparty import PIL from lazyllm import JsonFormatter, config, reset_on_pickle, Mode, LOG +from lazyllm.cpp import cpp_class from lazyllm.components.utils.file_operate import _image_to_base64 from .global_metadata import RAG_DOC_ID, RAG_DOC_PATH, RAG_KB_ID import uuid @@ -22,8 +23,47 @@ class MetadataMode(str, Enum): NONE = auto() +@cpp_class +class DocNodeCore: + def __init__(self, + text: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None, + uid: Optional[str] = None + ): + self._uid: str = uid or str(uuid.uuid4()) + self._text: str = text or '' + self._metadata: Dict[str, Any] = metadata or {} + + # Metadata keys that are excluded from text for the embed model. + self._excluded_embed_metadata_keys: List[str] = [] + # Metadata keys that are excluded from text for the LLM. + self._excluded_llm_metadata_keys: List[str] = [] + + def get_metadata_str(self, mode: MetadataMode = MetadataMode.ALL) -> str: + '''Metadata info string.''' + if mode == MetadataMode.NONE: + return '' + + metadata_keys = set(self._metadata.keys()) + if mode == MetadataMode.LLM: + for key in self.excluded_llm_metadata_keys: + if key in metadata_keys: + metadata_keys.remove(key) + elif mode == MetadataMode.EMBED: + for key in self.excluded_embed_metadata_keys: + if key in metadata_keys: + metadata_keys.remove(key) + + return '\n'.join([f'{key}: {self._metadata[key]}' for key in metadata_keys]) + + def get_text(self, metadata_mode: MetadataMode = MetadataMode.NONE) -> str: + metadata_str = self.get_metadata_str(metadata_mode).strip() + if not metadata_str: + return self.text if self.text else '' + return f'{metadata_str}\n\n{self.text}'.strip() + @reset_on_pickle(('_lock', threading.Lock)) -class DocNode: +class DocNode(DocNodeCore): def __init__(self, uid: Optional[str] = None, content: Optional[Union[str, List[Any]]] = None, group: Optional[str] = None, embedding: Optional[Dict[str, List[float]]] = None, parent: Optional[Union[str, 'DocNode']] = None, store=None, @@ -32,18 +72,12 @@ def __init__(self, uid: Optional[str] = None, content: Optional[Union[str, List[ if text and content: raise ValueError('`text` and `content` cannot be set at the same time.') if not content and not text: content = '' - self._uid: str = uid if uid else str(uuid.uuid4()) self._content: Optional[Union[str, List[Any]]] = content if content is not None else text + super().__init__(text=text, metadata=metadata, uid=uid) self._group: Optional[str] = group self._embedding: Optional[Dict[str, List[float]]] = embedding or {} - # metadata: the chunk's meta - self._metadata: Dict[str, Any] = metadata or {} # Global metadata: the file's global metadata (higher level) self._global_metadata = global_metadata or {} - # Metadata keys that are excluded from text for the embed model. - self._excluded_embed_metadata_keys: List[str] = [] - # Metadata keys that are excluded from text for the LLM. - self._excluded_llm_metadata_keys: List[str] = [] # NOTE: node in parent should be id when stored in db (use store to recover): parent: 'uid' self._parent: Optional[Union[str, 'DocNode']] = parent self._children: Dict[str, List['DocNode']] = defaultdict(list) @@ -51,7 +85,7 @@ def __init__(self, uid: Optional[str] = None, content: Optional[Union[str, List[ self._store = store self._node_groups: Dict[str, Dict] = node_groups or {} self._lock = threading.Lock() - self._embedding_state = set() + self.embedding_state = set() self.relevance_score = None self.similarity_score = None self._content_hash: Optional[str] = None @@ -220,6 +254,8 @@ def __str__(self) -> str: ) def __repr__(self) -> str: + if config.cpp_switch: + return '' return str(self) if config['mode'] == Mode.Debug else f'' def __eq__(self, other): @@ -257,48 +293,45 @@ def check_embedding_state(self, embed_key: str) -> None: while True: with self._lock: if not self.has_missing_embedding(embed_key): - self._embedding_state.discard(embed_key) + self.embedding_state.discard(embed_key) break time.sleep(1) def get_content(self) -> str: return self.get_text(MetadataMode.LLM) - def get_metadata_str(self, mode: MetadataMode = MetadataMode.ALL) -> str: - '''Metadata info string.''' - if mode == MetadataMode.NONE: - return '' - - metadata_keys = set(self.metadata.keys()) - if mode == MetadataMode.LLM: - for key in self.excluded_llm_metadata_keys: - if key in metadata_keys: - metadata_keys.remove(key) - elif mode == MetadataMode.EMBED: - for key in self.excluded_embed_metadata_keys: - if key in metadata_keys: - metadata_keys.remove(key) - - return '\n'.join([f'{key}: {self.metadata[key]}' for key in metadata_keys]) - - def get_text(self, metadata_mode: MetadataMode = MetadataMode.NONE) -> str: - metadata_str = self.get_metadata_str(metadata_mode).strip() - if not metadata_str: - return self.text if self.text else '' - return f'{metadata_str}\n\n{self.text}'.strip() - def to_dict(self) -> Dict: return dict(content=self._content, embedding=self.embedding, metadata=self.metadata) def copy(self, global_metadata: dict = None, metadata: dict = None, preserve_uid: bool = False) -> 'DocNode': - node = copy.copy(self) + if config.cpp_switch: + node = DocNode( + uid=self.uid if preserve_uid else None, + content=copy.deepcopy(self._content), + group=self._group, + embedding=copy.deepcopy(self._embedding), + parent=self._parent, + store=self._store, + node_groups=self._node_groups, + metadata=dict(self._metadata or {}), + global_metadata=dict(self._global_metadata or {}), + ) + node._children = copy.copy(self._children) + node._children_loaded = self._children_loaded + node.embedding_state = set(self.embedding_state) + node.relevance_score = self.relevance_score + node.similarity_score = self.similarity_score + node._content_hash = self._content_hash + else: + node = copy.copy(self) + if not preserve_uid: + node._uid = str(uuid.uuid4()) + node._metadata = dict(self._metadata or {}) + node._global_metadata = dict(self._global_metadata or {}) + node._copy_source = {'uid': self.uid, RAG_KB_ID: self.global_metadata.get(RAG_KB_ID), RAG_DOC_ID: self.global_metadata.get(RAG_DOC_ID)} - if not preserve_uid: - node._uid = str(uuid.uuid4()) - node._metadata = dict(self._metadata or {}) - node._global_metadata = dict(self._global_metadata or {}) if metadata: node._metadata.update(metadata) if global_metadata: @@ -367,7 +400,7 @@ def get_content(self, metadata_mode=MetadataMode.LLM) -> str: def image_path(self): return self._image_path - def get_text(self) -> str: # Disable access to self._content + def get_text(self, metadata_mode: MetadataMode = MetadataMode.NONE) -> str: # Disable access to self._content return self._image_path @property @@ -430,8 +463,8 @@ def _serialize_nodes(self) -> str: def _serialize_node(node: DocNode) -> str: formatted_node = { 'content': node.text, - 'metadata': node.metadata, - 'global_metadata': node.global_metadata, + 'metadata': dict(node.metadata), + 'global_metadata': dict(node.global_metadata), 'excluded_embed_metadata_keys': node.excluded_embed_metadata_keys, 'excluded_llm_metadata_keys': node.excluded_llm_metadata_keys, } diff --git a/lazyllm/tools/rag/store/document_store.py b/lazyllm/tools/rag/store/document_store.py index e1e9b4894..579f8a69f 100644 --- a/lazyllm/tools/rag/store/document_store.py +++ b/lazyllm/tools/rag/store/document_store.py @@ -383,8 +383,8 @@ def _serialize_node(self, node: DocNode) -> dict: doc_id=node.global_metadata.get(RAG_DOC_ID), group=node._group, content=node.text, - meta=node.metadata, - global_meta=node.global_metadata, + meta=dict(node.metadata), + global_meta=dict(node.global_metadata), number=node.metadata.get('lazyllm_store_num', 0), kb_id=node.global_metadata.get(RAG_KB_ID, DEFAULT_KB_ID), excluded_embed_metadata_keys=node.excluded_embed_metadata_keys, @@ -435,6 +435,7 @@ def _deserialize_node(self, data: dict, score: Optional[float] = None) -> DocNod node = DocNode(**common_parm, content=data.get('content', '')) node.excluded_embed_metadata_keys = data.get('excluded_embed_metadata_keys', []) node.excluded_llm_metadata_keys = data.get('excluded_llm_metadata_keys', []) + node._copy_source = data.get('copy_source', None) if 'embedding' in data: node.embedding = {k: v for k, v in data.get('embedding', {}).items()} return node.with_sim_score(score) if score else node diff --git a/lazyllm/tools/rag/transform/base.py b/lazyllm/tools/rag/transform/base.py index 67d64b8d9..8749ea912 100644 --- a/lazyllm/tools/rag/transform/base.py +++ b/lazyllm/tools/rag/transform/base.py @@ -16,6 +16,7 @@ import threading from lazyllm.thirdparty import tiktoken from lazyllm import config, ModuleBase +from lazyllm.cpp import cpp_proxy from pathlib import Path import inspect from lazyllm.thirdparty import nltk @@ -83,7 +84,7 @@ def _get_ref_nodes(self, node, ref_path): ) return current - def batch_forward( + def batch_forward( # noqa: C901 self, documents: Union[DocNode, List[DocNode]], node_group: str, ref_path: List[str] = None, **kwargs ) -> List[DocNode]: documents: List[DocNode] = documents if isinstance(documents, (tuple, list)) else [documents] @@ -115,7 +116,17 @@ def impl(node: DocNode): else: splits = self.forward(node, **kwargs) for s in splits: - s.parent = node + try: + s.parent = node + except Exception: + parent_uid = getattr(node, 'uid', None) + if not isinstance(parent_uid, str): + parent_uid = getattr(node, '_uid', None) + if isinstance(parent_uid, str): + try: + s.parent = parent_uid + except Exception: + pass s._group = node_group node.children[node_group] = splits return splits @@ -221,6 +232,10 @@ def _forward_single(self, node: Union[DocNode, RichDocNode], **kwargs: Any) -> L _UNSET = object() +@cpp_proxy(method_fallbacks={ + 'split_text': ('_split', '_merge', '_get_splits_by_fns'), + '_merge': ('_split',), +}) class _TextSplitterBase(NodeTransform): _default_params = {} _default_params_lock = threading.RLock() diff --git a/lazyllm/tools/rag/transform/sentence.py b/lazyllm/tools/rag/transform/sentence.py index 2f1729323..8657b0f03 100644 --- a/lazyllm/tools/rag/transform/sentence.py +++ b/lazyllm/tools/rag/transform/sentence.py @@ -1,6 +1,11 @@ from typing import List, Tuple from .base import _TextSplitterBase, _Split, _UNSET +from lazyllm.cpp import cpp_proxy +@cpp_proxy(method_fallbacks={ + 'split_text': ('_split', '_merge', '_get_splits_by_fns'), + '_merge': ('_split',), +}) class SentenceSplitter(_TextSplitterBase): def __init__(self, chunk_size: int = _UNSET, chunk_overlap: int = _UNSET, num_workers: int = _UNSET): super().__init__(chunk_size=chunk_size, overlap=chunk_overlap, num_workers=num_workers) diff --git a/lazyllm/tools/rag/utils.py b/lazyllm/tools/rag/utils.py index c6b0eebe9..a02cfccd0 100644 --- a/lazyllm/tools/rag/utils.py +++ b/lazyllm/tools/rag/utils.py @@ -871,8 +871,7 @@ def parallel_do_embedding(embed: Dict[str, Callable], embed_keys: Optional[Union modified_nodes.append(node) for k in miss: tasks_by_key[k].append(node) - if hasattr(node, '_embedding_state'): - node._embedding_state.add(k) + node.embedding_state.add(k) if not tasks_by_key: return [] @@ -907,9 +906,9 @@ def _process_key(k: str, knodes: List[DocNode]): except Exception as e: lazyllm.LOG.error(f'[LazyLLM - parallel_do_embedding][{k}] error: {e}') for n in knodes: - if hasattr(n, '_embedding_state') and k in n._embedding_state: + if k in n.embedding_state: with n._lock: - n._embedding_state.remove(k) + n.embedding_state.remove(k) raise e with ThreadPoolExecutor(max_workers=min(max_workers, len(tasks_by_key))) as ex: diff --git a/tests/basic_tests/RAG/test_transform.py b/tests/basic_tests/RAG/test_transform.py index 79244aee2..12e9710d9 100644 --- a/tests/basic_tests/RAG/test_transform.py +++ b/tests/basic_tests/RAG/test_transform.py @@ -261,6 +261,8 @@ def test_split(self): ] def test_merge(self): + if lazyllm.config.cpp_switch: + pytest.skip('C++ metadata only supports MetadataVType; this case uses None metadata values.') md_text = '\n\n# LinuxBoot on Ampere Mt. Jade Platform' \ '\nThe Ampere Altra Family processor based Mt. Jade platform is a high-performance ARM server platform, offering up to 256 processor cores in a ' \ 'dual socket configuration. The Tianocore EDK2 firmware for the Mt. Jade platform has been fully upstreamed to the tianocore/edk2-platforms repository, '\ @@ -374,6 +376,8 @@ def test_merge(self): assert merged[i].metadata.get('header') is None def test_keep_code_blocks(self): + if lazyllm.config.cpp_switch: + pytest.skip('C++ metadata only supports MetadataVType; this case writes None metadata values.') md_text = '\n\n# LinuxBoot on Ampere Mt. Jade Platform\nThe Ampere Altra Family processor based Mt. Jade platform is a high-performance ARM server platform, offering up to 256 processor cores in a dual socket configuration. The Tianocore EDK2 firmware for the Mt. Jade platform has been fully upstreamed to the tianocore/edk2-platforms repository, enabling the community to build and experiment with the platform\'s firmware using entirely open-source code. It also supports LinuxBoot, an open-source firmware framework that reduces boot time, enhances security, and increases flexibility compared to standard UEFI firmware.\n\nMt. Jade has also achieved a significant milestone by becoming [the first server certified under the Arm SystemReady LS certification program](https://community.arm.com/arm-community-blogs/b/architectures-and-processors-blog/posts/arm-systemready-ls). SystemReady LS ensures compliance with standardized boot and runtime environments for Linux-based systems, enabling seamless deployment across diverse hardware. This certification further emphasizes Mt. Jade\'s readiness for enterprise and cloud-scale adoption by providing assurance of compatibility, performance, and reliability.\n\nThis case study explores the LinuxBoot implementation on the Ampere Mt. Jade platform, inspired by the approach used in [Google\'s LinuxBoot deployment](Google_study.md).\n\n## Ampere EDK2-LinuxBoot Components\nThe Mt. Jade platform embraces a hybrid firmware architecture, combining UEFI/EDK2 for hardware initialization and LinuxBoot for advanced boot functionalities. The platform aligns closely with step 6 in the LinuxBoot adoption model.\n\n\n\nThe entire boot firmware stack for the Mt. Jade is open source and available in the Github.\n\n* **EDK2**: The PEI and minimal (stripped-down) DXE drivers, including both common and platform code, are fully open source and resides in Tianocore edk2-platforms and edk2 repositories.\n* **LinuxBoot**: The LinuxBoot binary ([flashkernel](../glossary.md)) for Mt. Jade is supported in the [linuxboot/linuxboot](https://github.com/linuxboot/linuxboot/tree/main/mainboards/ampere/jade) repository.\n\n## Ampere Solution for LinuxBoot as a Boot Device Selection\nAmpere has implemented and successfully upstreamed a solution for integrating LinuxBoot as a Boot Device Selection (BDS) option into the TianoCore EDK2 framework, as seen in commit [ArmPkg: Implement PlatformBootManagerLib for LinuxBoot](https://github.com/tianocore/edk2/commit/62540372230ecb5318a9c8a40580a14beeb9ded0). This innovation simplifies the boot process for the Mt. Jade platform and aligns with LinuxBoot\'s goals of efficiency and flexibility.\n\nUnlike the earlier practice that replaced the UEFI Shell with a LinuxBoot flashkernel, Ampere\'s solution introduces a custom BDS implementation that directly boots into the LinuxBoot environment as the active boot option. This approach bypasses the need to load the UEFI Shell or UiApp (UEFI Setup Menu), which depend on numerous unnecessary DXE drivers.\n\nTo further enhance flexibility, Ampere introduced a new GUID specifically for the LinuxBoot binary, ensuring clear separation from the UEFI Shell GUID. This distinction allows precise identification of LinuxBoot components in the firmware.\n\n## Build Process\nBuilding a flashable EDK2 firmware image with an integrated LinuxBoot flashkernel for the Ampere Mt. Jade platform involves two main steps: building the LinuxBoot flashkernel and integrating it into the EDK2 firmware build.\n\n### Step 1: Build the LinuxBoot Flashkernel\nThe LinuxBoot flash kernel is built as follows:\n\n```bash\ngit clone https://github.com/linuxboot/linuxboot.git\ncd linuxboot/mainboards/ampere/jade && make fetch flashkernel\n```\n\nAfter the build process completes, the flash kernel will be located at: linuxboot/mainboards/ampere/jade/flashkernel\n\n### Step 2: Build the EDK2 Firmware Image with the Flash Kernel\nThe EDK2 firmware image is built with the LinuxBoot flashkernel integrated into the flash image using the following steps:\n\n```bash\ngit clone https://github.com/tianocore/edk2-platforms.git\ngit clone https://github.com/tianocore/edk2.git\ngit clone https://github.com/tianocore/edk2-non-osi.git\n./edk2-platforms/Platform/Ampere/buildfw.sh -b RELEASE -t GCC -p Jade -l linuxboot/mainboards/ampere/jade/flashkernel\n```\n\nThe `buildfw.sh` script automatically integrates the LinuxBoot flash kernel (provided via the -l option) as part of the final EDK2 firmware image.\n\nThis process generates a flashable EDK2 firmware image with embedded LinuxBoot, ready for deployment on the Ampere Mt. Jade platform.\n\n## Booting with LinuxBoot\nWhen powered on, the system will boot into the u-root and automatically kexec to the target OS.\n\n```text\nRun /init as init process\n1970/01/01 00:00:10 Welcome to u-root!\n...\n```\n\n## Future Work\nWhile the LinuxBoot implementation on the Ampere Mt. Jade platform represents a significant milestone, several advanced features and improvements remain to be explored. These enhancements would extend the platform\'s capabilities, improve its usability, and reinforce its position as a leading open source firmware solution. Key areas for future development include:\n\n### Secure Boot with LinuxBoot\nOne of the critical areas for future development is enabling secure boot verification for the target operating system. In the LinuxBoot environment, the target OS is typically booted using kexec. However, it is unclear how Secure Boot operates in this context, as kexec bypasses traditional firmware-controlled secure boot mechanisms. Future work should investigate how to extend Secure Boot principles to kexec, ensuring that the OS kernel and its components are verified and authenticated before execution. This may involve implementing signature checks and utilizing trusted certificate chains directly within the LinuxBoot environment to mimic the functionality of UEFI Secure Boot during the kexec process.\n\n### TPM Support\nThe platform supports TPM, but its integration with LinuxBoot is yet to be defined. Future work could explore utilizing the TPM for secure boot measurements, and system integrity attestation.\n\n### Expanding Support for Additional Ampere Platforms\nBuilding on the success of LinuxBoot on Mt. Jade, future efforts should expand support to other Ampere platforms. This would ensure broader adoption and usability across different hardware configurations.\n\n### Optimizing the Transition Between UEFI and LinuxBoot\nImproving the efficiency of the handoff between UEFI and LinuxBoot could further reduce boot times. This optimization would involve refining the initialization process and minimizing redundant operations during the handoff.\n\n### Advanced Diagnostics and Monitoring Tools\nAdding more diagnostic and monitoring tools to the LinuxBoot u-root environment would enhance debugging and system management. These tools could provide deeper insights into system performance and potential issues, improving reliability and maintainability.\n\n## See Also\n* [LinuxBoot on Ampere Platforms: A new (old) approach to firmware](https://amperecomputing.com/blogs/linuxboot-on-ampere-platforms--a-new-old-approach-to-firmware)' # noqa: E501 markdown = MarkdownSplitter(keep_headers=False, keep_trace=False, overlap=30, keep_code_blocks=True) splits = markdown._split(md_text, 300) @@ -481,6 +485,8 @@ def test_basic_xml_split(self): assert 'filetype' not in node.metadata def test_xml_with_attributes(self): + if lazyllm.config.cpp_switch: + pytest.skip('C++ metadata only supports MetadataVType; xml attributes are nested dict values.') splitter = XMLSplitter(chunk_size=100, overlap=0, keep_trace=True, keep_tags=True) xml_text = ''' @@ -1593,6 +1599,8 @@ def setup_method(self): self.parser = TreeBuilderParser() def test_build_simple_tree(self): + if lazyllm.config.cpp_switch: + pytest.skip('C++ metadata only supports MetadataVType; tree builder stores DocNode list in metadata.children.') nodes = [ DocNode(text='1. Title', metadata={'text_level': 1}), DocNode(text='1.1 Subtitle', metadata={'text_level': 2}), @@ -1603,6 +1611,8 @@ def test_build_simple_tree(self): assert len(result) >= 1 def test_flat_nodes(self): + if lazyllm.config.cpp_switch: + pytest.skip('C++ metadata only supports MetadataVType; tree builder stores DocNode list in metadata.children.') nodes = [ DocNode(text='Content without level'), DocNode(text='Another content'), @@ -1615,6 +1625,8 @@ def test_empty_list(self): assert result == [] def test_custom_get_level(self): + if lazyllm.config.cpp_switch: + pytest.skip('C++ metadata only supports MetadataVType; tree builder stores DocNode list in metadata.children.') parser = TreeBuilderParser(get_level=lambda n: n.metadata.get('level', 0)) nodes = [ DocNode(text='Level 1', metadata={'level': 1}), @@ -1670,6 +1682,8 @@ def test_skip_level_under(self): assert len(result) >= 1 def test_with_children(self): + if lazyllm.config.cpp_switch: + pytest.skip('C++ metadata only supports MetadataVType values.') nodes = [ DocNode(text='1. Title', metadata={'text_level': 1, 'children': [ DocNode(text='Child 1', metadata={'text_level': 2}), diff --git a/tests/cpp_ext_tests/test_doc_node.py b/tests/cpp_ext_tests/test_doc_node.py deleted file mode 100644 index 40991bb87..000000000 --- a/tests/cpp_ext_tests/test_doc_node.py +++ /dev/null @@ -1,13 +0,0 @@ -class TestDocNode: - def setup_method(self): - from lazyllm import lazyllm_cpp - self.lazyllm_cpp = lazyllm_cpp - - def test_doc_node_set_get(self): - node = self.lazyllm_cpp.DocNode() - assert node.get_text() == '' - node.set_text('hello') - assert node.get_text() == 'hello' - - node2 = self.lazyllm_cpp.DocNode('world') - assert node2.get_text() == 'world' diff --git a/tests/test_cpp_class_decorator.py b/tests/test_cpp_class_decorator.py new file mode 100644 index 000000000..0191e3051 --- /dev/null +++ b/tests/test_cpp_class_decorator.py @@ -0,0 +1,109 @@ +import importlib.util +import os +import pytest +import sys +import types +from pathlib import Path +from types import SimpleNamespace + + +class _ConfigStub: + def __init__(self): + self._values = {'home': str(Path.home() / '.lazyllm')} + + def add(self, name, _type, default, env_name, *args, **kwargs): + raw = os.getenv(f'LAZYLLM_{env_name}') + if raw is None: + value = default + elif _type is bool: + value = raw.lower() in {'1', 'true', 'yes', 'on'} + else: + value = _type(raw) + self._values[name] = value + setattr(self, name, value) + + def __getitem__(self, key): + return self._values[key] + + +def _reload_cpp_module(): + module_path = Path(__file__).resolve().parents[1] / 'lazyllm' / 'cpp.py' + module_name = 'lazyllm.cpp' + + sys.modules.pop(module_name, None) + sys.modules.pop('lazyllm', None) + + pkg = types.ModuleType('lazyllm') + pkg.__path__ = [str(module_path.parent)] # type: ignore[attr-defined] + pkg.config = _ConfigStub() # type: ignore[attr-defined] + sys.modules['lazyllm'] = pkg + + spec = importlib.util.spec_from_file_location(module_name, module_path) + assert spec is not None and spec.loader is not None + cpp = importlib.util.module_from_spec(spec) + sys.modules[module_name] = cpp + spec.loader.exec_module(cpp) + return cpp + + +def test_cpp_class_keeps_python_class_when_disabled(monkeypatch): + monkeypatch.setenv('LAZYLLM_ENABLE_CPP_OVERRIDE', '0') + cpp = _reload_cpp_module() + + class PyOnly: + pass + + replaced = cpp.cpp_class(PyOnly) + assert replaced is PyOnly + + +def test_cpp_class_replaces_with_cpp_export_when_enabled(monkeypatch): + monkeypatch.setenv('LAZYLLM_ENABLE_CPP_OVERRIDE', '1') + cpp = _reload_cpp_module() + + class CppDummy: + pass + + monkeypatch.setattr(cpp, '_load_cpp_module', lambda: SimpleNamespace(Dummy=CppDummy)) + + class Dummy: + pass + + replaced = cpp.cpp_class(Dummy) + assert replaced is CppDummy + + +def test_cpp_class_rejects_non_class_object(monkeypatch): + monkeypatch.setenv('LAZYLLM_ENABLE_CPP_OVERRIDE', '1') + cpp = _reload_cpp_module() + + with pytest.raises(TypeError, match='can only decorate classes'): + cpp.cpp_class('NotAClass') + + +def test_cpp_class_raises_when_cpp_export_missing(monkeypatch): + monkeypatch.setenv('LAZYLLM_ENABLE_CPP_OVERRIDE', '1') + cpp = _reload_cpp_module() + monkeypatch.setattr(cpp, '_load_cpp_module', lambda: SimpleNamespace()) + + class Missing: + pass + + with pytest.raises(AttributeError, match="has no attribute 'Missing'"): + cpp.cpp_class(Missing) + + +def test_cpp_class_propagates_import_error_when_enabled(monkeypatch): + monkeypatch.setenv('LAZYLLM_ENABLE_CPP_OVERRIDE', '1') + cpp = _reload_cpp_module() + + def _boom(): + raise ImportError('boom') + + monkeypatch.setattr(cpp, '_load_cpp_module', _boom) + + class AnyClass: + pass + + with pytest.raises(ImportError, match='boom'): + cpp.cpp_class(AnyClass) diff --git a/tests/test_cpp_proxy_decorator.py b/tests/test_cpp_proxy_decorator.py new file mode 100644 index 000000000..df4ea94e5 --- /dev/null +++ b/tests/test_cpp_proxy_decorator.py @@ -0,0 +1,147 @@ +import importlib.util +import os +import pytest +import sys +import types +from pathlib import Path +from types import SimpleNamespace + + +class _ConfigStub: + def __init__(self): + self._values = {'home': str(Path.home() / '.lazyllm')} + + def add(self, name, _type, default, env_name, *args, **kwargs): + raw = os.getenv(f'LAZYLLM_{env_name}') + if raw is None: + value = default + elif _type is bool: + value = raw.lower() in {'1', 'true', 'yes', 'on'} + else: + value = _type(raw) + self._values[name] = value + setattr(self, name, value) + + def __getitem__(self, key): + return self._values[key] + + +def _reload_cpp_module(): + module_path = Path(__file__).resolve().parents[1] / 'lazyllm' / 'cpp.py' + module_name = 'lazyllm.cpp' + + sys.modules.pop(module_name, None) + sys.modules.pop('lazyllm', None) + + pkg = types.ModuleType('lazyllm') + pkg.__path__ = [str(module_path.parent)] # type: ignore[attr-defined] + pkg.config = _ConfigStub() # type: ignore[attr-defined] + sys.modules['lazyllm'] = pkg + + spec = importlib.util.spec_from_file_location(module_name, module_path) + assert spec is not None and spec.loader is not None + cpp = importlib.util.module_from_spec(spec) + sys.modules[module_name] = cpp + spec.loader.exec_module(cpp) + return cpp + + +class _DemoCPPImpl: + def __init__(self, count: int = 0): + self.count = count + self._value = 0 + + @property + def value(self): + return self._value + + @value.setter + def value(self, v): + self._value = v + + def foo(self, x): + return x + self.count + + +def test_cpp_proxy_keeps_python_class_when_disabled(monkeypatch): + monkeypatch.setenv('LAZYLLM_ENABLE_CPP_OVERRIDE', '0') + cpp = _reload_cpp_module() + + class Demo: + pass + + proxied = cpp.cpp_proxy(Demo) + assert proxied is Demo + + +def test_cpp_proxy_proxies_method_and_attr(monkeypatch): + monkeypatch.setenv('LAZYLLM_ENABLE_CPP_OVERRIDE', '1') + cpp = _reload_cpp_module() + monkeypatch.setattr(cpp, '_load_cpp_module', lambda: SimpleNamespace(DemoCPPImpl=_DemoCPPImpl)) + + @cpp.cpp_proxy + class Demo: + def __init__(self, count: int = 0, ignore: str = ''): + self.value = 3 + + def foo(self, x): + return -1 + + obj = Demo(count=2, ignore='x') + assert obj.foo(5) == 7 + assert obj._c_obj.count == 2 + + obj.value = 9 + assert obj._c_obj.value == 9 + + +def test_cpp_proxy_filters_kwargs_by_name_and_type(monkeypatch): + monkeypatch.setenv('LAZYLLM_ENABLE_CPP_OVERRIDE', '1') + cpp = _reload_cpp_module() + monkeypatch.setattr(cpp, '_load_cpp_module', lambda: SimpleNamespace(DemoCPPImpl=_DemoCPPImpl)) + + @cpp.cpp_proxy + class Demo: + def __init__(self, count: int = 0, ignore: str = ''): + self.value = 0 + + def foo(self, x): + return -1 + + obj = Demo(count='2', ignore='x') + # count 类型不匹配 int，不会透传到 C++ 构造函数。 + assert obj._c_obj.count == 0 + + +def test_cpp_proxy_raises_on_method_signature_mismatch(monkeypatch): + monkeypatch.setenv('LAZYLLM_ENABLE_CPP_OVERRIDE', '1') + cpp = _reload_cpp_module() + + class ImplWithMismatch(_DemoCPPImpl): + def foo(self, x, y): # noqa: D401 + return x + y + + monkeypatch.setattr(cpp, '_load_cpp_module', lambda: SimpleNamespace(DemoCPPImpl=ImplWithMismatch)) + + with pytest.raises(TypeError, match='Signature mismatch'): + @cpp.cpp_proxy + class Demo: + def __init__(self, count: int = 0): + self.value = 0 + + def foo(self, x): + return -1 + + +def test_cpp_proxy_raises_when_python_method_missing(monkeypatch): + monkeypatch.setenv('LAZYLLM_ENABLE_CPP_OVERRIDE', '1') + cpp = _reload_cpp_module() + monkeypatch.setattr(cpp, '_load_cpp_module', lambda: SimpleNamespace(DemoCPPImpl=_DemoCPPImpl)) + + @cpp.cpp_proxy + class Demo: + def __init__(self, count: int = 0): + self.value = 0 + + obj = Demo(count=1) + assert hasattr(obj, '_c_obj')