NVIDIA · WenJing95 · Sep 17, 2026
@@ -38,6 +38,15 @@ install(TARGETS trtmc_model_bart
 )
 
 if(TRTMC_BUILD_TESTS)
+  add_executable(test_bart_added_token_whitespace
+    ${PROJECT_SOURCE_DIR}/families/bart/tests/cpp/test_bart_added_token_whitespace.cpp
+    bpe_tokenizer.cpp
+  )
+  target_include_directories(test_bart_added_token_whitespace PRIVATE ${PROJECT_SOURCE_DIR})
+  target_link_libraries(test_bart_added_token_whitespace PRIVATE nlohmann_json::nlohmann_json)
+  target_compile_options(test_bart_added_token_whitespace PRIVATE -Wall -Wextra -Wpedantic)
+  add_test(NAME bart_added_token_whitespace COMMAND test_bart_added_token_whitespace)
+
   add_executable(test_bart_runtime_config
     ${PROJECT_SOURCE_DIR}/families/bart/tests/cpp/test_bart_runtime_config.cpp
   )

@@ -723,6 +723,24 @@ class BpeTokenizer final : public ITokenizer {
         return {best_id, best_len};
     }
 
+    static void strip_added_token_left_space(std::string& text) {
+        const char* begin = text.data();
+        const char* cursor = begin;
+        const char* end = begin + text.size();
+        size_t content_end = 0;
+        while (cursor < end) {
+            const char32_t cp = read_utf8(cursor, end);
+            // AddedToken lstrip uses the Unicode White_Space property.
+            const bool whitespace = (cp >= 0x09 && cp <= 0x0D) || cp == 0x20 || cp == 0x85 ||
+                                    cp == 0xA0 || cp == 0x1680 || (cp >= 0x2000 && cp <= 0x200A) ||
+                                    cp == 0x2028 || cp == 0x2029 || cp == 0x202F || cp == 0x205F ||
+                                    cp == 0x3000;
+            if (!whitespace)
+                content_end = static_cast<size_t>(cursor - begin);
+        }
+        text.resize(content_end);
+    }
+
     std::vector<Segment> split_added_tokens(const std::string& text) const {
         std::vector<Segment> segments;
         if (mAddedTokenPatterns.empty()) {
@@ -733,6 +751,10 @@ class BpeTokenizer final : public ITokenizer {
         while (pos < text.size()) {
             auto [best_id, best_len] = find_longest_added_token(text, pos);
             if (best_id >= 0) {
+                if (mLstripAddedTokenIds.count(best_id) && !segments.empty() &&
+                    segments.back().added_id < 0) {
+                    strip_added_token_left_space(segments.back().text);
+                }
                 segments.push_back({text.substr(pos, best_len), best_id});
                 pos += best_len;
             } else {
@@ -1154,6 +1176,8 @@ class BpeTokenizer final : public ITokenizer {
             // The special flag only controls decode filtering (mSpecialIds) and
             // post_processor BOS/EOS insertion.
             mAddedTokenPatterns.push_back({content, id});
+            if (tok.value("lstrip", false))
+                mLstripAddedTokenIds.insert(id);
         }
         // Sort by length descending for longest-match-first
         std::sort(mAddedTokenPatterns.begin(), mAddedTokenPatterns.end(),
@@ -1522,6 +1546,7 @@ class BpeTokenizer final : public ITokenizer {
 
     // Non-special added tokens: matched before pre-tokenization (longest first)
     std::vector<std::pair<std::string, int32_t>> mAddedTokenPatterns;
+    std::unordered_set<int32_t> mLstripAddedTokenIds;
 
     bool mAddSpecialTokens = false;
     bool mUsePreTokenizer = true;

@@ -0,0 +1,69 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include "families/bart/runtime/tokenizer.h"
+
+#include <iostream>
+#include <string>
+#include <vector>
+
+int main() {
+    // BART's <mask> consumes preceding whitespace; its other special tokens do not.
+    const std::string tokenizer_json = R"json({
+        "model": {"type": "BPE", "vocab": {
+            "<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3,
+            "H": 4, "i": 5, "\u0120": 6, "<mask>": 7,
+            "\u0109": 8, "\u010a": 9, "\u010b": 10, "\u010c": 11, "\u010d": 12,
+            "\u0122": 13, "\u0123": 14, "\u0124": 15, "\u0125": 16, "\u0126": 17,
+            "\u0127": 18, "\u0128": 19, "\u0129": 20, "\u012a": 21, "\u012b": 22,
+            "\u012c": 23, "\u012d": 24, "\u013c": 25, "\u0141": 26, "\u0142": 27,
+            "\u00a8": 28, "\u00a9": 29, "\u00af": 30, "\u00c2": 31,
+            "\u00e1": 32, "\u00e2": 33, "\u00e3": 34
+        }, "merges": []},
+        "added_tokens": [
+            {"id": 0, "content": "<s>", "special": true, "lstrip": false},
+            {"id": 2, "content": "</s>", "special": true},
+            {"id": 7, "content": "<mask>", "special": true, "lstrip": true}
+        ],
+        "pre_tokenizer": {"type": "ByteLevel", "add_prefix_space": false},
+        "post_processor": {"type": "RobertaProcessing", "cls": ["<s>", 0], "sep": ["</s>", 2]},
+        "decoder": {"type": "ByteLevel"}
+    })json";
+    auto tokenizer = trtmc::CreateBpeTokenizer(tokenizer_json.data(), tokenizer_json.size(), true);
+    int failures = 0;
+    const auto check = [&](const std::string& text, const std::vector<int32_t>& expected) {
+        if (tokenizer->encode(text) != expected) {
+            std::cerr << "Unexpected token IDs for: " << text << '\n';
+            ++failures;
+        }
+    };
+
+    check("Hi <mask> Hi", {0, 4, 5, 7, 6, 4, 5, 2});
+    check("Hi<mask> Hi", {0, 4, 5, 7, 6, 4, 5, 2});
+    check("Hi   <mask>", {0, 4, 5, 7, 2});
+    check("Hi \t\r\n<mask>", {0, 4, 5, 7, 2});
+    check("   <mask>", {0, 7, 2});
+    check("<mask> <mask>", {0, 7, 7, 2});
+    check("Hi <s> Hi", {0, 4, 5, 6, 0, 6, 4, 5, 2});
+    check("Hi </s>", {0, 4, 5, 6, 2, 2});
+    check("Hi Hi", {0, 4, 5, 6, 4, 5, 2});
+    check(u8"Hi\u200b<mask>", {0, 4, 5, 33, 13, 24, 7, 2});
+
+    const std::vector<std::string> whitespace = {
+        u8"\u0085", u8"\u00a0", u8"\u1680", u8"\u2000", u8"\u2001", u8"\u2002", u8"\u2003",
+        u8"\u2004", u8"\u2005", u8"\u2006", u8"\u2007", u8"\u2008", u8"\u2009", u8"\u200a",
+        u8"\u2028", u8"\u2029", u8"\u202f", u8"\u205f", u8"\u3000",
+    };
+    for (const auto& space : whitespace)
+        check("Hi" + space + "<mask>", {0, 4, 5, 7, 2});
+
+    auto without_special_tokens =
+        trtmc::CreateBpeTokenizer(tokenizer_json.data(), tokenizer_json.size(), false);
+    if (without_special_tokens->encode("Hi <mask>") != std::vector<int32_t>{4, 5, 7}) {
+        std::cerr << "lstrip must also apply when post-processing is disabled\n";
+        ++failures;
+    }
+    return failures == 0 ? 0 : 1;
+}