added preserved tokens and <|observation|> handling

Improved overall stability of code

Author: dhandhalyabhavik

common/chat.cpp

@@ -1331,60 +1331,174 @@ static void common_chat_parse_gpt_oss(common_chat_msg_parser & builder) {
 
 static common_chat_params common_chat_params_init_glm_4_5(const common_chat_template & tmpl, const struct templates_params & inputs) {
     common_chat_params data;
+    
     data.prompt = apply(tmpl, inputs);
     data.format = COMMON_CHAT_FORMAT_GLM_4_5;
+    
+    data.preserved_tokens = {
+        "<|system|>", "<|assistant|>", "<|observation|>",
+        "<tool_call>", "</tool_call>", "<arg_key>", "</arg_key>", 
+        "<arg_value>", "</arg_value>", "<think>", "</think>",
+        "<tool_response>", "</tool_response>",
+    };
+    
+    // Todo @bruce XML grammer for GLM codes
+    // Step 3: Grammar for tools (if any)
+    // if (inputs.tools.is_array() && !inputs.tools.empty()) {
+    //     data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
+        
+    //     data.grammar = build_grammar([&](const common_grammar_builder & builder) {
+    //         // Define flexible arg pair pattern
+    //         auto arg_pair = builder.add_rule("arg_pair",
+    //             std::string("\"<arg_key>\" [a-zA-Z_][a-zA-Z0-9_]* \"</arg_key>\" space ") +
+    //             "\"<arg_value>\" [^<]+ \"</arg_value>\" space"
+    //         );
+            
+    //         // Zero or more arg pairs
+    //         auto arg_section = builder.add_rule("arg_section", "(" + arg_pair + ")*");
+            
+    //         std::vector<std::string> tool_rules;
+    //         foreach_function(inputs.tools, [&](const json & tool) {
+    //             const auto & function = tool.at("function");
+    //             std::string name = function.at("name");
+                
+    //             tool_rules.push_back(
+    //                 builder.add_rule(name + "-call",
+    //                     "\"<tool_call>\" \"" + name + "\" space " +
+    //                     arg_section + " space " +
+    //                     "\"</tool_call>\""
+    //                 )
+    //             );
+    //         });
+
+    //         auto any_tool_call = builder.add_rule("any_tool_call", 
+    //             string_join(tool_rules, " | ")
+    //         );
+
+    //         auto multiple_tool_calls = builder.add_rule("multiple_tool_calls",
+    //             "(" + any_tool_call + " space)+"
+    //         );
+
+    //         builder.add_rule("root", multiple_tool_calls);
+    //     });
+
+    //     data.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<tool_call>"});
+    // }
+    
     return data;
 }
 
 static void common_chat_parse_glm_4_5(common_chat_msg_parser & builder) {
+    
+    auto handle_tool_call_end = [&] (common_chat_msg_parser & builder, auto end_pos) {
+        builder.move_to(end_pos);
+        builder.consume_literal("</tool_call>");
+        
+        size_t obs_pos = builder.input().find("<|observation|>", builder.pos());
+        if (obs_pos != std::string::npos) {
+            if (obs_pos > builder.pos()) {
+                std::string content = builder.input().substr(builder.pos(), obs_pos - builder.pos());
+                builder.add_content(content);
+            }
+            
+            builder.move_to(obs_pos);
+            builder.consume_literal("<|observation|>");
+        } else {
+            std::string remaining = builder.consume_rest();
+            if (!remaining.empty()) builder.add_content(remaining);
+        }
+    };
+    
     builder.consume_spaces();
+    
     builder.try_parse_reasoning("<think>", "</think>");
-    if (!builder.syntax().parse_tool_calls) {
-        builder.add_content(builder.consume_rest());
-        return;
-    }
-
-    // GLM 4.5 uses format: <tool_call>function_name\n<arg_key>key</arg_key>\n<arg_value>value</arg_value>\n</tool_call>
-    static const common_regex tool_call_start("<tool_call>([^\n<]+)");
-    static const common_regex arg_key_regex("<arg_key>([^<]+)</arg_key>");
-    static const common_regex arg_value_regex("<arg_value>([\\s\\S]*?)</arg_value>");
-    static const common_regex tool_call_end("</tool_call>");
-
-    while (auto res = builder.try_find_regex(tool_call_start)) {
-        // Move to the start of the tool call and consume it
-        builder.move_to(res->groups[0].begin);
-        builder.consume_regex(tool_call_start);
-        
-        std::string function_name = builder.str(res->groups[1]);
-        json arguments = json::object();
+    
+    size_t curr_pos = builder.pos();
+    while (builder.input().find("<tool_call>", builder.pos()) != std::string::npos) {
+        size_t tool_call_start = builder.input().find("<tool_call>", builder.pos());
+        if (tool_call_start > builder.pos()) {
+            std::string content = builder.input().substr(builder.pos(), tool_call_start - builder.pos());
+            builder.add_content(content);
+        }
 
+        size_t tool_call_end = builder.input().find("</tool_call>", tool_call_start);
+        if (tool_call_end == std::string::npos) return;
+
+        builder.move_to(tool_call_start);
+        builder.consume_literal("<tool_call>");
         builder.consume_spaces();
-        
-        // Parse all arg_key/arg_value pairs
-        while (auto key_res = builder.try_consume_regex(arg_key_regex)) {
-            std::string key = builder.str(key_res->groups[1]);
-            builder.consume_spaces();
+
+        size_t arg_key_start = builder.input().find("<arg_key>", tool_call_start);
+        if (arg_key_start == std::string::npos) {
+            std::string function_content = builder.input().substr(builder.pos(), tool_call_end - builder.pos());
+            std::string function_name = string_strip(function_content);
 
-            if (auto value_res = builder.try_consume_regex(arg_value_regex)) {
-                std::string value = builder.str(value_res->groups[1]);
-                arguments[key] = value;
+            if (!builder.add_tool_call(function_name, "", "{}")) {
+                LOG_INF("%s: failed to add tool call\n", __func__);
+            }
+
+            handle_tool_call_end(builder, tool_call_end);
+
+        } else {
+            std::string function_content = builder.input().substr(builder.pos(), arg_key_start - builder.pos());
+            std::string function_name = string_strip(function_content);
+            
+            json args_json = json::object();
+            builder.move_to(arg_key_start);
+            
+            while (builder.pos() < tool_call_end) {
+                if (!builder.try_consume_literal("<arg_key>")) {
+                    builder.consume_spaces();
+                    if (!builder.try_consume_literal("<arg_key>")) {
+                        break;
+                    }
+                }
+                
+                auto key_close = builder.try_find_literal("</arg_key>");
+                if (!key_close || key_close->groups[0].end > tool_call_end) {
+                    throw common_chat_msg_partial_exception("incomplete tool call");
+                    return;
+                }
+                
+                std::string key = string_strip(key_close->prelude);
+                
+                builder.consume_spaces();
+                
+                if (!builder.try_consume_literal("<arg_value>")) {
+                    throw common_chat_msg_partial_exception("incomplete tool call");
+                    return;
+                }
+                
+                auto value_close = builder.try_find_literal("</arg_value>");
+                if (!value_close || value_close->groups[0].end > tool_call_end) {
+                    throw common_chat_msg_partial_exception("incomplete tool call");
+                    return;
+                }
+                
+                std::string value = string_strip(value_close->prelude);
                 builder.consume_spaces();
+                args_json[key] = value;
+            }
+            
+            if (!builder.add_tool_call(function_name, "", args_json.dump())) {
+                LOG_INF("%s: failed to add tool call with arguments\n", __func__);
             } else {
-                throw common_chat_msg_partial_exception("Expected <arg_value> after <arg_key>");
+                LOG_INF("%s: successfully added tool call with arguments\n", __func__);
             }
+
+            handle_tool_call_end(builder, tool_call_end);
         }
-        
-        // Consume closing tag
-        builder.consume_regex(tool_call_end);
-        builder.consume_spaces();
-        
-        // Add the parsed tool call
-        if (!builder.add_tool_call(function_name, "", arguments.dump())) {
-            throw common_chat_msg_partial_exception("Failed to add GLM tool call");
+    
+        if (curr_pos == builder.pos()) {
+            // No progress made, avoid infinite loop
+            LOG_INF("%s: no progress in parsing, stopping to avoid infinite loop\n", __func__);
+            break;
         }
     }
 
-    builder.add_content(builder.consume_rest());
+    if (builder.pos() < builder.input().size()) {
+        builder.add_content(builder.consume_rest());
+    }
 }
 
 static common_chat_params common_chat_params_init_firefunction_v2(const common_chat_template & tmpl, const struct templates_params & inputs) {