fix(preprocessor): table-driven Unicode classes, file-driven added tokens, parse_special and neutralize policy, Qwen3 byte-split decode

examples/qwen3/main.cpp

@@ -1,3 +1,4 @@
+#include <cstdio>
 #include <iostream>
 #include <fmt/core.h>
 #include <mllm/mllm.hpp>
@@ -56,7 +57,12 @@ MLLM_MAIN({
       fmt::print("\n🤖 Response: ");
 
       // Use for loop
-      for (auto& step : qwen3.chat(inputs)) { std::wcout << qwen3_tokenizer.detokenize(step.cur_token_id) << std::flush; }
+      mllm::preprocessor::StreamingUtf8Decoder decoder;
+      for (auto& step : qwen3.chat(inputs)) {
+        fmt::print("{}", decoder.append(qwen3_tokenizer.detokenizeBytes(step.cur_token_id)));
+        std::fflush(stdout);
+      }
+      fmt::print("{}", decoder.finish());
 
       fmt::print("\n{}\n", std::string(60, '-'));
     } catch (const std::exception& e) { fmt::print("\n❌ Error: {}\n{}\n", e.what(), std::string(60, '-')); }

examples/qwen3_moe/main.cpp

@@ -1,3 +1,4 @@
+#include <cstdio>
 #include <iostream>
 #include <fmt/core.h>
 #include <mllm/mllm.hpp>
@@ -62,7 +63,12 @@ MLLM_MAIN({
       fmt::print("\n🤖 Response: ");
 
       // Use for loop
-      for (auto& step : qwen3_moe.chat(inputs)) { std::wcout << qwen3_moe_tokenizer.detokenize(step.cur_token_id) << std::flush; }
+      mllm::preprocessor::StreamingUtf8Decoder decoder;
+      for (auto& step : qwen3_moe.chat(inputs)) {
+        fmt::print("{}", decoder.append(qwen3_moe_tokenizer.detokenizeBytes(step.cur_token_id)));
+        std::fflush(stdout);
+      }
+      fmt::print("{}", decoder.finish());
 
       fmt::print("\n{}\n", std::string(60, '-'));
     } catch (const std::exception& e) { fmt::print("\n❌ Error: {}\n{}\n", e.what(), std::string(60, '-')); }

mllm/models/lfm2/tokenization_lfm2.hpp

@@ -27,7 +27,7 @@ inline bool tokenizerMatch(const std::wstring& input, size_t& pos, std::wstring&
   for (const auto& contraction : contractions) {
     bool match = pos + contraction.size() <= input.size();
     for (size_t index = 0; match && index < contraction.size(); ++index) {
-      match = std::towlower(input[pos + index]) == contraction[index];
+      match = preprocessor::toLower(input[pos + index]) == contraction[index];
     }
     if (match) {
       matched = input.substr(pos, contraction.size());
@@ -62,9 +62,9 @@ inline bool tokenizerMatch(const std::wstring& input, size_t& pos, std::wstring&
     const auto original = pos;
     const auto start = pos;
     if (input[pos] == L' ') ++pos;
-    if (pos < input.size() && !std::iswspace(input[pos]) && !preprocessor::isLetter(input[pos])
+    if (pos < input.size() && !preprocessor::isWhitespace(input[pos]) && !preprocessor::isLetter(input[pos])
         && !preprocessor::isDigit(input[pos])) {
-      while (pos < input.size() && !std::iswspace(input[pos]) && !preprocessor::isLetter(input[pos])
+      while (pos < input.size() && !preprocessor::isWhitespace(input[pos]) && !preprocessor::isLetter(input[pos])
              && !preprocessor::isDigit(input[pos])) {
         ++pos;
       }
@@ -79,7 +79,7 @@ inline bool tokenizerMatch(const std::wstring& input, size_t& pos, std::wstring&
     const auto start = pos;
     auto scan = pos;
     size_t last_break = std::wstring::npos;
-    while (scan < input.size() && std::iswspace(input[scan])) {
+    while (scan < input.size() && preprocessor::isWhitespace(input[scan])) {
       if (input[scan] == L'\r' || input[scan] == L'\n') last_break = scan + 1;
       ++scan;
     }
@@ -90,9 +90,9 @@ inline bool tokenizerMatch(const std::wstring& input, size_t& pos, std::wstring&
     }
   }
 
-  if (std::iswspace(input[pos])) {
+  if (preprocessor::isWhitespace(input[pos])) {
     const auto start = pos;
-    while (pos < input.size() && std::iswspace(input[pos])) ++pos;
+    while (pos < input.size() && preprocessor::isWhitespace(input[pos])) ++pos;
     if (pos >= input.size()) {
       matched = input.substr(start, pos - start);
       return true;
@@ -104,9 +104,9 @@ inline bool tokenizerMatch(const std::wstring& input, size_t& pos, std::wstring&
     }
     pos = start;
   }
-  if (std::iswspace(input[pos])) {
+  if (preprocessor::isWhitespace(input[pos])) {
     const auto start = pos;
-    while (pos < input.size() && std::iswspace(input[pos])) ++pos;
+    while (pos < input.size() && preprocessor::isWhitespace(input[pos])) ++pos;
     matched = input.substr(start, pos - start);
     return true;
   }

mllm/models/minicpm4/configuration_minicpm4.hpp

@@ -61,6 +61,8 @@ struct MiniCPM4Config : protected ConfigFile {
 
     chat_template_backend = preprocessor::parseChatTemplateBackend(data().value("chat_template_backend", "legacy"));
 
+    control_token_policy = preprocessor::parseControlTokenPolicy(data().value("control_token_policy", "reject"));
+
     linear_impl_type = aops::str2LinearImplTypes(data()["linear_impl_type"]);
   }
 
@@ -96,6 +98,7 @@ struct MiniCPM4Config : protected ConfigFile {
 
   aops::LinearImplTypes linear_impl_type = aops::LinearImplTypes::kDefault;
   preprocessor::ChatTemplateBackend chat_template_backend = preprocessor::ChatTemplateBackend::Legacy;
+  preprocessor::ControlTokenPolicy control_token_policy = preprocessor::ControlTokenPolicy::Reject;
 };
 
 }  // namespace mllm::models::minicpm4

mllm/models/minicpm4/tokenization_minicpm4.hpp

@@ -7,6 +7,7 @@
 #include <unordered_map>
 #include "mllm/preprocessor/tokenizers/BPEUTF8.hpp"  //BPEUTF8, MiniCPM4 use LlamaTokenizer!
 #include "mllm/models/ARGeneration.hpp"
+#include "mllm/preprocessor/chat_template/PromptTokenization.hpp"
 #include "mllm/models/minicpm4/configuration_minicpm4.hpp"
 #include "mllm/preprocessor/chat_template/LegacyChatMl.hpp"
 #include "mllm/preprocessor/tokenizers/Unicode.hpp"
@@ -25,6 +26,7 @@ class MiniCPM4Tokenizer final : public mllm::preprocessor::AutoTokenizerUTF8 {
       : chat_preprocessor_(std::move(chat_template), preprocessor::renderLegacyChatMlSingleTurn) {
     bpe_.initFromSentencePieceJson(file_path);
     chat_preprocessor_.setControlTokens(bpe_.controlTokens());
+    registerAddedTokens(bpe_.addedTokens());
 
     special_tokens_trie_.add("<|im_start|>");
     special_tokens_trie_.add("<|im_end|>");
@@ -85,22 +87,19 @@ class MiniCPM4Tokenizer final : public mllm::preprocessor::AutoTokenizerUTF8 {
     return decoded;
   }
 
-  std::vector<std::string> tokenize(const std::string& str) override {
-    auto segments = special_tokens_trie_.split(str);
+    std::vector<std::string> tokenize(const std::string& str) override { return tokenize(str, {}); }
 
+  std::vector<std::string> tokenize(const std::string& str, const preprocessor::TokenizeOptions& options) override {
     std::vector<std::string> all_tokens;
-    for (const auto& segment : segments) {
-      if (special_tokens_trie_.isSpecialToken(segment)) {
-        all_tokens.push_back(segment);
+    for (const auto& segment : special_tokens_trie_.splitSegments(str, {.parse_special = options.parse_special})) {
+      if (segment.is_special) {
+        all_tokens.push_back(segment.text);
         continue;
       }
-
-      std::string normalized = normalize(segment);
-
+      std::string normalized = normalize(segment.text);
       auto bpe_tokens = bpe_._bpe(normalized);
       all_tokens.insert(all_tokens.end(), bpe_tokens.begin(), bpe_tokens.end());
     }
-
     return all_tokens;
   }
 
@@ -121,19 +120,34 @@ class MiniCPM4Tokenizer final : public mllm::preprocessor::AutoTokenizerUTF8 {
                               .backend = config.chat_template_backend,
                               .template_options = {.model_directory = model_directory.empty()
                                                                           ? std::filesystem::path(file_path).parent_path()
-                                                                          : std::move(model_directory)}}) {}
+                                                                          : std::move(model_directory)},
+                           .control_token_policy = config.control_token_policy}) {}
 
   preprocessor::ChatTemplateBackend chatTemplateBackend() const noexcept { return chat_preprocessor_.backend(); }
 
   // Renders the single-turn runner prompt through the configured backend.
-  std::string renderChatTemplate(const MiniCPM4Message& message) const {
-    return chat_preprocessor_.render(preprocessor::makeSingleTurnChatTemplateRequest(message.prompt, "", std::nullopt));
+  preprocessor::ChatTemplateRequest requestFor(const MiniCPM4Message& message) const {
+    return preprocessor::makeSingleTurnChatTemplateRequest(message.prompt, "", std::nullopt);
   }
 
-  ARGenerationOutputPast convertMessage(const MiniCPM4Message& message) {
-    auto applied_string = renderChatTemplate(message);
+  std::string renderChatTemplate(const MiniCPM4Message& message) const { return chat_preprocessor_.render(requestFor(message)); }
+
+  // Origin-tagged prompt: under control_token_policy=neutralize the official
+  // template's provenance is kept so message content is tokenized without
+  // special-token parsing; otherwise the flat prompt is one template span.
+  std::vector<preprocessor::PromptSpan> renderPromptSpans(const MiniCPM4Message& message) const {
+    if (chat_preprocessor_.controlTokenPolicy() == preprocessor::ControlTokenPolicy::Neutralize) {
+      return chat_preprocessor_.renderSpans(requestFor(message));
+    }
+    return {{renderChatTemplate(message), false}};
+  }
 
-    auto tokens = tokenize(applied_string);
+  std::vector<std::string> tokenizePrompt(const MiniCPM4Message& message) {
+    return preprocessor::tokenizePromptSpans(*this, renderPromptSpans(message));
+  }
+
+  ARGenerationOutputPast convertMessage(const MiniCPM4Message& message) {
+    auto tokens = tokenizePrompt(message);
 
     std::vector<int64_t> ids;
     ids.reserve(tokens.size());

mllm/models/minicpm5/configuration_minicpm5.hpp

@@ -54,6 +54,8 @@ struct MiniCPM5Config : protected ConfigFile {
 
     chat_template_backend = preprocessor::parseChatTemplateBackend(config.value("chat_template_backend", "legacy"));
 
+    control_token_policy = preprocessor::parseControlTokenPolicy(config.value("control_token_policy", "reject"));
+
     if (config.contains("linear_impl_type")) {
       const auto linear_impl_name = config["linear_impl_type"].get<std::string>();
       linear_impl_type = aops::str2LinearImplTypes(linear_impl_name);
@@ -97,6 +99,7 @@ struct MiniCPM5Config : protected ConfigFile {
   int32_t max_cache_length = 2048;
   aops::LinearImplTypes linear_impl_type = aops::LinearImplTypes::kDefault;
   preprocessor::ChatTemplateBackend chat_template_backend = preprocessor::ChatTemplateBackend::Legacy;
+  preprocessor::ControlTokenPolicy control_token_policy = preprocessor::ControlTokenPolicy::Reject;
 };
 
 inline auto matchesOfficialMiniCPM5_1BRuntimeContract(const MiniCPM5Config& config) -> bool {