From 97fced2e25d89e0ab56aa120a6211e8c881e0843 Mon Sep 17 00:00:00 2001 From: Akshay Sonawane Date: Tue, 1 Sep 2026 20:01:13 +0000 Subject: [PATCH 01/11] Tool call changes --- docs/README.md | 1 + sdk_v2/cpp/src/catalog/azure_model_catalog.cc | 100 +++- sdk_v2/cpp/src/contracts/responses.h | 11 +- sdk_v2/cpp/src/contracts/responses_json.cc | 11 +- .../generative/chat/chat_session.cc | 494 ++++++++++-------- .../generative/chat/chat_template.cc | 24 +- .../generative/chat/chat_template.h | 3 + .../generative/chat/onnx_chat_generator.cc | 150 ++++-- .../generative/chat/onnx_chat_generator.h | 28 +- .../chat/reasoning_stream_splitter.h | 324 ++---------- .../openresponses/response_converter.cc | 3 + .../generative/toolcalling/grammar.cc | 8 +- .../tool_call_stream_accumulator.h | 82 ++- .../generative/toolcalling/tool_call_utils.cc | 285 ++++++++-- .../generative/toolcalling/tool_call_utils.h | 4 +- sdk_v2/cpp/src/items/message_item.cc | 3 +- sdk_v2/cpp/src/items/message_item.h | 2 + sdk_v2/cpp/src/service/responses_handler.cc | 105 +++- .../internal_api/chat/chat_template_test.cc | 19 +- .../internal_api/response_converter_test.cc | 110 +--- .../internal_api/toolcalling/grammar_test.cc | 8 +- .../tool_call_stream_accumulator_test.cc | 116 ++-- .../toolcalling/tool_call_utils_test.cc | 159 ++++++ 23 files changed, 1217 insertions(+), 833 deletions(-) diff --git a/docs/README.md b/docs/README.md index 5fa298a0f..a1eb60674 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,6 +2,7 @@ Documentation for Foundry Local can be found in the following resources: +- [MSBench Tool Calling Fixes](MSBench%20Tool%20Calling%20Fixes.md): Engineering notes for the Responses API and tool-calling changes that enabled a local Qwen model to produce SWE-bench patches through MSBench. - [Microsoft Learn](https://learn.microsoft.com/azure/foundry-local/): This is the official documentation for Foundry Local, providing comprehensive guides, tutorials, and reference materials to help you get started and make the most of Foundry Local. - SDK Reference: - [C# SDK Reference](../sdk/cs/README.md): This documentation provides detailed information about the C# SDK for Foundry Local, including API references, usage examples, and best practices for integrating Foundry Local into your applications. diff --git a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc index 853aefd9d..6b02dc8d4 100644 --- a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc @@ -6,12 +6,17 @@ #include "catalog/local_model_scanner.h" #include "model.h" #include "model_info.h" +#include "utils.h" #include #include +#include #include +#include +#include #include +#include #include #include @@ -19,6 +24,69 @@ namespace fl { namespace { +// Merge selected fields from the model directory's inference_model.json into +// a BYOM ModelInfo. The scanner only extracts the model name, so tool-calling +// and reasoning tags declared by the model author would otherwise be dropped. +void MergeInferenceModelJson(ModelInfo& info, const std::string& local_path) { + if (local_path.empty()) { + return; + } + + auto json_path = std::filesystem::path(local_path) / "inference_model.json"; + std::error_code ec; + if (!std::filesystem::exists(json_path, ec)) { + return; + } + + try { + std::ifstream file(json_path); + if (!file.is_open()) { + return; + } + auto j = nlohmann::json::parse(file, /*cb=*/nullptr, /*allow_exceptions=*/false); + if (j.is_discarded() || !j.is_object()) { + return; + } + + auto read_string = [&](const char* key, const char* prop) { + if (j.contains(key) && j[key].is_string()) { + info.string_properties[prop] = j[key].get(); + } + }; + auto read_bool_as_int = [&](const char* key, const char* prop) { + if (j.contains(key) && j[key].is_boolean()) { + info.int_properties[prop] = j[key].get() ? 1 : 0; + } + }; + + read_bool_as_int("supportsToolCalling", FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT); + read_bool_as_int("supportsReasoning", FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT); + read_string("toolCallStart", FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_START_STR); + read_string("toolCallEnd", FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_END_STR); + read_string("reasoningStart", FOUNDRY_LOCAL_MODEL_PROP_REASONING_START_STR); + read_string("reasoningEnd", FOUNDRY_LOCAL_MODEL_PROP_REASONING_END_STR); + } catch (...) { + // inference_model.json is best-effort metadata; ignore parse failures. + } +} + +ModelInfo MakeByomModelInfo(const std::string& model_id, const std::string& local_path) { + auto [name, version] = Utils::SplitModelNameAndVersion(model_id); + + ModelInfo info; + info.model_id = model_id; + info.name = name; + info.alias = name; + info.uri = "local://" + name; + info.version = version; + info.string_properties[FOUNDRY_LOCAL_MODEL_PROP_MODEL_PROVIDER_STR] = "Local"; + info.string_properties[FOUNDRY_LOCAL_MODEL_PROP_MODEL_TYPE_STR] = "ONNX"; + + MergeInferenceModelJson(info, local_path); + + return info; +} + std::vector DeduplicateByModelId(std::vector model_infos) { std::vector deduplicated; deduplicated.reserve(model_infos.size()); @@ -33,13 +101,6 @@ std::vector DeduplicateByModelId(std::vector model_infos) return deduplicated; } -void RemoveLegacyLocalEntries(std::vector& model_infos) { - std::erase_if(model_infos, [](const auto& info) { - const auto* provider = info.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_MODEL_PROVIDER_STR); - return provider && *provider == "Local"; - }); -} - } // namespace AzureModelCatalog::AzureModelCatalog(std::vector>> catalog_urls, @@ -107,26 +168,37 @@ AzureModelCatalog::CatalogResult AzureModelCatalog::GetLiveCatalogOrLocalSnapsho CatalogCache cache(cache_dir_, logger_); cache.Load(); auto cached = cache.GetCachedModels(); - auto snapshot_model_infos = cached ? std::move(*cached) : std::vector{}; - RemoveLegacyLocalEntries(snapshot_model_infos); return { - .model_infos = DeduplicateByModelId(std::move(snapshot_model_infos)), + .model_infos = cached ? DeduplicateByModelId(std::move(*cached)) : std::vector{}, .source = CatalogSource::kSnapshot, }; } -std::vector AzureModelCatalog::CreateModelsWithLocalPaths(const std::vector& model_infos, - const LocalModels& local_models) const { +std::vector AzureModelCatalog::AddLocalModels(std::vector& model_infos, + const LocalModels& local_models) const { std::vector models; - models.reserve(model_infos.size()); + models.reserve(model_infos.size() + local_models.size()); + std::unordered_set model_ids; + model_ids.reserve(model_infos.size() + local_models.size()); for (const auto& info : model_infos) { + model_ids.insert(info.model_id); + auto local_model = local_models.find(info.model_id); auto local_path = local_model != local_models.end() ? local_model->second : std::string{}; models.push_back(model_factory_(ModelInfo(info), std::move(local_path))); } + for (const auto& [model_id, local_path] : local_models) { + if (!model_ids.insert(model_id).second) { + continue; + } + + model_infos.push_back(MakeByomModelInfo(model_id, local_path)); + models.push_back(model_factory_(ModelInfo(model_infos.back()), local_path)); + } + return models; } @@ -143,7 +215,7 @@ std::vector AzureModelCatalog::FetchModels() const { logger_.Log(LogLevel::Information, fmt::format("Found {} locally cached models.", cached_model_ids.size())); auto catalog_result = GetLiveCatalogOrLocalSnapshot(cached_model_ids); - auto models = CreateModelsWithLocalPaths(catalog_result.model_infos, local_models); + auto models = AddLocalModels(catalog_result.model_infos, local_models); logger_.Log(LogLevel::Information, fmt::format("Populated model info for {} models.", models.size())); diff --git a/sdk_v2/cpp/src/contracts/responses.h b/sdk_v2/cpp/src/contracts/responses.h index bf65fc54e..26e318b19 100644 --- a/sdk_v2/cpp/src/contracts/responses.h +++ b/sdk_v2/cpp/src/contracts/responses.h @@ -74,7 +74,15 @@ struct FunctionCallResultInputItem { std::string output; }; -using InputItem = std::variant; +struct FunctionCallInputItem { + std::string type = "function_call"; + std::string call_id; + std::string name; + std::string arguments; +}; + +using InputItem = std::variant; // --------------------------------------------------------------------------- // Tool calling types (AD-010) @@ -336,6 +344,7 @@ void from_json(const nlohmann::json& j, InputImageContent& c); void from_json(const nlohmann::json& j, InputFileContent& c); void from_json(const nlohmann::json& j, InputAudioContent& c); void from_json(const nlohmann::json& j, InputMessage& m); +void from_json(const nlohmann::json& j, FunctionCallInputItem& f); void from_json(const nlohmann::json& j, FunctionCallResultInputItem& f); // --- Tool types from_json --- diff --git a/sdk_v2/cpp/src/contracts/responses_json.cc b/sdk_v2/cpp/src/contracts/responses_json.cc index 732ba70af..f94e22e53 100644 --- a/sdk_v2/cpp/src/contracts/responses_json.cc +++ b/sdk_v2/cpp/src/contracts/responses_json.cc @@ -164,6 +164,13 @@ void from_json(const nlohmann::json& j, FunctionCallResultInputItem& f) { f.output = j.at("output").get(); } +void from_json(const nlohmann::json& j, FunctionCallInputItem& f) { + f.type = j.value("type", "function_call"); + f.call_id = j.at("call_id").get(); + f.name = j.at("name").get(); + f.arguments = j.at("arguments").get(); +} + // ======================================================================== // Tool types from_json // ======================================================================== @@ -248,7 +255,9 @@ void from_json(const nlohmann::json& j, ResponseCreateParams& p) { for (const auto& entry : input) { std::string type = entry.value("type", ""); - if (type == "function_call_output") { + if (type == "function_call") { + items.push_back(entry.get()); + } else if (type == "function_call_output") { items.push_back(entry.get()); } else { // Default: message item diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc index 48a1a05ea..d1ec7179f 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc @@ -48,54 +48,6 @@ void ApplyToolChoiceToContext(std::optional tool_choice, ToolCallC } } -using TextSegment = ReasoningStreamSplitter::Segment; - -ReasoningStreamSplitter CreateReasoningSplitter(const ToolCallContext& tool_ctx, - GenAIModelInstance& model) { - if (!tool_ctx.supports_reasoning) { - return {"", ""}; - } - - const auto& tag_info = model.GetTagInfo(); - auto start = tool_ctx.reasoning_start.empty() ? tag_info.bor_str : tool_ctx.reasoning_start; - auto end = tool_ctx.reasoning_end.empty() ? tag_info.eor_str : tool_ctx.reasoning_end; - auto start_token_ids = - tag_info.bor_id.has_value() ? std::vector{*tag_info.bor_id} : std::vector{}; - auto end_token_ids = - tag_info.eor_id.has_value() ? std::vector{*tag_info.eor_id} : std::vector{}; - auto ignored_token_ids = model.GetPreprocessor().GetEosTokenIds(); - return {std::move(start), std::move(end), std::move(start_token_ids), std::move(end_token_ids), - std::move(ignored_token_ids)}; -} - -void AppendSegment(std::vector& destination, std::string text, flTextItemType type) { - if (text.empty()) { - return; - } - - if (!destination.empty() && destination.back().type == type) { - destination.back().text += text; - } else { - destination.push_back({std::move(text), type}); - } -} - -void AppendGeneratedSegment(std::vector& destination, std::string text, flTextItemType type) { - if (text.empty()) { - return; - } - - if (!destination.empty()) { - auto* previous = std::get_if(&destination.back()); - if (previous && previous->type == type) { - previous->text += text; - return; - } - } - - destination.push_back(TextSegment{std::move(text), type}); -} - } // namespace ChatSession::ChatSession(const fl::Model& catalog_model, GenAIModelInstance& model, ILogger& logger, ITelemetry& telemetry) @@ -193,17 +145,6 @@ ToolCallContext ChatSession::BuildToolCallContext(const Request& request) const } } - // Catalog metadata is immutable and may not contain markers for models whose - // tokenizer defines them dynamically. Read those markers from the loaded GenAI - // model without mutating the published ModelInfo. - const auto& tag_info = model_.GetTagInfo(); - if (tool_ctx.tool_call_start.empty()) { - tool_ctx.tool_call_start = tag_info.bot_str; - } - if (tool_ctx.tool_call_end.empty()) { - tool_ctx.tool_call_end = tag_info.eot_str; - } - // Check if the model supports chain-of-thought reasoning const auto* reasoning_val = info.GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT); if (reasoning_val && *reasoning_val == 1) { @@ -227,12 +168,6 @@ ToolCallContext ChatSession::BuildToolCallContext(const Request& request) const tool_ctx.reasoning_end = *val; } } - if (tool_ctx.reasoning_start.empty()) { - tool_ctx.reasoning_start = tag_info.bor_str; - } - if (tool_ctx.reasoning_end.empty()) { - tool_ctx.reasoning_end = tag_info.eor_str; - } // Accumulate tool definitions from the session. // Tool definitions may come from two sources: @@ -268,6 +203,14 @@ ToolCallContext ChatSession::BuildToolCallContext(const Request& request) const tool_ctx.tools_json.clear(); } + // Locally imported models may not have catalog metadata even when their + // chat template uses the standard Qwen tool-call markers. + if (tool_ctx.HasTools() && tool_ctx.tool_call_start.empty() && tool_ctx.tool_call_end.empty()) { + tool_ctx.supports_tool_calling = true; + tool_ctx.tool_call_start = ""; + tool_ctx.tool_call_end = ""; + } + // Determine text_output / tool_output from tool_choice parameter. // ParseToolChoice rejects unknown values with FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT. auto tool_choice = SearchOptions::ParseToolChoice(request.options); @@ -286,19 +229,128 @@ ToolCallContext ChatSession::BuildToolCallContext(const Request& request) const return tool_ctx; } -void ChatSession::ProcessGeneratedOutput(std::vector events, +// A segment of generated assistant text, tagged with whether it is ordinary visible text or reasoning content. +struct TextSegment { + std::string text; + flTextItemType type; +}; + +// Split assistant output around ... (or equivalent reasoning markers) into typed segments. +// +// - Text outside the markers is emitted as DEFAULT (visible) segments. +// - Text inside the markers is emitted as REASONING segments. The markers themselves are stripped. +// - Truncated reasoning (no closing marker) is treated as a REASONING segment running to end of input. +// - Empty segments are skipped. +// - When start_marker is empty, the entire input is returned as a single DEFAULT segment. +// +// Leading whitespace/newlines on visible segments that immediately follow a closed reasoning block are trimmed — +// matches the prior strip behavior, which dropped a trailing newline after plus any leading whitespace. +static std::vector SplitReasoningContent(const std::string& text, + const std::string& start_marker, + const std::string& end_marker) { + std::vector segments; + + if (start_marker.empty() || text.empty()) { + if (!text.empty()) { + segments.push_back({text, FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT}); + } + return segments; + } + + auto push_default = [&](std::string s) { + // Trim leading whitespace from visible segments (prior strip logic dropped these). + size_t first = s.find_first_not_of(" \t\n\r"); + if (first == std::string::npos) { + return; + } + s.erase(0, first); + if (!s.empty()) { + segments.push_back({std::move(s), FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT}); + } + }; + + auto push_reasoning = [&](std::string s) { + if (!s.empty()) { + segments.push_back({std::move(s), FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING}); + } + }; + + size_t pos = 0; + while (pos < text.size()) { + size_t start_pos = text.find(start_marker, pos); + + if (start_pos == std::string::npos) { + push_default(text.substr(pos)); + break; + } + + // Visible text before the reasoning block. + push_default(text.substr(pos, start_pos - pos)); + + size_t reasoning_begin = start_pos + start_marker.size(); + size_t end_pos = text.find(end_marker, reasoning_begin); + + if (end_pos == std::string::npos) { + // Truncated — reasoning runs to end of string. Drop nothing; expose what we have. + push_reasoning(text.substr(reasoning_begin)); + break; + } + + push_reasoning(text.substr(reasoning_begin, end_pos - reasoning_begin)); + pos = end_pos + end_marker.size(); + + // Drop a single trailing newline after (matches prior strip behavior). + if (pos < text.size() && text[pos] == '\n') { + ++pos; + } + } + + return segments; +} + +void ChatSession::ProcessGeneratedOutput(std::string text, const ToolCallContext& tool_ctx, const SearchOptions& effective_options, bool canceled, Response& response, int prompt_tokens, int total_tokens, - int reasoning_tokens) { + std::vector pre_parsed_calls) { int completion_tokens = total_tokens - prompt_tokens; + + // Check if the generated text contains tool calls. If the caller has already parsed them (streaming path), reuse + // those so call_ids stay stable across stream deltas and the final response — OpenAI Chat Completions semantics. bool has_tool_calls = false; - std::vector segments; + std::vector parsed_calls; - auto flush_segments = [&]() { - if (segments.empty()) { - return; + if (!pre_parsed_calls.empty()) { + parsed_calls = std::move(pre_parsed_calls); + has_tool_calls = true; + } else if (tool_ctx.HasTools() && tool_ctx.tool_output && tool_ctx.HasToolCallTokens()) { + parsed_calls = ParseToolCalls(text, tool_ctx.tool_call_start, tool_ctx.tool_call_end, tool_ctx.tools_json); + has_tool_calls = !parsed_calls.empty(); + } + + if (has_tool_calls) { + // Add structured tool call items to the response + auto tool_items = ToolCallsToItems(parsed_calls); + for (auto& ti : tool_items) { + response.items.push_back(std::move(ti)); } + } + + // Split assistant output around reasoning markers so reasoning content can be returned to the caller as a typed + // TextItem alongside the visible response text. RenderContent in chat_template.cc skips REASONING parts when + // re-applying the template to history, so storing reasoning here doesn't contaminate subsequent prompts. + std::vector segments; + + if (tool_ctx.supports_reasoning) { + std::string start = tool_ctx.reasoning_start.empty() ? "" : tool_ctx.reasoning_start; + std::string end = tool_ctx.reasoning_end.empty() ? "" : tool_ctx.reasoning_end; + segments = SplitReasoningContent(text, start, end); + } else if (!text.empty()) { + segments.push_back({std::move(text), FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT}); + } + // Build the assistant message from the segments. Tool-call-only outputs may produce zero segments — emit no + // message in that case, since MessageItem requires non-empty content. + if (!segments.empty()) { std::unique_ptr output_item; if (segments.size() == 1 && segments.front().type == FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT) { @@ -315,22 +367,7 @@ void ChatSession::ProcessGeneratedOutput(std::vector event } response.items.push_back(std::move(output_item)); - segments.clear(); - }; - - for (auto& event : events) { - if (auto* segment = std::get_if(&event)) { - AppendSegment(segments, std::move(segment->text), segment->type); - continue; - } - - flush_segments(); - auto& call = std::get(event); - response.items.push_back(std::make_unique(std::move(call.id), std::move(call.name), - std::move(call.arguments))); - has_tool_calls = true; } - flush_segments(); if (canceled) { response.finish_reason = FOUNDRY_LOCAL_FINISH_NONE; @@ -349,12 +386,10 @@ void ChatSession::ProcessGeneratedOutput(std::vector event response.usage.prompt_tokens = prompt_tokens; response.usage.completion_tokens = completion_tokens; response.usage.total_tokens = total_tokens; - response.usage.reasoning_tokens = reasoning_tokens; logger_.Log(LogLevel::Verbose, - fmt::format( - "Completion stats: Total Tokens: {}, Prompt Tokens: {}, Completion Tokens: {}, Reasoning Tokens: {}", - total_tokens, prompt_tokens, completion_tokens, reasoning_tokens)); + fmt::format("Completion stats: Total Tokens: {}, Prompt Tokens: {}, Completion Tokens: {}", + total_tokens, prompt_tokens, completion_tokens)); } void ChatSession::ProcessRequestImpl(const Request& request, Response& response) { @@ -432,28 +467,25 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) if (cached_generator_) { // Check if guidance requirements changed since the generator was created. Guidance (LARK grammar) is baked into - // the OGA generator at creation time and cannot be changed. If tool_choice went from "required" to "auto" (or - // vice versa), we must recreate the generator from full history. + // the OGA generator at creation time and cannot be changed. auto turn_tool_ctx = cached_tool_ctx_; UpdateToolContextForTurn(request, turn_tool_ctx); - bool prev_has_user_guidance = !cached_tool_ctx_.guidance_type.empty() && !cached_tool_ctx_.guidance_data.empty(); - bool curr_has_user_guidance = !turn_tool_ctx.guidance_type.empty() && !turn_tool_ctx.guidance_data.empty(); - bool prev_needs_guidance = prev_has_user_guidance || (cached_tool_ctx_.tool_output && !cached_tool_ctx_.text_output); - bool curr_needs_guidance = curr_has_user_guidance || (turn_tool_ctx.tool_output && !turn_tool_ctx.text_output); + bool prev_needs_guidance = cached_tool_ctx_.tool_output && cached_tool_ctx_.HasTools(); + bool curr_needs_guidance = turn_tool_ctx.tool_output && turn_tool_ctx.HasTools(); - // Guidance (grammar) is baked into the OGA generator at creation time and cannot be changed. - // Rebuild when: guidance requirements changed OR the previous turn had user-specified guidance - // (the finite grammar may have completed, causing IsDone() to return true on the next turn, - // and switching schemas requires a fresh grammar). - if (prev_needs_guidance != curr_needs_guidance || prev_has_user_guidance) { + if (prev_needs_guidance != curr_needs_guidance) { // Guidance requirements changed — invalidate. The branch below will rebuild from full history. cached_generator_.reset(); cached_tool_ctx_ = {}; } else { // Continuous decoding: append only the new messages to the existing generator. pre_turn_token_count = cached_generator_->TokenCount(); - prompt_tokens = cached_generator_->AppendMessages(new_messages, Model(), cached_tool_ctx_.tools_json); + const std::string reasoning_start_marker = + cached_tool_ctx_.reasoning_start.empty() ? std::string("") : cached_tool_ctx_.reasoning_start; + prompt_tokens = cached_generator_->AppendMessages( + new_messages, Model(), cached_tool_ctx_.tools_json, + cached_tool_ctx_.supports_reasoning ? reasoning_start_marker : std::string{}); // Refresh per-turn fields (tool_choice, guidance) while keeping session-level definitions stable. UpdateToolContextForTurn(request, cached_tool_ctx_); @@ -491,18 +523,24 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) } int max_output = effective_options.max_output_tokens.value_or(0); - const auto committed_tool_ctx = cached_tool_ctx_; // Generate token-by-token with optional streaming. // Check request.canceled each iteration — a streaming callback returning // non-zero sets this flag asynchronously via CallbackHandler. + std::string text; auto streaming_callback = CreateCallbackHandler(request); int output_tokens = 0; - std::vector generated_events; - // Marker IDs are derived from the configured strings with the model tokenizer. This detects special markers even - // when their decoded chunks are empty, while non-reasoning models retain the DEFAULT passthrough. - auto splitter = CreateReasoningSplitter(cached_tool_ctx_, Model()); + // Splitter: only active for reasoning models. For non-reasoning models start_marker is empty and the splitter + // degrades to a passthrough (every token becomes one DEFAULT segment), so the streaming path stays uniform. + ReasoningStreamSplitter splitter( + cached_tool_ctx_.supports_reasoning ? (cached_tool_ctx_.reasoning_start.empty() ? std::string("") + : cached_tool_ctx_.reasoning_start) + : std::string(), + cached_tool_ctx_.supports_reasoning ? (cached_tool_ctx_.reasoning_end.empty() ? std::string("") + : cached_tool_ctx_.reasoning_end) + : std::string(), + cached_generator_ && cached_generator_->PromptEndsInReasoning()); // Accumulator: separates visible text from tool-call blocks in the DEFAULT-segment stream. For models without // tool-call markers configured, both marker strings are empty and the accumulator degrades to passthrough. @@ -510,77 +548,63 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) // model's scratchpad and is not a real tool call. ToolCallStreamAccumulator tool_accumulator( cached_tool_ctx_.tool_output ? cached_tool_ctx_.tool_call_start : std::string{}, - cached_tool_ctx_.tool_output ? cached_tool_ctx_.tool_call_end : std::string{}); + cached_tool_ctx_.tool_output ? cached_tool_ctx_.tool_call_end : std::string{}, cached_tool_ctx_.tools_json); - std::string assistant_history_text; + // Tool calls parsed during streaming. Reused by ProcessGeneratedOutput so call_ids stay stable across stream + // deltas and the final response (OpenAI Chat Completions contract). Populated even when there is no streaming + // callback — the accumulator still parses on close — but in that case the final-response path re-parses anyway, + // which is fine because the IDs only need to be stable when a client is observing the stream. + std::vector streamed_tool_calls; - auto append_history_call = [&](const ParsedToolCall& call) { - nlohmann::ordered_json rendered; - rendered["name"] = call.name; - auto arguments = nlohmann::json::parse(call.arguments, nullptr, /*allow_exceptions=*/false); - rendered["arguments"] = arguments.is_discarded() ? nlohmann::json(call.arguments) : std::move(arguments); - if (!assistant_history_text.empty() && assistant_history_text.back() != '\n') { - assistant_history_text.push_back('\n'); - } - if (committed_tool_ctx.HasToolCallTokens()) { - assistant_history_text += committed_tool_ctx.tool_call_start + "\n" + rendered.dump() + "\n" + - committed_tool_ctx.tool_call_end; - } else { - assistant_history_text += rendered.dump(); - } - }; - - auto emit_tool_output = [&](ToolCallStreamAccumulator::Output out) { - for (auto& event : out.events) { - if (auto* text = std::get_if(&event)) { - assistant_history_text += *text; - AppendGeneratedSegment(generated_events, *text, FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT); + auto emit_segments = [&](const std::vector& segments) { + for (const auto& seg : segments) { + if (seg.type == FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING) { + // REASONING goes straight through — never feed it to the tool-call accumulator. if (streaming_callback) { - streaming_callback->PushItem( - std::make_unique(*text, FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT)); + streaming_callback->PushItem(std::make_unique(seg.text, seg.type)); } continue; } - auto call = std::move(std::get(event)); - append_history_call(call); - if (streaming_callback) { - streaming_callback->PushItem(std::make_unique(call.id, call.name, call.arguments)); + auto out = tool_accumulator.Push(seg.text); + + if (streaming_callback && !out.visible_text.empty()) { + streaming_callback->PushItem( + std::make_unique(std::move(out.visible_text), FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT)); } - generated_events.push_back(std::move(call)); - } - }; - auto emit_segments = [&](const std::vector& segments) { - for (const auto& seg : segments) { - if (seg.type != FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT) { - // Release a visible prefix held as a potential tool marker before appending later reasoning. - if (!tool_accumulator.InsideToolCall()) { - emit_tool_output(tool_accumulator.Flush()); - } - // REASONING goes straight through — never feed it to the tool-call accumulator. - AppendGeneratedSegment(generated_events, seg.text, seg.type); + for (auto& pc : out.ready_calls) { if (streaming_callback) { - streaming_callback->PushItem(std::make_unique(seg.text, seg.type)); + streaming_callback->PushItem(std::make_unique(pc.id, pc.name, pc.arguments)); } - continue; + streamed_tool_calls.push_back(std::move(pc)); } - - emit_tool_output(tool_accumulator.Push(seg.text)); } }; - auto flush_accumulator = [&]() { emit_tool_output(tool_accumulator.Flush()); }; + auto flush_accumulator = [&]() { + auto out = tool_accumulator.Flush(); + + if (streaming_callback && !out.visible_text.empty()) { + streaming_callback->PushItem( + std::make_unique(std::move(out.visible_text), FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT)); + } + + for (auto& pc : out.ready_calls) { + if (streaming_callback) { + streaming_callback->PushItem(std::make_unique(pc.id, pc.name, pc.arguments)); + } + streamed_tool_calls.push_back(std::move(pc)); + } + }; while (!cached_generator_->IsDone() && !request.canceled) { cached_generator_->GenerateNextToken(); - const auto token_id = cached_generator_->CurrentTokenId(); std::string token = cached_generator_->Decode(); ++output_tokens; - if (token_id.has_value()) { - emit_segments(splitter.Push(*token_id, std::move(token))); - } else if (!token.empty()) { + if (!token.empty()) { + text += token; emit_segments(splitter.Push(token)); } @@ -604,12 +628,12 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) cached_generator_->RewindTo(pre_turn_token_count); } - ProcessGeneratedOutput(std::move(generated_events), effective_options, request.canceled, response, - prompt_tokens, total_tokens, splitter.ReasoningTokenCount()); + ProcessGeneratedOutput(std::move(text), cached_tool_ctx_, effective_options, request.canceled, + response, prompt_tokens, total_tokens, std::move(streamed_tool_calls)); // Commit input messages + assistant reply to history only on success (not cancelled) if (!request.canceled) { - // LARK grammar (tool-call-only mode) is a single-shot finite parse. If generation was truncated while grammar was + // LARK tool grammar is a single-shot finite parse. If generation was truncated while grammar was // active, the parser is in an unrecoverable state. Additionally, a completed grammar signals EOS — IsDone() would // return true on the next turn. Invalidate after any grammar-guided generation so the next turn rebuilds. // @@ -617,7 +641,7 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) // cache and the model fails to close subsequent reasoning blocks. The chat template strips prior content // when re-applied to history, so a rebuild restores correct behavior. This matches C#, which always applies the // full template per turn. - bool grammar_was_active = cached_tool_ctx_.tool_output && !cached_tool_ctx_.text_output; + bool grammar_was_active = cached_tool_ctx_.tool_output && cached_tool_ctx_.HasTools(); bool reasoning_was_active = cached_tool_ctx_.supports_reasoning; if (grammar_was_active || reasoning_was_active) { @@ -625,7 +649,7 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) cached_tool_ctx_ = {}; } - CommitTurn(std::move(new_messages), std::move(assistant_history_text), pre_turn_token_count, total_tokens); + CommitTurn(std::move(new_messages), response, pre_turn_token_count, total_tokens); // After a media turn, drop the cached generator so any text follow-up // rebuilds from history. AppendMessages cannot extend a media-decoded @@ -720,13 +744,26 @@ void ChatSession::ProcessChatCompletionsJson(const std::string& request_json, co // multiple tokens (or chat templates that produce marker-shaped text gradually) would silently fail. The shared // accumulator buffers across tokens and is verified by unit tests. ToolCallStreamAccumulator tool_accumulator(tool_ctx.tool_output ? tool_ctx.tool_call_start : std::string{}, - tool_ctx.tool_output ? tool_ctx.tool_call_end : std::string{}); - - int next_tool_call_index = 0; - std::vector generated_events; - - // Use the same typed segments for streaming and final response construction. - auto splitter = CreateReasoningSplitter(tool_ctx, Model()); + tool_ctx.tool_output ? tool_ctx.tool_call_end : std::string{}, + tool_ctx.tools_json); + + // Tool calls parsed during streaming. Reused by ProcessGeneratedOutput so call_ids stay stable across stream + // deltas and the final ChatCompletionResponse (OpenAI Chat Completions contract). + std::vector streamed_tool_calls; + + // Reasoning-aware token splitter. For non-reasoning models the splitter is a passthrough (every token becomes one + // DEFAULT segment) so the loop body is uniform. For reasoning models, REASONING segments are suppressed from the + // Chat Completions stream — the OpenAI Chat Completions spec has no reasoning-delta concept; reasoning is exposed + // via the Responses API path in Stage 4. The non-streaming response already excludes reasoning text from + // `delta.content` via the typed-MessageItem build in ProcessGeneratedOutput. + ReasoningStreamSplitter splitter( + tool_ctx.supports_reasoning ? (tool_ctx.reasoning_start.empty() ? std::string("") + : tool_ctx.reasoning_start) + : std::string(), + tool_ctx.supports_reasoning ? (tool_ctx.reasoning_end.empty() ? std::string("") + : tool_ctx.reasoning_end) + : std::string(), + generator->PromptEndsInReasoning()); auto emit_visible_text = [&](std::string visible) { if (visible.empty() || !is_streaming) { @@ -738,63 +775,58 @@ void ChatSession::ProcessChatCompletionsJson(const std::string& request_json, co FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON)); }; - auto process_tool_output = [&](ToolCallStreamAccumulator::Output out) { - for (auto& event : out.events) { - if (auto* text = std::get_if(&event)) { - AppendGeneratedSegment(generated_events, *text, FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT); - emit_visible_text(*text); - continue; - } + auto emit_ready_calls = [&](std::vector& ready) { + if (ready.empty()) { + return; + } + + if (is_streaming) { + std::vector tc_list; + tc_list.reserve(ready.size()); + int tc_index = 0; - auto call = std::move(std::get(event)); - if (is_streaming) { - ChatCompletionToolCall streamed; - streamed.index = next_tool_call_index++; - streamed.id = call.id; - streamed.type = "function"; - streamed.function.name = call.name; - streamed.function.arguments = call.arguments; - auto chunk_json = chat_completions::FormatToolCallStreamingChunk( - {streamed}, completion_id, created, model_name); - streaming_callback->PushItem(std::make_unique( - std::move(chunk_json), FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON)); + for (const auto& pc : ready) { + ChatCompletionToolCall tc; + tc.index = tc_index++; + tc.id = pc.id; + tc.type = "function"; + tc.function.name = pc.name; + tc.function.arguments = pc.arguments; + tc_list.push_back(std::move(tc)); } - generated_events.push_back(std::move(call)); + + auto chunk_json = chat_completions::FormatToolCallStreamingChunk(tc_list, completion_id, created, model_name); + streaming_callback->PushItem(std::make_unique(std::move(chunk_json), + FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON)); + } + + for (auto& pc : ready) { + streamed_tool_calls.push_back(std::move(pc)); } }; auto process_segments = [&](const std::vector& segments) { for (const auto& seg : segments) { - // REASONING segments: never feed reasoning text to the tool-call accumulator — tool-call-shaped text inside - // ... is scratchpad, not a real call. Emit via reasoning_content, not content. + // REASONING segments: intentionally dropped from the Chat Completions stream. Never feed reasoning text to + // the tool-call accumulator — tool-call-shaped text inside ... is scratchpad, not a real call. if (seg.type != FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT) { - if (!tool_accumulator.InsideToolCall()) { - process_tool_output(tool_accumulator.Flush()); - } - AppendGeneratedSegment(generated_events, seg.text, seg.type); - - if (is_streaming && !seg.text.empty()) { - auto chunk_json = chat_completions::FormatReasoningStreamingChunk( - seg.text, completion_id, created, model_name); - streaming_callback->PushItem(std::make_unique( - std::move(chunk_json), FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON)); - } continue; } - process_tool_output(tool_accumulator.Push(seg.text)); + auto out = tool_accumulator.Push(seg.text); + emit_visible_text(std::move(out.visible_text)); + emit_ready_calls(out.ready_calls); } }; - // Generate token-by-token. + // Generate token-by-token + std::string text; while (!generator->IsDone() && !original_request.canceled) { generator->GenerateNextToken(); - const auto token_id = generator->CurrentTokenId(); std::string token = generator->Decode(); - if (token_id.has_value()) { - process_segments(splitter.Push(*token_id, std::move(token))); - } else if (!token.empty()) { + if (!token.empty()) { + text += token; process_segments(splitter.Push(token)); } } @@ -802,12 +834,19 @@ void ChatSession::ProcessChatCompletionsJson(const std::string& request_json, co // Drain any buffered partial-marker bytes at end-of-stream. Reasoning splitter first so any final DEFAULT bytes // feed into the tool accumulator; then drain the tool accumulator. process_segments(splitter.Flush()); - process_tool_output(tool_accumulator.Flush()); + { + auto out = tool_accumulator.Flush(); + emit_visible_text(std::move(out.visible_text)); + emit_ready_calls(out.ready_calls); + } int total_tokens = generator->TokenCount(); - ProcessGeneratedOutput(std::move(generated_events), options, original_request.canceled, response, - prompt_tokens, total_tokens, splitter.ReasoningTokenCount()); + // Process the generated output into response items (MessageItem, ToolCallItem, etc.) + // This also updates finish_reason, and usage on the response. Streamed-parsed tool calls are reused so call_ids + // stay stable across stream deltas and the final ChatCompletionResponse. + ProcessGeneratedOutput(std::move(text), tool_ctx, options, original_request.canceled, + response, prompt_tokens, total_tokens, std::move(streamed_tool_calls)); // Emit final streaming chunk with finish_reason if (is_streaming) { @@ -833,7 +872,7 @@ const std::vector& ChatSession::GetHistory() const { return history_; } -void ChatSession::CommitTurn(std::vector&& new_messages, std::string assistant_history, +void ChatSession::CommitTurn(std::vector&& new_messages, const Response& response, int pre_turn_token_count, int post_turn_token_count) { size_t history_start = history_.size(); size_t input_count = new_messages.size(); @@ -843,14 +882,29 @@ void ChatSession::CommitTurn(std::vector&& new_messages, std::strin history_.push_back(std::move(msg)); } - if (assistant_history.empty()) { - // A successful turn still owns an assistant role when all generated content was hidden reasoning. Preserve the - // role boundary so rebuilding the next turn never produces consecutive user messages. - MessageItem assistant; - assistant.role = FOUNDRY_LOCAL_ROLE_ASSISTANT; - history_.push_back(std::move(assistant)); - } else { - history_.emplace_back(FOUNDRY_LOCAL_ROLE_ASSISTANT, std::move(assistant_history)); + // Commit the assistant reply and its tool calls as one template message. A + // reused Responses session does not replay previous_output, so history must + // retain the assistant call that precedes the next tool result. + std::optional assistant_reply; + for (const auto& item : response.items) { + if (item->type == FOUNDRY_LOCAL_ITEM_MESSAGE) { + const auto& msg = static_cast(*item); + if (msg.role == FOUNDRY_LOCAL_ROLE_ASSISTANT && !msg.content.empty()) { + auto tool_calls = assistant_reply ? std::move(assistant_reply->tool_calls) + : std::vector{}; + assistant_reply = msg; + assistant_reply->tool_calls = std::move(tool_calls); + } + } else if (item->type == FOUNDRY_LOCAL_ITEM_TOOL_CALL) { + if (!assistant_reply) { + assistant_reply.emplace(); + assistant_reply->role = FOUNDRY_LOCAL_ROLE_ASSISTANT; + } + assistant_reply->tool_calls.push_back(static_cast(*item)); + } + } + if (assistant_reply) { + history_.push_back(std::move(*assistant_reply)); } turns_.push_back({history_start, input_count, pre_turn_token_count, post_turn_token_count}); diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc index 8fbd3f4ed..38333ed1d 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc @@ -41,9 +41,7 @@ std::string RenderMessageForPrompt(const MessageItem& msg) { return text; } -std::string BuildChatPrompt(const std::vector& messages, - GenAIModelInstance& model, - const std::string& tools_json) { +std::string BuildChatMessagesJson(const std::vector& messages) { if (messages.empty()) { FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "messages must not be empty"); } @@ -52,10 +50,26 @@ std::string BuildChatPrompt(const std::vector& messages, // Format: [{"role": "system", "content": "..."}, {"role": "user", "content": "..."}, ...] nlohmann::json messages_json = nlohmann::json::array(); for (const auto& msg : messages) { - messages_json.push_back({{"role", Utils::RoleToString(msg.role)}, {"content", RenderMessageForPrompt(msg)}}); + nlohmann::json message = { + {"role", Utils::RoleToString(msg.role)}, {"content", RenderMessageForPrompt(msg)}}; + if (!msg.tool_calls.empty()) { + message["tool_calls"] = nlohmann::json::array(); + for (const auto& tool_call : msg.tool_calls) { + message["tool_calls"].push_back({ + {"name", tool_call.name}, {"arguments", tool_call.arguments}}); + } + } + messages_json.push_back(std::move(message)); } - std::string messages_str = messages_json.dump(); + return messages_json.dump(); +} + +std::string BuildChatPrompt(const std::vector& messages, + GenAIModelInstance& model, + const std::string& tools_json) { + std::string messages_str = BuildChatMessagesJson(messages); + const char* tools_ptr = tools_json.empty() ? nullptr : tools_json.c_str(); // ApplyChatTemplate uses the model's built-in template (template_str=nullptr) and appends the assistant diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h index 7880e9a12..be703c0f6 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h @@ -28,6 +28,9 @@ class GenAIModelInstance; /// in one place. std::string RenderMessageForPrompt(const MessageItem& msg); +/// Serialize messages into the JSON shape consumed by the model chat template. +std::string BuildChatMessagesJson(const std::vector& messages); + /// Build a chat prompt string from a list of messages. /// Uses the tokenizer's built-in chat template (via GenAIModelInstance::ApplyChatTemplate). /// diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc index c842a2bf0..482e71493 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc @@ -12,8 +12,36 @@ #include #include +#include + namespace fl { +namespace { + +// Reasoning-model chat templates (Qwen3, DeepSeek-R1, etc.) can pre-fill the +// reasoning open marker as part of the assistant generation prompt, e.g. +// `<|im_start|>assistant\n\n`. When this happens the model's first +// generated byte is reasoning content — never the marker itself. The stream +// splitter must start in the reasoning state or it will forward chain-of-thought +// tokens to the caller as visible text and leak `` when the model closes +// the block. +bool PromptOpensReasoning(const std::string& prompt, const std::string& reasoning_start) { + if (reasoning_start.empty() || prompt.empty()) { + return false; + } + const size_t end = prompt.find_last_not_of(" \t\r\n"); + if (end == std::string::npos) { + return false; + } + const size_t trimmed_len = end + 1; + if (trimmed_len < reasoning_start.size()) { + return false; + } + return prompt.compare(trimmed_len - reasoning_start.size(), reasoning_start.size(), reasoning_start) == 0; +} + +} // namespace + OnnxChatGenerator::~OnnxChatGenerator() = default; // --------------------------------------------------------------------------- @@ -23,15 +51,19 @@ OnnxChatGenerator::~OnnxChatGenerator() = default; OnnxChatGenerator::OnnxChatGenerator(std::unique_ptr gen_params, std::unique_ptr generator, std::unique_ptr stream, + std::unique_ptr stream_with_special, GenAIModelInstance& model, int prompt_token_count, + bool prompt_ends_in_reasoning, std::unique_ptr named_tensors) : gen_params_(std::move(gen_params)), generator_(std::move(generator)), stream_(std::move(stream)), + stream_with_special_(std::move(stream_with_special)), named_tensors_(std::move(named_tensors)), model_(model), - prompt_token_count_(prompt_token_count) {} + prompt_token_count_(prompt_token_count), + prompt_ends_in_reasoning_(prompt_ends_in_reasoning) {} // --------------------------------------------------------------------------- // ChatGenerator interface @@ -50,20 +82,11 @@ bool OnnxChatGenerator::IsDone() const { void OnnxChatGenerator::GenerateNextToken() { if (cancelled_) { - current_token_.reset(); return; } - current_token_.reset(); - try { generator_->GenerateNextToken(); - - // GetNextTokens returns the batch of next tokens; chat generation always uses batch size 1. - const auto next_tokens = generator_->GetNextTokens(); - if (!next_tokens.empty()) { - current_token_ = next_tokens[0]; - } } catch (const std::runtime_error& e) { // If cancelled while generating, the OGA engine throws when the session is terminated. // This is expected — not an error. @@ -76,41 +99,45 @@ void OnnxChatGenerator::GenerateNextToken() { } std::string OnnxChatGenerator::Decode() { - if (cancelled_ || !current_token_.has_value()) { + if (cancelled_) { return ""; } - const auto token_id = *current_token_; - current_token_.reset(); + // Get the most recently generated token ID. + // GetNextTokens returns the batch of next tokens; we use index 0 (batch size = 1). + auto next_tokens = generator_->GetNextTokens(); - // Fast path: if this token matches a known tag ID, return the pre-decoded string. - // Decode is always single-stream for normal tokens. - const auto& tag_info = model_.GetTagInfo(); - - if (tag_info.bot_id.has_value() && token_id == *tag_info.bot_id) { - stream_->Decode(token_id); - return tag_info.bot_str; - } - if (tag_info.eot_id.has_value() && token_id == *tag_info.eot_id) { - stream_->Decode(token_id); - return tag_info.eot_str; - } - if (tag_info.bor_id.has_value() && token_id == *tag_info.bor_id) { - stream_->Decode(token_id); - return tag_info.bor_str; - } - if (tag_info.eor_id.has_value() && token_id == *tag_info.eor_id) { - stream_->Decode(token_id); - return tag_info.eor_str; + if (next_tokens.empty()) { + return ""; } - // Single decode for all non-tag tokens. + int32_t token_id = next_tokens[0]; + + // Decode through the normal tokenizer stream const char* token_text = stream_->Decode(token_id); - return token_text ? std::string(token_text) : ""; -} -std::optional OnnxChatGenerator::CurrentTokenId() const { - return current_token_; + // Also decode through the special-token stream to detect tool call and think tokens. + // If the special stream gives a different result and it's a known special token type + // that isn't an EOS token, surface the special representation instead. + // Matches C# OnnxChatGenerator.Decode behavior. + const char* special_text = stream_with_special_->Decode(token_id); + + std::string token_str = token_text ? std::string(token_text) : ""; + + if (special_text != nullptr && token_text != nullptr && std::string(special_text) != token_str) { + std::string special_str(special_text); + bool is_tool_call_token = special_str.find("tool_call") != std::string::npos; + bool is_think_token = special_str.find("think") != std::string::npos; + + const auto& eos_ids = model_.GetPreprocessor().GetEosTokenIds(); + bool is_eos = std::find(eos_ids.begin(), eos_ids.end(), token_id) != eos_ids.end(); + + if (!is_eos && (is_tool_call_token || is_think_token)) { + return special_str; + } + } + + return token_str; } int OnnxChatGenerator::TokenCount() const { @@ -139,7 +166,8 @@ void OnnxChatGenerator::Cancel() { int OnnxChatGenerator::AppendMessages(const std::vector& new_messages, GenAIModelInstance& model, - const std::string& tools_json) { + const std::string& tools_json, + const std::string& reasoning_start) { if (new_messages.empty()) { FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "new_messages must not be empty"); } @@ -156,6 +184,12 @@ int OnnxChatGenerator::AppendMessages(const std::vector& new_messag FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, std::string("failed to append token sequences: ") + e.what()); } + // The assistant generation prompt is re-emitted for the new turn. Update the + // reasoning-prefill hint so the next stream splitter starts in the right state. + if (!reasoning_start.empty()) { + prompt_ends_in_reasoning_ = PromptOpensReasoning(prompt, reasoning_start); + } + return new_token_count; } @@ -272,6 +306,13 @@ std::unique_ptr OnnxChatGenerator::CreateImpl(const std::vect FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "model has no multimodal processor available for media input"); } + + // Match upstream's single-image limit. Easy to relax once the wider + // pipeline (and ORT GenAI templates) reliably handle multi-image inputs. + if (images.size() > 1) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, + "only one image per request is supported"); + } } // 1. Build the chat prompt using the model's template. @@ -381,17 +422,12 @@ std::unique_ptr OnnxChatGenerator::CreateImpl(const std::vect } } - // Guard: apply guidance based on its source. - // User-specified guidance (via response_format) is always applied — the user explicitly requested it. - // Auto-generated tool grammar is only applied for tool-call-only mode (tool output requested, no text output). - // Text-only reasoning (cot_text_only) cannot use auto-generated grammar because a completed grammar signals EOS - // to the ORT GenAI generator — making IsDone() return true immediately on the next turn, breaking multi-turn - // continuous decoding. For tool-call-only mode the generator is typically invalidated after a successful call - // anyway, so this is acceptable. - bool user_specified_guidance = !tool_ctx.guidance_type.empty() && !tool_ctx.guidance_data.empty(); - bool tool_call_only = tool_ctx.tool_output && !tool_ctx.text_output; - - if (!guidance_type.empty() && !guidance_data.empty() && (user_specified_guidance || tool_call_only)) { + // Apply auto-generated tool guidance for both auto and required tool choice. The grammar itself permits text when + // text_output is true. ChatSession invalidates the generator after guided turns because a completed finite grammar + // signals EOS and cannot be reused safely for continuous decoding. + bool tool_guidance_enabled = tool_ctx.tool_output && tool_ctx.HasTools(); + + if (!guidance_type.empty() && !guidance_data.empty() && tool_guidance_enabled) { try { gen_params->SetGuidance(guidance_type.c_str(), guidance_data.c_str()); } catch (const std::runtime_error& e) { @@ -418,16 +454,30 @@ std::unique_ptr OnnxChatGenerator::CreateImpl(const std::vect FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, std::string("failed to create generator: ") + e.what()); } - // 7. Create tokenizer stream (single-decode path). + // 7. Create two tokenizer streams: + // - Normal stream: standard decoding (special tokens filtered) + // - Special stream: includes special tokens (for tool call detection) + auto stream = model.GetPreprocessor().CreateTokenizerStream(); + auto stream_with_special = model.GetPreprocessor().CreateSpecialTokenizerStream(); + + // Detect whether the chat template's assistant generation prompt pre-fills + // the reasoning open marker. If so, the model starts generating inside a + // reasoning block; the stream splitter must be initialized accordingly. + const std::string reasoning_start_marker = + tool_ctx.reasoning_start.empty() ? std::string("") : tool_ctx.reasoning_start; + const bool prompt_ends_in_reasoning = + tool_ctx.supports_reasoning && PromptOpensReasoning(prompt, reasoning_start_marker); // `std::make_unique` constructs inside the library helper, which does not have // access to this class's private constructor. return std::unique_ptr(new OnnxChatGenerator(std::move(gen_params), std::move(generator), std::move(stream), + std::move(stream_with_special), model, input_token_count, + prompt_ends_in_reasoning, std::move(named_tensors))); } diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.h b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.h index 7d5ab1408..478949786 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.h @@ -13,7 +13,6 @@ #include #include #include -#include #include #include @@ -39,17 +38,35 @@ class OnnxChatGenerator : public ChatGenerator { bool IsDone() const override; void GenerateNextToken() override; std::string Decode() override; - std::optional CurrentTokenId() const override; int TokenCount() const override; int PromptTokenCount() const override; void Cancel() override; + /// True when the most recently rendered prompt ends with the reasoning open + /// marker (e.g. `\n`) supplied by the chat template's generation prompt. + /// Reasoning-model chat templates pre-fill this marker so the model's first + /// generated token is reasoning content rather than the marker itself. The + /// stream splitter consumes this flag to start in the reasoning state. + bool PromptEndsInReasoning() const { return prompt_ends_in_reasoning_; } + /// Encode new messages and append their tokens to the generator's sequence. /// Used for continuous decoding — only the new turn's messages are encoded and appended. /// Returns the number of new prompt tokens appended. + /// + /// @param reasoning_start Reasoning-open marker (e.g. ``) used to detect + /// whether the chat template prefilled it as part of the assistant + /// generation prompt. Pass an empty string to skip the detection; the + /// reasoning-prefill flag is left unchanged in that case. int AppendMessages(const std::vector& new_messages, GenAIModelInstance& model, - const std::string& tools_json); + const std::string& tools_json, + const std::string& reasoning_start = {}); + + /// Recompute whether the appended prompt ends in an open reasoning block. + /// Callers must invoke this after `AppendMessages` when they use the streaming + /// splitter — the assistant generation prompt is re-emitted each turn and may + /// or may not prefill the reasoning open marker depending on the template. + void SetPromptEndsInReasoning(bool value) { prompt_ends_in_reasoning_ = value; } /// Rewind the generator to a previous token position. /// Used for error recovery — restores the KV cache to the state before the last turn. @@ -96,8 +113,10 @@ class OnnxChatGenerator : public ChatGenerator { OnnxChatGenerator(std::unique_ptr gen_params, std::unique_ptr generator, std::unique_ptr stream, + std::unique_ptr stream_with_special, GenAIModelInstance& model, int prompt_token_count, + bool prompt_ends_in_reasoning, std::unique_ptr named_tensors = nullptr); // Shared implementation for text and media creation paths. Empty image and @@ -115,6 +134,7 @@ class OnnxChatGenerator : public ChatGenerator { std::unique_ptr gen_params_; std::unique_ptr generator_; std::unique_ptr stream_; + std::unique_ptr stream_with_special_; // for tool call token detection // Holds the named tensors produced by OgaMultiModalProcessor media processing // for the lifetime of the generator. Generator retains shared_ptr // copies internally, but we keep the wrapper alive for symmetry with @@ -122,7 +142,7 @@ class OnnxChatGenerator : public ChatGenerator { std::unique_ptr named_tensors_; GenAIModelInstance& model_; // non-owning reference — model outlives generator int prompt_token_count_ = 0; - std::optional current_token_; + bool prompt_ends_in_reasoning_ = false; std::atomic cancelled_{false}; }; diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/reasoning_stream_splitter.h b/sdk_v2/cpp/src/inferencing/generative/chat/reasoning_stream_splitter.h index a9e9e72eb..d01af8a64 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/reasoning_stream_splitter.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/reasoning_stream_splitter.h @@ -4,23 +4,25 @@ #include "foundry_local/foundry_local_c.h" -#include -#include -#include #include #include #include namespace fl { -/// Token-aware state machine that splits generated output around reasoning markers into typed segments. +/// Token-level state machine that splits a stream of generated text chunks around reasoning markers +/// (e.g. `` / ``) into typed segments. /// -/// Marker token IDs come from ORT GenAI's model metadata. Matching IDs before inspecting decoded text is required -/// for special tokens, whose decoded chunks can be empty when the tokenizer skips special tokens. Token-prefix -/// buffering also supports compatibility callers that provide markers composed of multiple token IDs. +/// The streaming code calls `Push(token)` for every decoded token and forwards the returned segments to the +/// caller as typed `TextItem`s. At end-of-generation, `Flush()` drains any buffered bytes (e.g. a trailing +/// partial marker that turned out not to be one). /// -/// The text-only Push overload preserves the prior decoded-marker behavior for callers without token IDs. When -/// `start_marker` is empty, both modes degrade to a DEFAULT passthrough for non-reasoning models. +/// Why a state machine: the marker can straddle multiple tokens (a tokenizer might split `` into +/// ``). We must not emit the partial prefix as visible text and then realize on the next +/// token that it was actually a marker. The buffer holds the suffix that could still grow into the marker. +/// +/// When `start_marker` is empty, the splitter degrades to a passthrough that always emits DEFAULT segments — +/// non-reasoning models share this code without a behavior change. class ReasoningStreamSplitter { public: struct Segment { @@ -28,56 +30,46 @@ class ReasoningStreamSplitter { flTextItemType type; }; - ReasoningStreamSplitter(std::string start_marker, - std::string end_marker, - std::vector start_token_ids = {}, - std::vector end_token_ids = {}, - std::vector ignored_token_ids = {}) + /// @param start_inside_reasoning True when the chat template pre-fills the reasoning + /// open marker (e.g. `\n`) before the model's first token. In that case + /// the model's first byte is reasoning content and the splitter must start in + /// the reasoning state instead of waiting for a start marker that will never + /// appear. + ReasoningStreamSplitter(std::string start_marker, std::string end_marker, + bool start_inside_reasoning = false) : start_marker_(std::move(start_marker)), end_marker_(std::move(end_marker)), - start_token_ids_(std::move(start_token_ids)), - end_token_ids_(std::move(end_token_ids)), - ignored_token_ids_(std::move(ignored_token_ids)) {} - - /// Feed one generated token into the splitter. Marker IDs are consumed even when decoded_text is empty. - std::vector Push(int32_t token_id, std::string decoded_text) { - if (!HasTextMarkers()) { - if (decoded_text.empty() || IsIgnoredToken(token_id)) { - return {}; - } + inside_reasoning_(start_inside_reasoning) {} - return {{std::move(decoded_text), FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT}}; + /// Feed a token into the splitter. Returns zero or more segments to emit. + std::vector Push(const std::string& token) { + std::vector out; + + if (token.empty()) { + return out; } - if (!HasTokenMarkers()) { - return PushText(decoded_text, IsIgnoredToken(token_id)); + if (start_marker_.empty()) { + out.push_back({token, FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT}); + return out; } - std::vector out; - pending_tokens_.push_back({token_id, std::move(decoded_text)}); - DrainTokens(out, /*flushing=*/false); - return out; - } + buffer_ += token; + Drain(out, /*flushing=*/false); - /// Feed a decoded token into the text-only fallback. - std::vector Push(const std::string& token) { - return PushText(token, false); + return out; } - /// Drain pending content at end-of-generation. A partial marker is content in the current reasoning state. + /// Drain any remaining buffered bytes at end-of-stream. Buffered bytes that looked like a partial marker + /// turn out not to be — emit them with the current type. std::vector Flush() { std::vector out; - if (!HasTextMarkers()) { + if (start_marker_.empty()) { return out; } - if (HasTokenMarkers()) { - DrainTokens(out, /*flushing=*/true); - DrainText(out, /*flushing=*/true); - } else { - DrainText(out, /*flushing=*/true); - } + Drain(out, /*flushing=*/true); return out; } @@ -86,167 +78,8 @@ class ReasoningStreamSplitter { /// downstream decisions (e.g. suppressing chunks) without inspecting segment types. bool InsideReasoning() const noexcept { return inside_reasoning_; } - /// Number of generated content tokens classified as reasoning. Boundary marker tokens are excluded. - int ReasoningTokenCount() const noexcept { return reasoning_token_count_; } - private: - struct PendingToken { - int32_t id; - std::string text; - }; - - struct PendingTextToken { - std::string text; - bool reasoning_counted = false; - bool ignored = false; - }; - - bool HasTextMarkers() const noexcept { - return !start_marker_.empty() && !end_marker_.empty(); - } - - bool HasTokenMarkers() const noexcept { - return !start_token_ids_.empty() && !end_token_ids_.empty(); - } - - std::vector PushText(const std::string& token, bool ignored) { - std::vector out; - - if (token.empty()) { - return out; - } - - if (!HasTextMarkers()) { - if (!ignored) { - out.push_back({token, FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT}); - } - return out; - } - - pending_text_tokens_.push_back({token, false, ignored}); - text_buffer_ += token; - DrainText(out, /*flushing=*/false); - return out; - } - - void DrainTokens(std::vector& out, bool flushing) { - while (!pending_tokens_.empty()) { - const auto& marker = inside_reasoning_ ? end_token_ids_ : start_token_ids_; - const auto found = FindTokenSequence(pending_tokens_, marker); - - if (found < pending_tokens_.size()) { - const auto state_before_prefix = inside_reasoning_; - EmitPendingTokens(out, found); - if (inside_reasoning_ != state_before_prefix) { - continue; - } - - // A decoded-marker prefix buffered before this ID marker is ordinary content because the complete boundary - // is represented by the IDs below. - DrainText(out, /*flushing=*/true); - if (inside_reasoning_ != state_before_prefix) { - continue; - } - - pending_tokens_.erase( - pending_tokens_.begin(), - pending_tokens_.begin() + static_cast(marker.size())); - inside_reasoning_ = !inside_reasoning_; - trim_default_prefix_ = !inside_reasoning_; - continue; - } - - if (flushing) { - const auto state_before_flush = inside_reasoning_; - EmitPendingTokens(out, pending_tokens_.size()); - if (inside_reasoning_ == state_before_flush) { - return; - } - - continue; - } - - const auto hold = LongestTokenSuffixThatIsPrefixOf(pending_tokens_, marker); - const auto safe = pending_tokens_.size() - hold; - const auto state_before_safe_tokens = inside_reasoning_; - EmitPendingTokens(out, safe); - if (inside_reasoning_ != state_before_safe_tokens) { - continue; - } - - return; - } - } - - void EmitPendingTokens(std::vector& out, size_t count) { - if (count == 0) { - return; - } - - for (size_t i = 0; i < count; ++i) { - auto token = std::move(pending_tokens_.front()); - pending_tokens_.erase(pending_tokens_.begin()); - const auto was_inside_reasoning = inside_reasoning_; - - if (IsIgnoredToken(token.id)) { - // EOS and configured control tokens are neither reasoning content nor visible output, regardless of how - // the tokenizer chooses to decode them. - } else if (token.text.empty()) { - if (inside_reasoning_) { - ++reasoning_token_count_; - } - } else { - pending_text_tokens_.push_back({token.text}); - text_buffer_ += token.text; - DrainText(out, /*flushing=*/false); - } - - if (inside_reasoning_ != was_inside_reasoning) { - return; - } - } - } - - static size_t FindTokenSequence(const std::vector& tokens, - const std::vector& marker) { - if (marker.empty() || tokens.size() < marker.size()) { - return tokens.size(); - } - - for (size_t pos = 0; pos + marker.size() <= tokens.size(); ++pos) { - const auto matches = std::equal( - marker.begin(), marker.end(), tokens.begin() + static_cast(pos), - [](int32_t marker_id, const PendingToken& token) { return marker_id == token.id; }); - if (matches) { - return pos; - } - } - - return tokens.size(); - } - - bool IsIgnoredToken(int32_t token_id) const { - return std::find(ignored_token_ids_.begin(), ignored_token_ids_.end(), token_id) != - ignored_token_ids_.end(); - } - - static size_t LongestTokenSuffixThatIsPrefixOf(const std::vector& tokens, - const std::vector& marker) { - const auto max_length = std::min(tokens.size(), marker.size()); - for (size_t length = max_length; length > 0; --length) { - const auto token_start = tokens.end() - static_cast(length); - const auto matches = std::equal( - marker.begin(), marker.begin() + static_cast(length), token_start, - [](int32_t marker_id, const PendingToken& token) { return marker_id == token.id; }); - if (matches) { - return length; - } - } - - return 0; - } - - void DrainText(std::vector& out, bool flushing) { + void Drain(std::vector& out, bool flushing) { while (true) { const std::string& marker = inside_reasoning_ ? end_marker_ : start_marker_; flTextItemType current_type = inside_reasoning_ ? FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING @@ -255,89 +88,51 @@ class ReasoningStreamSplitter { // Marker may be empty (e.g. end_marker not configured). With no end marker we can never close a // reasoning block — drain the buffer with the current type and stop. if (marker.empty()) { - EmitTextSegment(out, ConsumeText(text_buffer_.size(), current_type, /*is_content=*/true), current_type); + EmitSegment(out, std::move(buffer_), current_type); + buffer_.clear(); return; } - size_t found = text_buffer_.find(marker); + size_t found = buffer_.find(marker); if (found != std::string::npos) { // Emit prefix with current type, consume marker, flip state. - EmitTextSegment(out, ConsumeText(found, current_type, /*is_content=*/true), current_type); - ConsumeText(marker.size(), current_type, /*is_content=*/false); - const auto closed_reasoning = inside_reasoning_; - inside_reasoning_ = !inside_reasoning_; - trim_default_prefix_ = !inside_reasoning_; + EmitSegment(out, buffer_.substr(0, found), current_type); + + size_t after = found + marker.size(); - // Preserve the established behavior of dropping a newline immediately after a closed reasoning block. - if (closed_reasoning && !text_buffer_.empty() && text_buffer_.front() == '\n') { - ConsumeText(1, FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT, /*is_content=*/false); - trim_default_prefix_ = false; + // Drop a single trailing newline immediately after the closing marker — matches the non-streaming + // SplitReasoningContent behavior so callers see the same visible text either way. + if (inside_reasoning_ && after < buffer_.size() && buffer_[after] == '\n') { + ++after; } + buffer_.erase(0, after); + inside_reasoning_ = !inside_reasoning_; + continue; // re-scan the remaining buffer for the next marker } // No full marker. If we're flushing, emit everything and stop. Otherwise hold back the longest suffix // of buffer_ that could still grow into the marker. if (flushing) { - EmitTextSegment(out, ConsumeText(text_buffer_.size(), current_type, /*is_content=*/true), current_type); + EmitSegment(out, std::move(buffer_), current_type); + buffer_.clear(); return; } - size_t hold = LongestSuffixThatIsPrefixOf(text_buffer_, marker); - size_t safe = text_buffer_.size() - hold; + size_t hold = LongestSuffixThatIsPrefixOf(buffer_, marker); + size_t safe = buffer_.size() - hold; if (safe > 0) { - EmitTextSegment(out, ConsumeText(safe, current_type, /*is_content=*/true), current_type); + EmitSegment(out, buffer_.substr(0, safe), current_type); + buffer_.erase(0, safe); } return; } } - std::string ConsumeText(size_t length, flTextItemType type, bool is_content) { - std::string text; - text.reserve(length); - text_buffer_.erase(0, length); - - auto remaining = length; - while (remaining > 0 && !pending_text_tokens_.empty()) { - auto& token = pending_text_tokens_.front(); - const auto consumed = std::min(remaining, token.text.size()); - - // Ignored token bytes participate in marker matching but are never emitted or counted. - if (is_content && !token.ignored) { - text.append(token.text, 0, consumed); - if (type == FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING && !token.reasoning_counted) { - ++reasoning_token_count_; - token.reasoning_counted = true; - } - } - - token.text.erase(0, consumed); - remaining -= consumed; - if (token.text.empty()) { - pending_text_tokens_.erase(pending_text_tokens_.begin()); - } - } - - return text; - } - - void EmitTextSegment(std::vector& out, std::string text, flTextItemType type) { - if (type == FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT && trim_default_prefix_ && !text.empty()) { - if (text.starts_with("\r\n")) { - text.erase(0, 2); - } else if (text.starts_with('\n')) { - text.erase(0, 1); - } - trim_default_prefix_ = false; - } - - EmitSegment(out, std::move(text), type); - } - static void EmitSegment(std::vector& out, std::string text, flTextItemType type) { if (text.empty()) { return; @@ -361,15 +156,8 @@ class ReasoningStreamSplitter { std::string start_marker_; std::string end_marker_; - std::vector start_token_ids_; - std::vector end_token_ids_; - std::vector ignored_token_ids_; - std::vector pending_tokens_; - std::vector pending_text_tokens_; - std::string text_buffer_; + std::string buffer_; bool inside_reasoning_ = false; - bool trim_default_prefix_ = false; - int reasoning_token_count_ = 0; }; } // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc index cb76d52d4..4ef1470cb 100644 --- a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc +++ b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc @@ -328,6 +328,9 @@ static void AddTypedInputItems(Request& request, if (auto* fc_result = std::get_if(&input_item)) { auto i = std::make_unique(fc_result->call_id, fc_result->output); request.AddOwnedItem(std::move(i)); + } else if (auto* fc = std::get_if(&input_item)) { + auto i = std::make_unique(fc->call_id, fc->name, fc->arguments); + request.AddOwnedItem(std::move(i)); } else if (auto* msg = std::get_if(&input_item)) { // Build typed parts from the message's content array. std::vector> parts; diff --git a/sdk_v2/cpp/src/inferencing/generative/toolcalling/grammar.cc b/sdk_v2/cpp/src/inferencing/generative/toolcalling/grammar.cc index 49430c018..08a6f296a 100644 --- a/sdk_v2/cpp/src/inferencing/generative/toolcalling/grammar.cc +++ b/sdk_v2/cpp/src/inferencing/generative/toolcalling/grammar.cc @@ -382,7 +382,13 @@ std::string BuildLarkGrammar(const ToolCallContext& ctx, // Add grammar for text output if (ctx.text_output) { - grammar << "TEXT: /[^{<](.|\\n)*/\n"; + if (ctx.tool_output) { + // Keep marker-delimited calls out of the free-text branch so they must + // pass through the schema-constrained functioncall rule. + grammar << (known_tool_tokens ? "TEXT: /[^{<][^<]*/\n" : "TEXT: /[^{<][^{]*/\n"); + } else { + grammar << "TEXT: /[^{<](.|\\n)*/\n"; + } } // Add grammar for tool output diff --git a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_stream_accumulator.h b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_stream_accumulator.h index 91a1f84cc..95a1de8b7 100644 --- a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_stream_accumulator.h +++ b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_stream_accumulator.h @@ -7,7 +7,6 @@ #include #include #include -#include #include namespace fl { @@ -20,10 +19,10 @@ namespace fl { /// across tokens, parse it once the closing marker arrives, and surface the structured `ParsedToolCall`s. /// /// `Push(chunk)` accepts any text chunk (a single decoded token, or a multi-token segment produced by the upstream -/// `ReasoningStreamSplitter`) and returns ordered events containing: -/// - text that is safe to emit to the caller (everything outside a tool-call block, minus any pending suffix that -/// could still grow into the start marker); -/// - fully parsed tool calls whose closing marker arrived in this chunk. +/// `ReasoningStreamSplitter`) and returns: +/// - `visible_text`: text that is safe to emit to the caller (everything outside a tool-call block, minus any +/// pending suffix that could still grow into the start marker). +/// - `ready_calls`: zero or more fully parsed tool calls whose closing marker arrived in this chunk. /// /// Marker matching is buffered, mirroring `ReasoningStreamSplitter`: a marker can straddle multiple tokens, so the /// accumulator holds back the longest suffix of its scan buffer that could still extend into the marker rather than @@ -33,24 +32,25 @@ namespace fl { /// returned as visible text — they turned out not to be a tool call, so the caller still sees what the model /// produced. Matches `ReasoningStreamSplitter::Flush()`. /// -/// When either marker is empty, the accumulator degrades to a passthrough and returns its input as a text event. -/// This keeps the call site uniform for non-tool-calling models. +/// When either marker is empty, the accumulator degrades to a passthrough: `Push` returns its input verbatim as +/// `visible_text` with no `ready_calls`. This keeps the call site uniform for non-tool-calling models. /// /// Callers must not feed REASONING-tagged content into `Push` — reasoning is the model's scratchpad and any /// tool-call-shaped text inside `...` is not a real tool call. The upstream `ReasoningStreamSplitter` /// already routes REASONING segments through a separate path; this accumulator sits below the DEFAULT-segment branch. class ToolCallStreamAccumulator { public: - using Event = std::variant; - struct Output { - std::vector events; + std::string visible_text; + std::vector ready_calls; }; - ToolCallStreamAccumulator(std::string start_marker, std::string end_marker) - : start_marker_(std::move(start_marker)), end_marker_(std::move(end_marker)) {} + ToolCallStreamAccumulator(std::string start_marker, std::string end_marker, std::string tools_json = {}) + : start_marker_(std::move(start_marker)), + end_marker_(std::move(end_marker)), + tools_json_(std::move(tools_json)) {} - /// Feed a chunk into the accumulator. Returns ordered visible-text and completed-tool-call events. + /// Feed a chunk into the accumulator. Returns visible text and any tool calls completed by this chunk. Output Push(const std::string& chunk) { Output out; @@ -60,7 +60,7 @@ class ToolCallStreamAccumulator { if (start_marker_.empty() || end_marker_.empty()) { // Passthrough mode — no tool-call detection. - EmitVisible(out, chunk); + out.visible_text = chunk; return out; } @@ -70,8 +70,8 @@ class ToolCallStreamAccumulator { return out; } - /// Drain at end-of-stream. An unterminated tool-call block becomes visible text — it turned out not to be a real - /// tool call (no closing marker arrived), so the caller still sees what the model produced. + /// Drain at end-of-stream. A complete or narrowly repairable JSON call is recovered even if the model omitted the + /// closing marker; genuinely truncated blocks become visible text. Output Flush() { Output out; @@ -88,19 +88,6 @@ class ToolCallStreamAccumulator { bool InsideToolCall() const noexcept { return inside_tool_call_; } private: - static void EmitVisible(Output& out, std::string text) { - if (text.empty()) { - return; - } - if (!out.events.empty()) { - if (auto* previous = std::get_if(&out.events.back())) { - *previous += text; - return; - } - } - out.events.emplace_back(std::move(text)); - } - void Drain(Output& out, bool flushing) { while (true) { const std::string& marker = inside_tool_call_ ? end_marker_ : start_marker_; @@ -114,15 +101,9 @@ class ToolCallStreamAccumulator { tool_call_buffer_ += buffer_.substr(0, found + marker.size()); buffer_.erase(0, found + marker.size()); - auto parsed = ParseToolCalls(tool_call_buffer_, start_marker_, end_marker_); - if (parsed.empty()) { - // A marker-shaped block that cannot be parsed is model text, not a tool call. Preserve it rather than - // silently dropping generated output. - EmitVisible(out, tool_call_buffer_); - } else { - for (auto& pc : parsed) { - out.events.emplace_back(std::move(pc)); - } + auto parsed = ParseToolCalls(tool_call_buffer_, start_marker_, end_marker_, tools_json_); + for (auto& pc : parsed) { + out.ready_calls.push_back(std::move(pc)); } tool_call_buffer_.clear(); @@ -131,7 +112,7 @@ class ToolCallStreamAccumulator { // Opening marker: emit prefix as visible text, then start buffering the tool-call block (including the // marker — ParseToolCalls expects the full `...` substring). if (found > 0) { - EmitVisible(out, buffer_.substr(0, found)); + out.visible_text.append(buffer_, 0, found); } tool_call_buffer_ = buffer_.substr(found, marker.size()); buffer_.erase(0, found + marker.size()); @@ -144,13 +125,25 @@ class ToolCallStreamAccumulator { // No full marker. if (flushing) { if (inside_tool_call_) { - // Unterminated tool-call block: surface the buffered bytes as visible text so the caller still sees what - // the model produced. Matches ReasoningStreamSplitter::Flush behavior for unterminated reasoning. - EmitVisible(out, tool_call_buffer_ + buffer_); + std::string incomplete = tool_call_buffer_ + buffer_; + std::string candidate = incomplete; + if (const size_t stray_reasoning_end = candidate.find(""); + stray_reasoning_end != std::string::npos) { + candidate.erase(stray_reasoning_end); + } + candidate += end_marker_; + auto parsed = ParseToolCalls(candidate, start_marker_, end_marker_, tools_json_); + if (parsed.empty()) { + out.visible_text += incomplete; + } else { + for (auto& pc : parsed) { + out.ready_calls.push_back(std::move(pc)); + } + } tool_call_buffer_.clear(); inside_tool_call_ = false; } else { - EmitVisible(out, buffer_); + out.visible_text.append(buffer_); } buffer_.clear(); return; @@ -172,7 +165,7 @@ class ToolCallStreamAccumulator { size_t safe = buffer_.size() - hold; if (safe > 0) { - EmitVisible(out, buffer_.substr(0, safe)); + out.visible_text.append(buffer_, 0, safe); buffer_.erase(0, safe); } } @@ -197,6 +190,7 @@ class ToolCallStreamAccumulator { std::string start_marker_; std::string end_marker_; + std::string tools_json_; std::string buffer_; // pending bytes from Push() that haven't yet been routed std::string tool_call_buffer_; // accumulated bytes of the in-progress tool-call block (incl. start marker) bool inside_tool_call_ = false; diff --git a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_utils.cc b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_utils.cc index c35fb50f7..cdd35c6d5 100644 --- a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_utils.cc +++ b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_utils.cc @@ -5,7 +5,6 @@ #include -#include #include namespace fl { @@ -29,65 +28,267 @@ std::string RandomAlphanumeric(int length) { return result; } -std::optional ParseOneToolCall(const nlohmann::json& call) { - if (!call.is_object()) { - return std::nullopt; +bool HasAdvertisedTool(const nlohmann::json& tools, const std::string& name) { + if (!tools.is_array()) { + return false; } - auto name_it = call.find("name"); - if (name_it == call.end() || !name_it->is_string() || name_it->get().empty()) { - return std::nullopt; + for (const auto& tool : tools) { + if (!tool.is_object()) { + continue; + } + if (tool.value("name", std::string{}) == name || + (tool.contains("function") && tool["function"].is_object() && + tool["function"].value("name", std::string{}) == name)) { + return true; + } } + return false; +} - ParsedToolCall tc; - tc.name = name_it->get(); +bool HasAdvertisedParameter(const nlohmann::json& tools, + const std::string& tool_name, + const std::string& parameter_name) { + if (!tools.is_array()) { + return false; + } - if (auto args_it = call.find("arguments"); args_it != call.end()) { - tc.arguments = args_it->is_string() ? args_it->get() : args_it->dump(); - } else if (auto params_it = call.find("parameters"); params_it != call.end()) { - tc.arguments = params_it->is_string() ? params_it->get() : params_it->dump(); + for (const auto& tool : tools) { + if (!tool.is_object()) { + continue; + } + const auto& descriptor = + tool.contains("function") && tool["function"].is_object() ? tool["function"] : tool; + if (descriptor.value("name", std::string{}) != tool_name || + !descriptor.contains("parameters") || !descriptor["parameters"].is_object()) { + continue; + } + const auto& parameters = descriptor["parameters"]; + return parameters.contains("properties") && parameters["properties"].is_object() && + parameters["properties"].contains(parameter_name); } + return false; +} - return tc; +std::string NormalizeToolName(std::string name, const nlohmann::json& advertised_tools) { + auto trim = [](std::string value) { + const size_t start = value.find_first_not_of(" \t\r\n\"'"); + if (start == std::string::npos) { + return std::string{}; + } + const size_t end = value.find_last_not_of(" \t\r\n\"'"); + return value.substr(start, end - start + 1); + }; + + name = trim(std::move(name)); + if (name.starts_with("function=")) { + name = trim(name.substr(sizeof("function=") - 1)); + } + + if (name == "exec_command" && + ((advertised_tools.is_array() && advertised_tools.empty()) || + HasAdvertisedTool(advertised_tools, "shell")) && + !HasAdvertisedTool(advertised_tools, "exec_command")) { + return "shell"; + } + if (name == "cmd" && HasAdvertisedTool(advertised_tools, "shell") && + !HasAdvertisedTool(advertised_tools, "cmd")) { + return "shell"; + } + return name; } /// Try to parse a JSON string as a list of tool calls. /// Handles both array and single-object formats: /// [{"name": "fn", "arguments": {...}}] /// {"name": "fn", "arguments": {...}} -std::vector DeserializeToolCalls(const std::string& json_text) { - try { - auto json = nlohmann::json::parse(json_text); - std::vector results; - - if (json.is_array()) { - results.reserve(json.size()); +std::vector DeserializeToolCalls(const std::string& json_text, + const nlohmann::json& advertised_tools) { + std::vector results; - for (const auto& item : json) { - auto parsed = ParseOneToolCall(item); - if (!parsed) { - return {}; + try { + const size_t content_start = json_text.find_first_not_of(" \t\r\n"); + if (content_start == std::string::npos) { + return results; + } + const size_t content_end = json_text.find_last_not_of(" \t\r\n"); + const std::string normalized_text = + json_text.substr(content_start, content_end - content_start + 1); + + auto json = nlohmann::json::parse(normalized_text, nullptr, false); + bool repaired_missing_name_prefix = false; + + // Some models finish a complete object one outer brace early. Repair + // exactly one unmatched object brace; leave other truncation untouched. + if (json.is_discarded() && normalized_text.starts_with('{')) { + int object_depth = 0; + int array_depth = 0; + bool in_string = false; + bool escaped = false; + for (char ch : normalized_text) { + if (in_string) { + if (escaped) { + escaped = false; + } else if (ch == '\\') { + escaped = true; + } else if (ch == '"') { + in_string = false; + } + } else if (ch == '"') { + in_string = true; + } else if (ch == '{') { + ++object_depth; + } else if (ch == '}') { + --object_depth; + } else if (ch == '[') { + ++array_depth; + } else if (ch == ']') { + --array_depth; } + } + if (!in_string && object_depth == 1 && array_depth == 0) { + json = nlohmann::json::parse(normalized_text + "}", nullptr, false); + } + } - results.push_back(std::move(*parsed)); + // Some models omit {"name":" and start with 1) { + const std::string tool_name = normalized_text.substr(1, name_end - 1); + std::string repaired = + R"({"name":)" + nlohmann::json(tool_name).dump() + + normalized_text.substr(name_end + 1); + json = nlohmann::json::parse(repaired, nullptr, false); + + // A wrapped arguments object closes itself but may omit the closing + // brace for the reconstructed outer call object. + if (json.is_discarded() && + (repaired.find(R"(,"arguments":{)") != std::string::npos || + repaired.find(R"(,"parameters":{)") != std::string::npos || + repaired.find(R"(,"args":{)") != std::string::npos)) { + repaired += "}"; + json = nlohmann::json::parse(repaired, nullptr, false); + } + repaired_missing_name_prefix = !json.is_discarded(); } - } else if (json.is_object()) { - auto parsed = ParseOneToolCall(json); - if (!parsed) { - return {}; + } + + // The missing-name form may place argument members directly after the + // tool name. Preserve recognized wrappers; otherwise collect the direct + // members into the canonical arguments object. + if (repaired_missing_name_prefix && json.is_object() && json.contains("name") && + !json.contains("arguments") && !json.contains("parameters") && !json.contains("args")) { + nlohmann::json arguments = json; + arguments.erase("name"); + json = {{"name", json["name"]}, {"arguments", std::move(arguments)}}; + } + + // Some models emit {"tool_name","arg":value} with argument members + // directly after the tool name. Wrap those members as arguments. + if (json.is_discarded() && normalized_text.starts_with("{\"")) { + const size_t name_end = normalized_text.find('"', 2); + const size_t closing_brace = normalized_text.find_last_of('}'); + if (name_end != std::string::npos && name_end + 1 < normalized_text.size() && + normalized_text[name_end + 1] == ',' && closing_brace > name_end + 1) { + const std::string tool_name = normalized_text.substr(2, name_end - 2); + const std::string repaired = + R"({"name":)" + nlohmann::json(tool_name).dump() + R"(,"arguments":{)" + + normalized_text.substr(name_end + 2, closing_brace - name_end - 2) + "}}"; + json = nlohmann::json::parse(repaired, nullptr, false); } + } - results.push_back(std::move(*parsed)); + // Some models emit {"tool_name": "arg": value} instead of wrapping the + // arguments in an object. Repair only that narrow shape, then require the + // result to pass the normal JSON parser. + if (json.is_discarded()) { + const size_t colon = normalized_text.find(':'); + const size_t closing_brace = normalized_text.find_last_of('}'); + if (colon != std::string::npos && closing_brace != std::string::npos && colon < closing_brace) { + std::string repaired = normalized_text.substr(0, colon + 1) + "{" + + normalized_text.substr(colon + 1, closing_brace - colon - 1) + "}" + + normalized_text.substr(closing_brace); + json = nlohmann::json::parse(repaired, nullptr, false); + } } - for (auto& result : results) { - result.id = GenerateToolCallId(); + if (json.is_discarded()) { + return results; } - return results; + auto parse_one = [&](const nlohmann::json& call) { + if (!call.is_object()) { + return; + } + + ParsedToolCall tc; + tc.id = GenerateToolCallId(); + + if (call.contains("name") && call["name"].is_string()) { + tc.name = call["name"].get(); + } else if (call.contains("function") && call["function"].is_string()) { + tc.name = call["function"].get(); + } else if (call.size() == 1) { + const auto& [name, arguments] = *call.items().begin(); + tc.name = NormalizeToolName(name, advertised_tools); + if (name == "cmd" && tc.name == "shell") { + tc.arguments = nlohmann::json({{"cmd", arguments}}).dump(); + } else { + tc.arguments = arguments.is_string() ? arguments.get() : arguments.dump(); + } + results.push_back(std::move(tc)); + return; + } else { + return; + } + + tc.name = NormalizeToolName(std::move(tc.name), advertised_tools); + + // Arguments can be under "arguments", "parameters", or "args". + if (call.contains("arguments")) { + if (call["arguments"].is_string()) { + tc.arguments = call["arguments"].get(); + } else { + auto arguments = call["arguments"]; + if (tc.name == "shell" && arguments.is_object() && !arguments.contains("cmd") && + arguments.contains("command") && + HasAdvertisedParameter(advertised_tools, "shell", "cmd")) { + arguments["cmd"] = std::move(arguments["command"]); + arguments.erase("command"); + } + tc.arguments = arguments.dump(); + } + } else if (call.contains("parameters")) { + if (call["parameters"].is_string()) { + tc.arguments = call["parameters"].get(); + } else { + tc.arguments = call["parameters"].dump(); + } + } else if (call.contains("args")) { + if (call["args"].is_string()) { + tc.arguments = call["args"].get(); + } else { + tc.arguments = call["args"].dump(); + } + } + + results.push_back(std::move(tc)); + }; + + if (json.is_array()) { + for (const auto& item : json) { + parse_one(item); + } + } else if (json.is_object()) { + parse_one(json); + } } catch (const nlohmann::json::exception&) { - return {}; + // Invalid tool-call shape — return whatever we have so far (may be empty) } + + return results; } } // namespace @@ -98,9 +299,12 @@ std::string GenerateToolCallId() { std::vector ParseToolCalls(const std::string& text, const std::string& tool_call_start, - const std::string& tool_call_end) { + const std::string& tool_call_end, + const std::string& tools_json) { std::vector all_calls; + const auto advertised_tools = nlohmann::json::parse(tools_json, nullptr, false); + if (tool_call_start.empty() || tool_call_end.empty()) { return all_calls; } @@ -120,8 +324,15 @@ std::vector ParseToolCalls(const std::string& text, break; } + // If a malformed call was left open before another call, parse the + // innermost complete block rather than combining both payloads. + size_t nested_start = text.rfind(tool_call_start, end_pos); + if (nested_start != std::string::npos && nested_start > start_pos) { + content_start = nested_start + tool_call_start.size(); + } + std::string content = text.substr(content_start, end_pos - content_start); - auto calls = DeserializeToolCalls(content); + auto calls = DeserializeToolCalls(content, advertised_tools); for (auto& call : calls) { all_calls.push_back(std::move(call)); diff --git a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_utils.h b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_utils.h index e094c4374..ce6a8de99 100644 --- a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_utils.h +++ b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_utils.h @@ -28,10 +28,12 @@ struct ParsedToolCall { /// @param text The full generated text (may contain mixed text and tool calls) /// @param tool_call_start The start marker token string /// @param tool_call_end The end marker token string +/// @param tools_json Advertised OpenAI-format tools, used to resolve legacy aliases /// @return Parsed tool calls, empty if none found std::vector ParseToolCalls(const std::string& text, const std::string& tool_call_start, - const std::string& tool_call_end); + const std::string& tool_call_end, + const std::string& tools_json = {}); /// Generate a unique tool call ID (e.g., "call_abc123def"). std::string GenerateToolCallId(); diff --git a/sdk_v2/cpp/src/items/message_item.cc b/sdk_v2/cpp/src/items/message_item.cc index 540ce11ef..9bb1d5455 100644 --- a/sdk_v2/cpp/src/items/message_item.cc +++ b/sdk_v2/cpp/src/items/message_item.cc @@ -14,7 +14,7 @@ namespace fl { MessageItem::MessageItem(const MessageItem& other) - : Item(other), role(other.role), name(other.name) { + : Item(other), role(other.role), tool_calls(other.tool_calls), name(other.name) { content.reserve(other.content.size()); for (const auto& part : other.content) { if (!part.view) { @@ -31,6 +31,7 @@ MessageItem& MessageItem::operator=(const MessageItem& other) { Item::operator=(other); role = other.role; + tool_calls = other.tool_calls; name = other.name; content.clear(); content.reserve(other.content.size()); diff --git a/sdk_v2/cpp/src/items/message_item.h b/sdk_v2/cpp/src/items/message_item.h index 7cde82546..578c763ab 100644 --- a/sdk_v2/cpp/src/items/message_item.h +++ b/sdk_v2/cpp/src/items/message_item.h @@ -4,6 +4,7 @@ #include "items/item.h" #include "items/text_item.h" +#include "items/tool_call_item.h" #include "exception.h" #include @@ -62,6 +63,7 @@ struct MessagePart { struct MessageItem : Item { flMessageRole role; std::vector content; + std::vector tool_calls; std::string name; // C API usage diff --git a/sdk_v2/cpp/src/service/responses_handler.cc b/sdk_v2/cpp/src/service/responses_handler.cc index 78595da20..8b489769f 100644 --- a/sdk_v2/cpp/src/service/responses_handler.cc +++ b/sdk_v2/cpp/src/service/responses_handler.cc @@ -26,6 +26,35 @@ namespace fl { using namespace fl::responses; +namespace { + +std::string NormalizeResponseToolName(std::string name, + const std::optional>& tools) { + const size_t start = name.find_first_not_of(" \t\r\n\"'"); + if (start == std::string::npos) { + return {}; + } + const size_t end = name.find_last_not_of(" \t\r\n\"'"); + name = name.substr(start, end - start + 1); + + if (name != "exec_command") { + return name; + } + + bool has_shell = false; + bool has_exec_command = false; + if (tools.has_value()) { + for (const auto& tool : *tools) { + has_shell = has_shell || tool.function.name == "shell"; + has_exec_command = has_exec_command || tool.function.name == "exec_command"; + } + } + const bool has_tool_metadata = tools.has_value() && !tools->empty(); + return !has_exec_command && (has_shell || !has_tool_metadata) ? "shell" : name; +} + +} // namespace + // ======================================================================== // ResponsesHandler — POST /v1/responses // ======================================================================== @@ -80,13 +109,13 @@ std::shared_ptr ResponsesHandler::ParseAnd } std::shared_ptr ResponsesHandler::ResolveModel( - const std::string& model_name, Model*& model, GenAIModelInstance*& loaded) { + const std::string& model_name, Model*& model, GenAIModelInstance*& loaded) { model = ctx_.catalog.GetModelVariant(model_name); if (!model) { return ErrorResponse(Status::CODE_404, "Model not found", "No model matching '" + model_name + "'"); } - loaded = ctx_.model_load_manager.GetLoadedModel(model->Id(), model->GetPath()); + loaded = ctx_.model_load_manager.GetLoadedModel(model->Id()); if (!loaded) { return ErrorResponse(Status::CODE_400, "Model not loaded", "Model '" + model_name + "' must be loaded before inference"); @@ -250,6 +279,13 @@ std::shared_ptr ResponsesHandler::HandleNo fl::Response session_response; session->ProcessRequest(session_request, session_response); + for (auto& item : session_response.items) { + if (item->type == FOUNDRY_LOCAL_ITEM_TOOL_CALL) { + auto* tool_call = static_cast(item.get()); + tool_call->name = NormalizeResponseToolName(std::move(tool_call->name), params.tools); + } + } + auto [output, output_text] = ResponseConverter::FromSessionResponse(session_response); auto response = ResponseConverter::BuildResponseObject(response_id, created_at, model_name, params, @@ -475,17 +511,6 @@ std::shared_ptr ResponsesHandler::HandleSt push_event("response.content_part.added", part_added); }; - auto emit_tool_call = [&](const fl::ToolCallItem& call) { - close_current(); - - const int output_index = next_output_index++; - auto output = ResponseConverter::BuildFunctionCallStreamOutput(call, output_index, seq); - for (const auto& event : output.events) { - push_event(StreamEventTypeToString(event.type), event); - } - closed_items.push_back(std::move(output.completed_item)); - }; - try { // Register inside the try so a shutdown rejection (Register throws) is reported as a stream failure // instead of escaping this raw std::thread and calling std::terminate. @@ -502,17 +527,57 @@ std::shared_ptr ResponsesHandler::HandleSt } if (item->type == FOUNDRY_LOCAL_ITEM_TOOL_CALL) { - emit_tool_call(static_cast(*item)); - return 0; - } + close_current(); - if (item->type != FOUNDRY_LOCAL_ITEM_TEXT) { - logger.Log(LogLevel::Debug, - fmt::format("Responses streaming: skipping non-text item type {}", - static_cast(item->type))); + auto* tool_call = static_cast(item.get()); + FunctionCallOutputItem function_call; + function_call.id = ResponseConverter::GenerateId("fc"); + function_call.call_id = tool_call->call_id.empty() ? ResponseConverter::GenerateId("call") + : tool_call->call_id; + function_call.name = NormalizeResponseToolName(tool_call->name, params_copy.tools); + function_call.arguments = tool_call->arguments; + function_call.status = ResponseStatus::kInProgress; + int output_index = next_output_index++; + + StreamEvent item_added; + item_added.type = StreamEventType::kOutputItemAdded; + item_added.sequence_number = seq++; + item_added.output_index = output_index; + item_added.item = function_call; + push_event("response.output_item.added", item_added); + + StreamEvent arguments_delta; + arguments_delta.type = StreamEventType::kFunctionCallArgumentsDelta; + arguments_delta.sequence_number = seq++; + arguments_delta.output_index = output_index; + arguments_delta.item_id = function_call.id; + arguments_delta.function_call_id = function_call.call_id; + arguments_delta.delta = function_call.arguments; + push_event("response.function_call_arguments.delta", arguments_delta); + + StreamEvent arguments_done; + arguments_done.type = StreamEventType::kFunctionCallArgumentsDone; + arguments_done.sequence_number = seq++; + arguments_done.output_index = output_index; + arguments_done.item_id = function_call.id; + arguments_done.function_name = function_call.name; + arguments_done.function_call_id = function_call.call_id; + arguments_done.function_arguments = function_call.arguments; + push_event("response.function_call_arguments.done", arguments_done); + + function_call.status = ResponseStatus::kCompleted; + StreamEvent item_done; + item_done.type = StreamEventType::kOutputItemDone; + item_done.sequence_number = seq++; + item_done.output_index = output_index; + item_done.item = function_call; + push_event("response.output_item.done", item_done); + + closed_items.push_back(std::move(function_call)); return 0; } + assert(item->type == FOUNDRY_LOCAL_ITEM_TEXT); auto* text_item = static_cast(item.get()); ItemKind incoming = (text_item->text_type == FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING) diff --git a/sdk_v2/cpp/test/internal_api/chat/chat_template_test.cc b/sdk_v2/cpp/test/internal_api/chat/chat_template_test.cc index a8f2364a5..a983da5e4 100644 --- a/sdk_v2/cpp/test/internal_api/chat/chat_template_test.cc +++ b/sdk_v2/cpp/test/internal_api/chat/chat_template_test.cc @@ -98,11 +98,20 @@ TEST_F(ChatTemplateTest, MultiTurnConversation) { EXPECT_NE(prompt.find("3+3"), std::string::npos); } -TEST(ChatTemplateUnitTest, EmptyAssistantMessageRendersAsEmptyContent) { - MessageItem empty_assistant; - empty_assistant.role = FOUNDRY_LOCAL_ROLE_ASSISTANT; - - EXPECT_EQ(RenderMessageForPrompt(empty_assistant), ""); +TEST(ChatTemplateSerializationTest, AssistantToolCallPrecedesToolResponse) { + std::vector messages = { + {FOUNDRY_LOCAL_ROLE_USER, "Inspect the repository."}, + {FOUNDRY_LOCAL_ROLE_ASSISTANT, "I'll inspect it now."}, + {FOUNDRY_LOCAL_ROLE_TOOL, "file.txt"}}; + messages[1].tool_calls.emplace_back("call_1", "shell", R"({"cmd":"ls"})"); + + std::string messages_json = BuildChatMessagesJson(messages); + const auto call_pos = messages_json.find(R"("tool_calls":[{"arguments":"{\"cmd\":\"ls\"}","name":"shell"}])"); + const auto result_pos = messages_json.find("file.txt"); + + ASSERT_NE(call_pos, std::string::npos) << messages_json; + ASSERT_NE(result_pos, std::string::npos) << messages_json; + EXPECT_LT(call_pos, result_pos) << messages_json; } TEST_F(ChatTemplateTest, EmptyMessagesThrows) { diff --git a/sdk_v2/cpp/test/internal_api/response_converter_test.cc b/sdk_v2/cpp/test/internal_api/response_converter_test.cc index 25407fe71..cd99c4a37 100644 --- a/sdk_v2/cpp/test/internal_api/response_converter_test.cc +++ b/sdk_v2/cpp/test/internal_api/response_converter_test.cc @@ -16,6 +16,7 @@ #include "items/message_item.h" #include "items/text_item.h" #include "items/tool_call_item.h" +#include "items/tool_result_item.h" using namespace fl; using namespace fl::responses; @@ -41,90 +42,6 @@ static ResponseCreateParams MakeTestParams() { return params; } -TEST(ResponseConverterTest, FromSessionResponse_ReasoningOnlyMessageIsNotOutputText) { - Response response; - std::vector> parts; - parts.push_back(std::make_unique("private scratchpad", FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING)); - response.items.push_back( - std::make_unique(FOUNDRY_LOCAL_ROLE_ASSISTANT, std::move(parts))); - - auto [output, output_text] = FromSessionResponse(response, "msg"); - - ASSERT_EQ(output.size(), 1u); - ASSERT_TRUE(std::holds_alternative(output.front())); - EXPECT_EQ(std::get(output.front()).summary.front().text, "private scratchpad"); - EXPECT_TRUE(output_text.empty()); -} - -TEST(ResponseConverterTest, FromSessionResponse_InterleavedReasoningPreservesOutputOrder) { - Response response; - std::vector> parts; - parts.push_back(std::make_unique("think one", FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING)); - parts.push_back(std::make_unique("answer one", FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT)); - parts.push_back(std::make_unique("think two", FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING)); - parts.push_back(std::make_unique("answer two", FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT)); - response.items.push_back(std::make_unique(FOUNDRY_LOCAL_ROLE_ASSISTANT, std::move(parts))); - - auto [output, output_text] = FromSessionResponse(response, "msg"); - - ASSERT_EQ(output.size(), 4u); - EXPECT_TRUE(std::holds_alternative(output[0])); - EXPECT_TRUE(std::holds_alternative(output[1])); - EXPECT_TRUE(std::holds_alternative(output[2])); - EXPECT_TRUE(std::holds_alternative(output[3])); - EXPECT_EQ(output_text, "answer oneanswer two"); -} - -TEST(ResponseConverterTest, BuildFunctionCallStreamOutputEmitsCompleteLifecycle) { - ToolCallItem call("call_test", "get_weather", R"({"city":"Seattle"})"); - int sequence_number = 7; - - auto output = BuildFunctionCallStreamOutput(call, 3, sequence_number); - - ASSERT_EQ(output.events.size(), 4u); - EXPECT_EQ(sequence_number, 11); - - const auto& added = output.events[0]; - EXPECT_EQ(added.type, StreamEventType::kOutputItemAdded); - EXPECT_EQ(added.sequence_number, 7); - EXPECT_EQ(added.output_index, 3); - ASSERT_TRUE(added.item.has_value()); - const auto& added_item = std::get(*added.item); - EXPECT_EQ(added_item.id, output.completed_item.id); - EXPECT_EQ(added_item.call_id, "call_test"); - EXPECT_EQ(added_item.name, "get_weather"); - EXPECT_TRUE(added_item.arguments.empty()); - EXPECT_EQ(added_item.status, ResponseStatus::kInProgress); - - const auto& delta = output.events[1]; - EXPECT_EQ(delta.type, StreamEventType::kFunctionCallArgumentsDelta); - EXPECT_EQ(delta.sequence_number, 8); - EXPECT_EQ(delta.output_index, 3); - EXPECT_EQ(delta.item_id, output.completed_item.id); - EXPECT_EQ(delta.delta, R"({"city":"Seattle"})"); - EXPECT_EQ(delta.function_call_id, "call_test"); - - const auto& arguments_done = output.events[2]; - EXPECT_EQ(arguments_done.type, StreamEventType::kFunctionCallArgumentsDone); - EXPECT_EQ(arguments_done.sequence_number, 9); - EXPECT_EQ(arguments_done.output_index, 3); - EXPECT_EQ(arguments_done.item_id, output.completed_item.id); - EXPECT_EQ(arguments_done.function_name, "get_weather"); - EXPECT_EQ(arguments_done.function_call_id, "call_test"); - EXPECT_EQ(arguments_done.function_arguments, R"({"city":"Seattle"})"); - - const auto& item_done = output.events[3]; - EXPECT_EQ(item_done.type, StreamEventType::kOutputItemDone); - EXPECT_EQ(item_done.sequence_number, 10); - EXPECT_EQ(item_done.output_index, 3); - ASSERT_TRUE(item_done.item.has_value()); - const auto& completed_item = std::get(*item_done.item); - EXPECT_EQ(completed_item.arguments, R"({"city":"Seattle"})"); - EXPECT_EQ(completed_item.status, ResponseStatus::kCompleted); - EXPECT_EQ(output.completed_item.arguments, R"({"city":"Seattle"})"); - EXPECT_EQ(output.completed_item.status, ResponseStatus::kCompleted); -} - // ======================================================================== // BuildFailedResponseObject // ======================================================================== @@ -374,6 +291,31 @@ TEST(ResponseConverterTest, ToInputItems_FunctionCallOutput_GetsFcoPrefix) { EXPECT_TRUE(id.find("fco_") == 0); } +TEST(ResponseConverterTest, ToSessionRequest_FunctionCallAndOutput_PreserveToolTurn) { + nlohmann::json json = { + {"model", "test-model"}, + {"input", nlohmann::json::array({ + {{"type", "function_call"}, + {"call_id", "call_1"}, + {"name", "shell"}, + {"arguments", R"({"cmd":"pwd"})"}}, + {{"type", "function_call_output"}, + {"call_id", "call_1"}, + {"output", "/testbed"}}, + })}}; + + auto params = json.get(); + auto request = ToSessionRequest(params); + + ASSERT_EQ(request.items.size(), 2u); + auto* call = dynamic_cast(request.items[0]); + ASSERT_NE(call, nullptr); + EXPECT_EQ(call->call_id, "call_1"); + auto* result = dynamic_cast(request.items[1]); + ASSERT_NE(result, nullptr); + EXPECT_EQ(result->call_id, "call_1"); +} + // ======================================================================== // ToSessionRequest — vision input (input_image content) // diff --git a/sdk_v2/cpp/test/internal_api/toolcalling/grammar_test.cc b/sdk_v2/cpp/test/internal_api/toolcalling/grammar_test.cc index dc7c5f567..8240f5b6f 100644 --- a/sdk_v2/cpp/test/internal_api/toolcalling/grammar_test.cc +++ b/sdk_v2/cpp/test/internal_api/toolcalling/grammar_test.cc @@ -372,7 +372,7 @@ TEST(BuildLarkGrammarTest, CoT_TextOrTool_KnownThinkIds_KnownToolIds) { cot: THINK_TEXT "\n" THINK_TEXT: /[^<]+/ output: TEXT | toolcall -TEXT: /[^{<](.|\n)*/ +TEXT: /[^{<][^<]*/ toolcall: functioncall functioncall: %json )" + json_schema + "\n"; @@ -397,7 +397,7 @@ TEST(BuildLarkGrammarTest, CoT_TextOrTool_UnknownThinkIds_KnownToolIds) { cot: "" THINK_TEXT "" "\n" THINK_TEXT: /[^<]+/ output: TEXT | toolcall -TEXT: /[^{<](.|\n)*/ +TEXT: /[^{<][^<]*/ toolcall: functioncall functioncall: %json )" + json_schema + "\n"; @@ -422,7 +422,7 @@ TEST(BuildLarkGrammarTest, CoT_TextOrTool_KnownThinkIds_UnknownToolIds) { cot: THINK_TEXT "\n" THINK_TEXT: /[^<]+/ output: TEXT | functioncall -TEXT: /[^{<](.|\n)*/ +TEXT: /[^{<][^{]*/ functioncall: %json )" + json_schema + "\n"; @@ -444,7 +444,7 @@ TEST(BuildLarkGrammarTest, CoT_TextOrTool_UnknownThinkIds_UnknownToolIds) { cot: "" THINK_TEXT "" "\n" THINK_TEXT: /[^<]+/ output: TEXT | functioncall -TEXT: /[^{<](.|\n)*/ +TEXT: /[^{<][^{]*/ functioncall: %json )" + json_schema + "\n"; diff --git a/sdk_v2/cpp/test/internal_api/toolcalling/tool_call_stream_accumulator_test.cc b/sdk_v2/cpp/test/internal_api/toolcalling/tool_call_stream_accumulator_test.cc index 8d10258e3..dac8c5145 100644 --- a/sdk_v2/cpp/test/internal_api/toolcalling/tool_call_stream_accumulator_test.cc +++ b/sdk_v2/cpp/test/internal_api/toolcalling/tool_call_stream_accumulator_test.cc @@ -9,7 +9,6 @@ #include -#include #include #include @@ -17,31 +16,16 @@ using namespace fl; namespace { -// Concatenate text events from a sequence of Push results. +// Concatenate the visible_text from a sequence of Push results — handy for asserting that "what came through +// the visible channel" equals what we'd have produced without tool-call extraction. std::string CollectVisible(const std::vector& outs) { std::string s; for (const auto& o : outs) { - for (const auto& event : o.events) { - if (const auto* text = std::get_if(&event)) { - s += *text; - } - } + s += o.visible_text; } return s; } -std::vector CollectCalls(std::vector& outs) { - std::vector calls; - for (auto& output : outs) { - for (auto& event : output.events) { - if (auto* call = std::get_if(&event)) { - calls.push_back(std::move(*call)); - } - } - } - return calls; -} - // Run a sequence of chunks through the accumulator, calling Flush at the end. Returns one Output per chunk plus // the Flush Output appended last. std::vector RunChunks(ToolCallStreamAccumulator& acc, @@ -64,15 +48,16 @@ std::vector RunChunks(ToolCallStreamAccumulat TEST(ToolCallStreamAccumulatorTest, EmptyMarkersIsPassthrough) { ToolCallStreamAccumulator acc("", ""); auto out = acc.Push("any text including markers"); - ASSERT_EQ(out.events.size(), 1u); - EXPECT_EQ(std::get(out.events[0]), "any text including markers"); + EXPECT_EQ(out.visible_text, "any text including markers"); + EXPECT_TRUE(out.ready_calls.empty()); EXPECT_FALSE(acc.InsideToolCall()); } TEST(ToolCallStreamAccumulatorTest, EmptyChunkProducesNothing) { ToolCallStreamAccumulator acc("", ""); auto out = acc.Push(""); - EXPECT_TRUE(out.events.empty()); + EXPECT_TRUE(out.visible_text.empty()); + EXPECT_TRUE(out.ready_calls.empty()); } // ======================================================================== @@ -84,9 +69,7 @@ TEST(ToolCallStreamAccumulatorTest, PlainTextPassesThroughVerbatim) { auto outs = RunChunks(acc, {"Hello", ", ", "world!"}); EXPECT_EQ(CollectVisible(outs), "Hello, world!"); for (const auto& o : outs) { - EXPECT_TRUE(std::none_of(o.events.begin(), o.events.end(), [](const auto& event) { - return std::holds_alternative(event); - })); + EXPECT_TRUE(o.ready_calls.empty()); } } @@ -101,10 +84,8 @@ TEST(ToolCallStreamAccumulatorTest, SingleToolCallInOneChunk) { auto outs = RunChunks(acc, {chunk}); EXPECT_EQ(CollectVisible(outs), "prefix suffix"); - ASSERT_EQ(outs[0].events.size(), 3u); - EXPECT_EQ(std::get(outs[0].events[0]), "prefix "); - EXPECT_EQ(std::get(outs[0].events[1]).name, "add"); - EXPECT_EQ(std::get(outs[0].events[2]), " suffix"); + ASSERT_EQ(outs[0].ready_calls.size(), 1u); + EXPECT_EQ(outs[0].ready_calls[0].name, "add"); } TEST(ToolCallStreamAccumulatorTest, SingleToolCallSplitAcrossManyChunks) { @@ -127,7 +108,13 @@ TEST(ToolCallStreamAccumulatorTest, SingleToolCallSplitAcrossManyChunks) { EXPECT_EQ(CollectVisible(outs), "before after") << "Marker and JSON bytes must not leak into visible text"; - auto all = CollectCalls(outs); + // Find the tool call across whichever Push produced it (it should be the one that completed the end marker). + std::vector all; + for (auto& o : outs) { + for (auto& pc : o.ready_calls) { + all.push_back(std::move(pc)); + } + } ASSERT_EQ(all.size(), 1u); EXPECT_EQ(all[0].name, "mul"); EXPECT_NE(all[0].arguments.find("7"), std::string::npos); @@ -148,7 +135,12 @@ TEST(ToolCallStreamAccumulatorTest, MarkerByteByByte) { EXPECT_TRUE(CollectVisible(outs).empty()) << "Single tool-call block with no surrounding text produces no visible"; - auto all = CollectCalls(outs); + std::vector all; + for (auto& o : outs) { + for (auto& pc : o.ready_calls) { + all.push_back(std::move(pc)); + } + } ASSERT_EQ(all.size(), 1u); EXPECT_EQ(all[0].name, "f"); } @@ -166,7 +158,12 @@ TEST(ToolCallStreamAccumulatorTest, TwoSequentialToolCalls) { EXPECT_EQ(CollectVisible(outs), " middle "); - auto all = CollectCalls(outs); + std::vector all; + for (auto& o : outs) { + for (auto& pc : o.ready_calls) { + all.push_back(std::move(pc)); + } + } ASSERT_EQ(all.size(), 2u); EXPECT_EQ(all[0].name, "a"); EXPECT_EQ(all[1].name, "b"); @@ -197,49 +194,23 @@ TEST(ToolCallStreamAccumulatorTest, UnterminatedToolCallBecomesVisibleOnFlush) { // No tool call should have been emitted — the block never closed. for (const auto& o : outs) { - EXPECT_TRUE(std::none_of(o.events.begin(), o.events.end(), [](const auto& event) { - return std::holds_alternative(event); - })); + EXPECT_TRUE(o.ready_calls.empty()); } EXPECT_FALSE(acc.InsideToolCall()) << "Flush should leave accumulator in outside state"; } -TEST(ToolCallStreamAccumulatorTest, CompletedMalformedToolCallBecomesVisible) { - ToolCallStreamAccumulator acc("", ""); - auto outs = RunChunks(acc, {"before not json after"}); - - EXPECT_EQ(CollectVisible(outs), "before not json after"); - EXPECT_TRUE(CollectCalls(outs).empty()); -} - -TEST(ToolCallStreamAccumulatorTest, CompletedToolCallWithoutNameBecomesVisible) { - ToolCallStreamAccumulator acc("", ""); - const std::string generated = R"({"arguments":{"value":1}})"; - auto outs = RunChunks(acc, {generated}); - - EXPECT_EQ(CollectVisible(outs), generated); - EXPECT_TRUE(CollectCalls(outs).empty()); -} - -TEST(ToolCallStreamAccumulatorTest, CompletedToolCallWithNonStringNameBecomesVisible) { - ToolCallStreamAccumulator acc("", ""); - const std::string generated = R"({"name":123,"arguments":{}})"; - auto outs = RunChunks(acc, {generated}); - - EXPECT_EQ(CollectVisible(outs), generated); - EXPECT_TRUE(CollectCalls(outs).empty()); -} - -TEST(ToolCallStreamAccumulatorTest, CompletedMixedValidAndInvalidArrayBecomesVisible) { - ToolCallStreamAccumulator acc("", ""); - const std::string generated = - R"([{"name":"fn1","arguments":{}},{"name":123}])"; - auto outs = RunChunks(acc, {"before ", generated, " after"}); +TEST(ToolCallStreamAccumulatorTest, FlushRecoversCompleteCallWithWrongClosingTag) { + std::string tools = + R"([{"type":"function","name":"shell","parameters":{"type":"object","properties":{"cmd":{"type":"string"}}}}])"; + ToolCallStreamAccumulator acc("", "", tools); + auto outs = RunChunks( + acc, {R"({"function":"exec_command","arguments":{"cmd":"pwd"})"}); - EXPECT_EQ(CollectVisible(outs), "before " + generated + " after") - << "A partially-invalid block must be preserved whole, not partially parsed"; - EXPECT_TRUE(CollectCalls(outs).empty()); + EXPECT_TRUE(CollectVisible(outs).empty()); + ASSERT_EQ(outs.back().ready_calls.size(), 1u); + EXPECT_EQ(outs.back().ready_calls[0].name, "shell"); + EXPECT_EQ(outs.back().ready_calls[0].arguments, R"({"cmd":"pwd"})"); } // ======================================================================== @@ -270,12 +241,11 @@ TEST(ToolCallStreamAccumulatorTest, FalseStartPrefixReleasesAfterDisambiguation) // First chunk ends with "(out1.events[0]), "hello "); + EXPECT_EQ(out1.visible_text, "hello "); // Next chunk reveals the prefix was actually part of unrelated XML-ish text. The held-back ""); - ASSERT_EQ(out2.events.size(), 1u); - EXPECT_EQ(std::get(out2.events[0]), ""); + EXPECT_EQ(out2.visible_text, ""); + EXPECT_TRUE(out2.ready_calls.empty()); } diff --git a/sdk_v2/cpp/test/internal_api/toolcalling/tool_call_utils_test.cc b/sdk_v2/cpp/test/internal_api/toolcalling/tool_call_utils_test.cc index 84090e687..4068ea58b 100644 --- a/sdk_v2/cpp/test/internal_api/toolcalling/tool_call_utils_test.cc +++ b/sdk_v2/cpp/test/internal_api/toolcalling/tool_call_utils_test.cc @@ -107,6 +107,165 @@ TEST(ParseToolCallsTest, ParametersKeyWorksAsAlternative) { EXPECT_NE(calls[0].arguments.find("b"), std::string::npos); } +TEST(ParseToolCallsTest, SingleKeyToolCall) { + std::string text = + R"({"exec_command":{"cmd":"grep -n test file.py"}})"; + auto calls = ParseToolCalls(text, "", ""); + + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "exec_command"); + EXPECT_EQ(calls[0].arguments, R"({"cmd":"grep -n test file.py"})"); +} + +TEST(ParseToolCallsTest, SingleKeyToolCallWithMissingArgumentsBrace) { + std::string text = + R"({"update_plan":"explanation":"Done","plan":[{"step":"verify","status":"completed"}]})"; + auto calls = ParseToolCalls(text, "", ""); + + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "update_plan"); + EXPECT_EQ(calls[0].arguments, + R"({"explanation":"Done","plan":[{"status":"completed","step":"verify"}]})"); +} + +TEST(ParseToolCallsTest, RecoversNestedToolCallWithArgs) { + std::string text = + R"({"name":"exec_command","args":{"cmd":"pwd"}})"; + auto calls = ParseToolCalls(text, "", ""); + + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "exec_command"); + EXPECT_EQ(calls[0].arguments, R"({"cmd":"pwd"})"); +} + +TEST(ParseToolCallsTest, RecoversMissingNameObjectPrefix) { + std::string text = + R"()"; + auto calls = ParseToolCalls(text, "", ""); + + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "exec_command"); + EXPECT_EQ(calls[0].arguments, R"({"cmd":"ls /testbed","workdir":"/testbed"})"); +} + +TEST(ParseToolCallsTest, RecoversMultilineMissingNamePrefixWithDirectArguments) { + std::string text = R"( +)"; + auto calls = ParseToolCalls(text, "", ""); + + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "exec_command"); + EXPECT_EQ(calls[0].arguments, + R"({"cmd":"ls /testbed && git -C /testbed log --oneline -3"})"); +} + +TEST(ParseToolCallsTest, RecoversCommaAfterToolName) { + std::string text = + R"({"exec_command","cmd":"ls /testbed","workdir":"/testbed"})"; + auto calls = ParseToolCalls(text, "", ""); + + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "exec_command"); + EXPECT_EQ(calls[0].arguments, R"({"cmd":"ls /testbed","workdir":"/testbed"})"); +} + +TEST(ParseToolCallsTest, MapsLegacyExecCommandToAdvertisedShell) { + std::string text = + R"({"name":"exec_command","arguments":{"cmd":"git diff"}})"; + std::string tools = + R"([{"type":"function","name":"shell","parameters":{"type":"object"}}])"; + auto calls = ParseToolCalls(text, "", "", tools); + + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "shell"); + EXPECT_EQ(calls[0].arguments, R"({"cmd":"git diff"})"); +} + +TEST(ParseToolCallsTest, StripsLeadingQuoteBeforeMappingLegacyExecCommand) { + std::string text = + R"(<"exec_command","arguments":{"cmd":"git diff"})"; + std::string tools = + R"([{"type":"function","name":"shell","parameters":{"type":"object"}}])"; + auto calls = ParseToolCalls(text, "", "", tools); + + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "shell"); + EXPECT_EQ(calls[0].arguments, R"({"cmd":"git diff"})"); +} + +TEST(ParseToolCallsTest, MapsLegacyExecCommandWhenToolMetadataIsEmpty) { + std::string text = + R"(<"exec_command","arguments":{"cmd":"git diff"})"; + auto calls = ParseToolCalls(text, "", "", "[]"); + + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "shell"); + EXPECT_EQ(calls[0].arguments, R"({"cmd":"git diff"})"); +} + +TEST(ParseToolCallsTest, PreservesAdvertisedExecCommand) { + std::string text = + R"({"name":"exec_command","arguments":{"cmd":"git diff"}})"; + std::string tools = + R"([{"type":"function","function":{"name":"exec_command","parameters":{"type":"object"}}}])"; + auto calls = ParseToolCalls(text, "", "", tools); + + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "exec_command"); +} + +TEST(ParseToolCallsTest, MapsFunctionPrefixedExecCommandToAdvertisedShell) { + std::string tools = + R"([{"type":"function","name":"shell","parameters":{"type":"object"}}])"; + const std::vector texts = { + R"({"name":"function=\"exec_command","arguments":{"cmd":"pwd"}})", + R"({"name":"function=exec_command","arguments":{"cmd":"pwd"}})"}; + + for (const auto& text : texts) { + auto calls = ParseToolCalls(text, "", "", tools); + + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "shell"); + EXPECT_EQ(calls[0].arguments, R"({"cmd":"pwd"})"); + } +} + +TEST(ParseToolCallsTest, RecoversSingletonCmdAsAdvertisedShellCall) { + std::string text = R"({"cmd":"pwd"})"; + std::string tools = + R"([{"type":"function","name":"shell","parameters":{"type":"object"}}])"; + auto calls = ParseToolCalls(text, "", "", tools); + + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "shell"); + EXPECT_EQ(calls[0].arguments, R"({"cmd":"pwd"})"); +} + +TEST(ParseToolCallsTest, MapsCommandArgumentToAdvertisedShellCmd) { + std::string text = + R"({"name":"exec_command","arguments":{"command":"pwd"}})"; + std::string tools = + R"([{"type":"function","name":"shell","parameters":{"type":"object","properties":{"cmd":{"type":"string"}}}}])"; + auto calls = ParseToolCalls(text, "", "", tools); + + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "shell"); + EXPECT_EQ(calls[0].arguments, R"({"cmd":"pwd"})"); +} + +TEST(ParseToolCallsTest, RecoversFunctionKeyAndMissingOuterBrace) { + std::string text = + R"({"function":"exec_command","arguments":{"cmd":"pwd"})"; + std::string tools = + R"([{"type":"function","name":"shell","parameters":{"type":"object","properties":{"cmd":{"type":"string"}}}}])"; + auto calls = ParseToolCalls(text, "", "", tools); + + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "shell"); + EXPECT_EQ(calls[0].arguments, R"({"cmd":"pwd"})"); +} + TEST(ParseToolCallsTest, InvalidJsonReturnsEmpty) { std::string text = R"(not valid json)"; auto calls = ParseToolCalls(text, "", ""); From 28f57598981fad2d6ab4514ba8e6f95f452dfd10 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Thu, 3 Sep 2026 18:00:56 -0500 Subject: [PATCH 02/11] Enable model-selected Engine chat batching Use each model's genai_config batching contract so independent chat sessions can share an ORT GenAI Engine while Generator models retain their existing behavior. Files changed: - Add the owner-thread Engine dispatcher and per-session adapter. - Route chat creation, continuation, cancellation, usage, and option changes through the selected backend. - Parse and validate static and dynamic Engine configuration. - Add configuration, template, concurrency, and search-option coverage. - Pin ORT GenAI and packaging to 0.15.3. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9b90e66e-af55-404d-b606-7899f6f73c54 --- .pipelines/foundry-local-packaging.yml | 2 +- .pipelines/v2/sdk_v2-pipeline-plan.md | 2 +- sdk_v2/cpp/CMakeLists.txt | 19 + .../generative/chat/chat_generator.cc | 4 + .../generative/chat/chat_generator.h | 26 ++ .../generative/chat/chat_session.cc | 90 ++++- .../generative/chat/chat_session.h | 7 +- .../generative/chat/chat_template.cc | 20 + .../generative/chat/chat_template.h | 6 + .../generative/chat/onnx_chat_engine.cc | 347 ++++++++++++++++++ .../generative/chat/onnx_chat_engine.h | 107 ++++++ .../generative/chat/onnx_chat_generator.cc | 39 +- .../generative/chat/onnx_chat_generator.h | 4 +- .../chat/onnx_engine_chat_generator.cc | 154 ++++++++ .../chat/onnx_engine_chat_generator.h | 64 ++++ .../generative/chat/search_options.cc | 57 ++- .../generative/chat/search_options.h | 11 + .../inferencing/generative/genai_config.cc | 93 +++++ .../src/inferencing/generative/genai_config.h | 28 ++ .../generative/genai_model_instance.cc | 22 +- .../generative/genai_model_instance.h | 4 + .../internal_api/chat/chat_session_test.cc | 23 ++ .../internal_api/chat/chat_template_test.cc | 11 + .../internal_api/chat/search_options_test.cc | 24 ++ .../test/internal_api/genai_config_test.cc | 83 +++++ sdk_v2/cpp/test/test_main.cc | 11 +- sdk_v2/deps_versions.json | 2 +- 27 files changed, 1194 insertions(+), 66 deletions(-) create mode 100644 sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc create mode 100644 sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.h create mode 100644 sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.cc create mode 100644 sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.h diff --git a/.pipelines/foundry-local-packaging.yml b/.pipelines/foundry-local-packaging.yml index 2ee3d2e0b..684e7c1bc 100644 --- a/.pipelines/foundry-local-packaging.yml +++ b/.pipelines/foundry-local-packaging.yml @@ -60,7 +60,7 @@ variables: value: '1.28.0' - ${{ if eq(parameters.isRelease, true) }}: - name: cppGenaiVersion - value: '0.15.2' + value: '0.15.3' - ${{ else }}: - name: cppGenaiVersion value: '0.16.0-dev1001400138' diff --git a/.pipelines/v2/sdk_v2-pipeline-plan.md b/.pipelines/v2/sdk_v2-pipeline-plan.md index 44fc170f6..4d646d7f7 100644 --- a/.pipelines/v2/sdk_v2-pipeline-plan.md +++ b/.pipelines/v2/sdk_v2-pipeline-plan.md @@ -285,7 +285,7 @@ purposes: Versions are pipeline-level variables, currently: * `ortVersion` `1.28.0` (`Microsoft.ML.OnnxRuntime`) -* `genaiVersion` `0.15.2` for releases; selected ORT-Nightly version for non-release CI +* `genaiVersion` `0.15.3` for releases; selected ORT-Nightly version for non-release CI (`Microsoft.ML.OnnxRuntimeGenAI.Foundry`) * `winmlVersion` `2.1.70` (`Microsoft.Windows.AI.MachineLearning`, WinML 2.x reg-free) diff --git a/sdk_v2/cpp/CMakeLists.txt b/sdk_v2/cpp/CMakeLists.txt index 1a070e397..5487ebccd 100644 --- a/sdk_v2/cpp/CMakeLists.txt +++ b/sdk_v2/cpp/CMakeLists.txt @@ -109,6 +109,15 @@ endif() # ORT and ORT GenAI — acquired via FetchContent from nuget.org. list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") find_package(OnnxRuntimeGenAI REQUIRED) + +file(STRINGS "${ORT_GENAI_HEADER_DIR}/ort_genai_c.h" _OGA_ENGINE_DECLARATION REGEX "OgaCreateEngine") +if(_OGA_ENGINE_DECLARATION) + set(FOUNDRY_LOCAL_HAS_OGA_ENGINE ON) + message(STATUS "ORT GenAI Engine API: enabled") +else() + set(FOUNDRY_LOCAL_HAS_OGA_ENGINE OFF) + message(STATUS "ORT GenAI Engine API: unavailable") +endif() find_package(OnnxRuntime REQUIRED) # WinML EP Catalog — Windows-only, for hardware EP discovery and download. The @@ -277,6 +286,13 @@ set(FOUNDRY_LOCAL_SOURCES ${FOUNDRY_LOCAL_INTERNAL_HEADERS} ) +if(FOUNDRY_LOCAL_HAS_OGA_ENGINE) + list(APPEND FOUNDRY_LOCAL_SOURCES + src/inferencing/generative/chat/onnx_chat_engine.cc + src/inferencing/generative/chat/onnx_engine_chat_generator.cc + ) +endif() + # 1DS bridge — always compiled for Foundry Local Core. list(APPEND FOUNDRY_LOCAL_SOURCES src/telemetry/one_ds_telemetry.cc) @@ -401,6 +417,9 @@ configure_file( # static library targets re-use these object files, avoiding a double build. # -------------------------------------------------------------------------- add_library(foundry_local_objects OBJECT ${FOUNDRY_LOCAL_SOURCES}) +if(FOUNDRY_LOCAL_HAS_OGA_ENGINE) + target_compile_definitions(foundry_local_objects PRIVATE FOUNDRY_LOCAL_HAS_OGA_ENGINE=1) +endif() set_target_properties(foundry_local_objects PROPERTIES POSITION_INDEPENDENT_CODE ON) foundry_local_configure_target(foundry_local_objects PUBLIC) # FL_STATIC_LIBRARY makes FL_EXPORT empty on Windows so the two exported entry diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.cc index c90c4d785..416982ff6 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.cc @@ -4,6 +4,10 @@ namespace fl { +std::optional ChatGenerator::GetTurnUsage() const { + return std::nullopt; +} + std::string ChatGenerator::GenerateAll() { std::string result; diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.h b/sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.h index 94b38dcc6..9f23ed191 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.h @@ -5,9 +5,19 @@ #include #include #include +#include namespace fl { +class GenAIModelInstance; +struct MessageItem; +struct SearchOptions; + +struct ChatTurnUsage { + int prompt_tokens = 0; + int generated_tokens = 0; +}; + /// Abstract interface for token-by-token text generation. /// One generator per request — not reusable, not thread-safe. /// Follows the classic pull-based iterator pattern: @@ -46,6 +56,22 @@ class ChatGenerator { /// After cancellation, IsDone() should return true on the next check. virtual void Cancel() = 0; + /// Append a new conversational turn to retained model state. + virtual int AppendMessages(const std::vector& new_messages, + GenAIModelInstance& model, + const std::string& tools_json, + const SearchOptions& options, + const std::string& reasoning_start = {}) = 0; + + /// Returns whether this backend can rewind retained model state directly. + virtual bool CanRewind() const = 0; + + /// Rewind retained model state to a prior token position. + virtual void RewindTo(int token_count) = 0; + + /// Return exact usage for the most recently completed turn when the backend exposes it. + virtual std::optional GetTurnUsage() const; + protected: ChatGenerator() = default; }; diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc index d1ec7179f..9f93bec3d 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc @@ -5,6 +5,9 @@ #include "contracts/chat_completions.h" #include "contracts/chat_completions_converter.h" +#ifdef FOUNDRY_LOCAL_HAS_OGA_ENGINE +#include "inferencing/generative/chat/onnx_engine_chat_generator.h" +#endif #include "inferencing/generative/chat/onnx_chat_generator.h" #include "inferencing/generative/chat/reasoning_stream_splitter.h" #include "inferencing/generative/genai_model_instance.h" @@ -48,6 +51,22 @@ void ApplyToolChoiceToContext(std::optional tool_choice, ToolCallC } } +std::unique_ptr CreateTextChatGenerator(const std::vector& messages, + const SearchOptions& options, + GenAIModelInstance& model, + const ToolCallContext& tool_ctx) { + if (model.GetGenAIConfig().GetChatBackendKind() != ChatBackendKind::kGenerator) { +#ifdef FOUNDRY_LOCAL_HAS_OGA_ENGINE + return OnnxEngineChatGenerator::Create(messages, options, model, tool_ctx); +#else + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, + "model requires the ORT GenAI Engine API, but this build does not provide it"); +#endif + } + + return OnnxChatGenerator::Create(messages, options, model, tool_ctx, /*use_full_context=*/true); +} + } // namespace ChatSession::ChatSession(const fl::Model& catalog_model, GenAIModelInstance& model, ILogger& logger, ITelemetry& telemetry) @@ -71,7 +90,9 @@ ChatSession::ChatSession(ChatSession&& other) noexcept history_(std::move(other.history_)), turns_(std::move(other.turns_)), session_options_(std::move(other.session_options_)), - cached_generator_(std::move(other.cached_generator_)) { + cached_generator_(std::move(other.cached_generator_)), + cached_tool_ctx_(std::move(other.cached_tool_ctx_)), + cached_search_options_(std::move(other.cached_search_options_)) { other.owns_session_ = false; } @@ -465,6 +486,12 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) int prompt_tokens = 0; int pre_turn_token_count = 0; + if (cached_generator_ && + !cached_search_options_.HasSameRetainedGenerationSettings(effective_options)) { + cached_generator_.reset(); + cached_tool_ctx_ = {}; + } + if (cached_generator_) { // Check if guidance requirements changed since the generator was created. Guidance (LARK grammar) is baked into // the OGA generator at creation time and cannot be changed. @@ -474,7 +501,9 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) bool prev_needs_guidance = cached_tool_ctx_.tool_output && cached_tool_ctx_.HasTools(); bool curr_needs_guidance = turn_tool_ctx.tool_output && turn_tool_ctx.HasTools(); - if (prev_needs_guidance != curr_needs_guidance) { + const bool static_engine = + Model().GetGenAIConfig().GetChatBackendKind() == ChatBackendKind::kStaticEngine; + if (prev_needs_guidance != curr_needs_guidance || static_engine) { // Guidance requirements changed — invalidate. The branch below will rebuild from full history. cached_generator_.reset(); cached_tool_ctx_ = {}; @@ -482,10 +511,10 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) // Continuous decoding: append only the new messages to the existing generator. pre_turn_token_count = cached_generator_->TokenCount(); const std::string reasoning_start_marker = - cached_tool_ctx_.reasoning_start.empty() ? std::string("") : cached_tool_ctx_.reasoning_start; + cached_tool_ctx_.reasoning_start.empty() ? std::string("") : cached_tool_ctx_.reasoning_start; prompt_tokens = cached_generator_->AppendMessages( - new_messages, Model(), cached_tool_ctx_.tools_json, - cached_tool_ctx_.supports_reasoning ? reasoning_start_marker : std::string{}); + new_messages, Model(), cached_tool_ctx_.tools_json, effective_options, + cached_tool_ctx_.supports_reasoning ? reasoning_start_marker : std::string{}); // Refresh per-turn fields (tool_choice, guidance) while keeping session-level definitions stable. UpdateToolContextForTurn(request, cached_tool_ctx_); @@ -502,7 +531,7 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) all_messages.insert(all_messages.end(), history_.begin(), history_.end()); all_messages.insert(all_messages.end(), new_messages.begin(), new_messages.end()); - std::unique_ptr generator; + std::unique_ptr generator; if (media_turn) { // Media is single-shot: the generator is dropped after the turn (see // CommitTurn cleanup below) because AppendMessages can't extend a @@ -511,18 +540,18 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) // gigabytes (262k tokens × 28 layers × 8 heads × 128 dims for // qwen3-vl-2b ≈ 120 GB). Bound it to prompt + max_output_tokens. generator = OnnxChatGenerator::CreateWithMedia(all_messages, effective_options, Model(), images, audios, - tool_ctx, /*use_full_context*/ false); + tool_ctx, /*use_full_context*/ false); } else { - generator = OnnxChatGenerator::Create(all_messages, effective_options, Model(), tool_ctx, - /*use_full_context*/ true); + generator = CreateTextChatGenerator(all_messages, effective_options, Model(), tool_ctx); } prompt_tokens = generator->PromptTokenCount(); cached_generator_ = std::move(generator); cached_tool_ctx_ = std::move(tool_ctx); + cached_search_options_ = effective_options; } - int max_output = effective_options.max_output_tokens.value_or(0); + const int max_output = ResolveMaxOutputTokens(effective_options); // Generate token-by-token with optional streaming. // Check request.canceled each iteration — a streaming callback returning @@ -620,17 +649,33 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) emit_segments(splitter.Flush()); flush_accumulator(); + if (request.canceled) { + cached_generator_->Cancel(); + } + int total_tokens = cached_generator_->TokenCount(); + if (const auto turn_usage = cached_generator_->GetTurnUsage()) { + prompt_tokens = turn_usage->prompt_tokens; + total_tokens = turn_usage->prompt_tokens + turn_usage->generated_tokens; + } + bool discard_generator = false; if (request.canceled) { - // Rewind the generator to undo this turn's input. The generator remains valid - // for the next attempt — the caller can re-send the same input. - cached_generator_->RewindTo(pre_turn_token_count); + if (cached_generator_->CanRewind()) { + cached_generator_->RewindTo(pre_turn_token_count); + } else { + discard_generator = true; + } } ProcessGeneratedOutput(std::move(text), cached_tool_ctx_, effective_options, request.canceled, response, prompt_tokens, total_tokens, std::move(streamed_tool_calls)); + if (discard_generator) { + cached_generator_.reset(); + cached_tool_ctx_ = {}; + } + // Commit input messages + assistant reply to history only on success (not cancelled) if (!request.canceled) { // LARK tool grammar is a single-shot finite parse. If generation was truncated while grammar was @@ -644,7 +689,9 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) bool grammar_was_active = cached_tool_ctx_.tool_output && cached_tool_ctx_.HasTools(); bool reasoning_was_active = cached_tool_ctx_.supports_reasoning; - if (grammar_was_active || reasoning_was_active) { + const bool static_engine = + Model().GetGenAIConfig().GetChatBackendKind() == ChatBackendKind::kStaticEngine; + if (grammar_was_active || reasoning_was_active || static_engine) { cached_generator_.reset(); cached_tool_ctx_ = {}; } @@ -725,7 +772,7 @@ void ChatSession::ProcessChatCompletionsJson(const std::string& request_json, co } // Create generator - auto generator = OnnxChatGenerator::Create(messages, options, Model(), tool_ctx); + auto generator = CreateTextChatGenerator(messages, options, Model(), tool_ctx); int prompt_tokens = generator->PromptTokenCount(); auto streaming_callback = CreateCallbackHandler(original_request); @@ -840,7 +887,15 @@ void ChatSession::ProcessChatCompletionsJson(const std::string& request_json, co emit_ready_calls(out.ready_calls); } + if (original_request.canceled) { + generator->Cancel(); + } + int total_tokens = generator->TokenCount(); + if (const auto turn_usage = generator->GetTurnUsage()) { + prompt_tokens = turn_usage->prompt_tokens; + total_tokens = turn_usage->prompt_tokens + turn_usage->generated_tokens; + } // Process the generated output into response items (MessageItem, ToolCallItem, etc.) // This also updates finish_reason, and usage on the response. Streamed-parsed tool calls are reused so call_ids @@ -937,8 +992,11 @@ void ChatSession::UndoTurns(size_t count) { // Undoing all turns — destroy the generator entirely cached_generator_.reset(); cached_tool_ctx_ = {}; - } else { + } else if (cached_generator_->CanRewind()) { cached_generator_->RewindTo(target.pre_turn_token_count); + } else { + cached_generator_.reset(); + cached_tool_ctx_ = {}; } } diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h index 57253963d..d7811930b 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h @@ -18,7 +18,7 @@ namespace fl { class GenAIModelInstance; -class OnnxChatGenerator; +class ChatGenerator; using GeneratedOutputEvent = std::variant; @@ -120,11 +120,14 @@ class ChatSession : public Session { // Cached generator for continuous decoding (non-JSON path only). // Null until first non-JSON ProcessRequestImpl call. - std::unique_ptr cached_generator_; + std::unique_ptr cached_generator_; // Tool context used when creating the cached generator. // Reused for subsequent turns to maintain tool definition consistency. ToolCallContext cached_tool_ctx_; + + // Search settings baked into the retained generator or Engine request. + SearchOptions cached_search_options_; }; } // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc index 38333ed1d..1b85d4348 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.cc @@ -77,6 +77,26 @@ std::string BuildChatPrompt(const std::vector& messages, return model.GetPreprocessor().ApplyChatTemplate(messages_str.c_str(), tools_ptr, /*add_generation_prompt=*/true); } +std::string BuildChatContinuationPrompt(const std::vector& messages, + GenAIModelInstance& model, + const std::string& tools_json) { + constexpr std::string_view kAssistantMarker = "__foundry_engine_assistant_boundary__"; + + std::vector marked_messages; + marked_messages.reserve(messages.size() + 1); + marked_messages.emplace_back(FOUNDRY_LOCAL_ROLE_ASSISTANT, std::string(kAssistantMarker)); + marked_messages.insert(marked_messages.end(), messages.begin(), messages.end()); + + auto marked_prompt = BuildChatPrompt(marked_messages, model, tools_json); + const auto marker_position = marked_prompt.find(kAssistantMarker); + if (marker_position == std::string::npos) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, + "chat template did not preserve assistant content needed to build an Engine continuation"); + } + + return marked_prompt.substr(marker_position + kAssistantMarker.size()); +} + std::unique_ptr EncodePrompt(const std::string& prompt, GenAIModelInstance& model) { return model.GetPreprocessor().Encode(prompt.c_str()); diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h index be703c0f6..59543757e 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_template.h @@ -42,6 +42,12 @@ std::string BuildChatPrompt(const std::vector& messages, GenAIModelInstance& model, const std::string& tools_json = ""); +/// Build the fragment appended after an Engine-generated assistant response. +/// Engine does not retain the generated EOS token, so this includes the template's assistant-turn boundary. +std::string BuildChatContinuationPrompt(const std::vector& messages, + GenAIModelInstance& model, + const std::string& tools_json = ""); + /// Encode a prompt string into token sequences using the model's shared tokenizer (thread-safe). /// Returns a unique_ptr to OgaSequences. Caller takes ownership. /// diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc new file mode 100644 index 000000000..6cb803f16 --- /dev/null +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc @@ -0,0 +1,347 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "inferencing/generative/chat/onnx_chat_engine.h" + +#include "exception.h" +#include "inferencing/generative/genai_model_instance.h" +#include "inferencing/generative/toolcalling/tool_call_context.h" + +#include + +#include +#include +#include + +namespace fl { + +struct OnnxChatEngine::NativeConversation { + std::unique_ptr request; + std::shared_ptr state; +}; + +OnnxChatEngine::OnnxChatEngine(GenAIModelInstance& model) : model_(model) { + std::promise initialized; + auto ready = initialized.get_future(); + worker_ = std::thread(&OnnxChatEngine::WorkerLoop, this, std::move(initialized)); + try { + ready.get(); + } catch (...) { + if (worker_.joinable()) { + worker_.join(); + } + throw; + } +} + +OnnxChatEngine::~OnnxChatEngine() { + { + std::lock_guard lock(command_mutex_); + stopping_ = true; + } + command_cv_.notify_one(); + + if (worker_.joinable()) { + worker_.join(); + } +} + +std::shared_ptr OnnxChatEngine::CreateConversation( + const SearchOptions& options, const ToolCallContext& tool_ctx, int input_token_count) { + auto conversation = std::shared_ptr(new Conversation()); + auto completion = std::make_shared>(); + auto ready = completion->get_future(); + + Enqueue( + [this, conversation, options, tool_ctx, input_token_count, completion]() { + auto params = OgaGeneratorParams::Create(model_.GetOgaModel()); + ApplySearchOptions(options, input_token_count, model_.GetGenAIConfig(), *params, model_.EP(), + /*use_full_context=*/true); + ApplyGuidanceOptions(tool_ctx, *params); + auto request = engine_->CreateRequest(*params); + conversations_.emplace(conversation.get(), + std::make_unique( + NativeConversation{std::move(request), conversation})); + completion->set_value(); + }, + [completion](std::exception_ptr error) { completion->set_exception(error); }); + + ready.get(); + return conversation; +} + +uint64_t OnnxChatEngine::BeginTurn(const std::shared_ptr& conversation, + std::span input_ids, + std::optional max_output_tokens) { + if (input_ids.empty()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "Engine turn input must not be empty"); + } + + auto tokens = std::vector(input_ids.begin(), input_ids.end()); + auto completion = std::make_shared>(); + auto ready = completion->get_future(); + + Enqueue( + [this, conversation, tokens = std::move(tokens), max_output_tokens, completion]() { + auto& native = FindNative(conversation); + std::unique_ptr options; + if (max_output_tokens.has_value()) { + options = native.request->CreateTurnOptions(); + options->SetMaxGeneratedTokens(static_cast(*max_output_tokens)); + } + + { + std::lock_guard lock(conversation->mutex); + if (!conversation->turn_finished) { + throw std::runtime_error("Cannot begin an Engine turn while another turn is active."); + } + conversation->tokens.clear(); + conversation->error = nullptr; + conversation->result = {}; + conversation->turn_finished = false; + } + + const uint64_t turn_id = native.request->BeginTurn(tokens.data(), tokens.size(), options.get()); + { + std::lock_guard lock(conversation->mutex); + conversation->turn_id = turn_id; + conversation->sequence_length += tokens.size(); + } + completion->set_value(turn_id); + }, + [conversation, completion](std::exception_ptr error) { + { + std::lock_guard lock(conversation->mutex); + conversation->error = error; + conversation->turn_finished = true; + } + conversation->cv.notify_all(); + completion->set_exception(error); + }); + + return ready.get(); +} + +std::optional OnnxChatEngine::WaitForToken(const std::shared_ptr& conversation) { + std::unique_lock lock(conversation->mutex); + conversation->cv.wait(lock, [&]() { + return !conversation->tokens.empty() || conversation->turn_finished || conversation->error; + }); + + if (conversation->error) { + std::rethrow_exception(conversation->error); + } + if (conversation->tokens.empty()) { + return std::nullopt; + } + + const int32_t token = conversation->tokens.front(); + conversation->tokens.pop_front(); + return token; +} + +bool OnnxChatEngine::IsTurnFinished(const std::shared_ptr& conversation) const { + std::lock_guard lock(conversation->mutex); + return conversation->turn_finished && conversation->tokens.empty(); +} + +OnnxChatEngine::TurnResult OnnxChatEngine::GetTurnResult( + const std::shared_ptr& conversation) const { + std::unique_lock lock(conversation->mutex); + conversation->cv.wait(lock, [&]() { return conversation->turn_finished || conversation->error; }); + if (conversation->error) { + std::rethrow_exception(conversation->error); + } + return conversation->result; +} + +size_t OnnxChatEngine::SequenceLength(const std::shared_ptr& conversation) const { + std::lock_guard lock(conversation->mutex); + return conversation->sequence_length; +} + +void OnnxChatEngine::Cancel(const std::shared_ptr& conversation) { + Enqueue( + [this, conversation]() { + auto& native = FindNative(conversation); + uint64_t turn_id; + { + std::lock_guard lock(conversation->mutex); + turn_id = conversation->turn_id; + } + if (turn_id != 0) { + native.request->CancelTurn(turn_id); + } + }, + [conversation](std::exception_ptr error) { + std::lock_guard lock(conversation->mutex); + conversation->error = error; + conversation->turn_finished = true; + conversation->cv.notify_all(); + }); +} + +void OnnxChatEngine::Close(const std::shared_ptr& conversation) { + auto completion = std::make_shared>(); + auto ready = completion->get_future(); + Enqueue( + [this, conversation, completion]() { + auto it = conversations_.find(conversation.get()); + if (it != conversations_.end()) { + it->second->request->Close(); + conversations_.erase(it); + } + { + std::lock_guard lock(conversation->mutex); + conversation->closed = true; + conversation->turn_finished = true; + } + conversation->cv.notify_all(); + completion->set_value(); + }, + [completion](std::exception_ptr error) { completion->set_exception(error); }); + ready.get(); +} + +void OnnxChatEngine::Enqueue(std::function command, + std::function fail) { + std::exception_ptr error; + { + std::lock_guard lock(command_mutex_); + error = fatal_error_; + if (stopping_) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Engine dispatcher is shutting down"); + } + if (!error) { + commands_.push_back({std::move(command), std::move(fail)}); + } + } + + if (error) { + fail(error); + return; + } + command_cv_.notify_one(); +} + +void OnnxChatEngine::WorkerLoop(std::promise initialized) { + try { + engine_ = OgaEngine::Create(model_.GetOgaModel()); + event_buffer_ = engine_->CreateEventBuffer(model_.GetGenAIConfig().EngineMaxBatchSize().value_or(1) * 2); + initialized.set_value(); + } catch (...) { + event_buffer_.reset(); + engine_.reset(); + initialized.set_exception(std::current_exception()); + return; + } + + try { + while (true) { + std::deque commands; + { + std::unique_lock lock(command_mutex_); + if (commands_.empty() && !engine_->HasPendingRequests() && !stopping_) { + command_cv_.wait(lock, [&]() { return stopping_ || !commands_.empty(); }); + } + commands.swap(commands_); + if (stopping_ && commands.empty() && !engine_->HasPendingRequests()) { + break; + } + } + + for (auto& command : commands) { + try { + command.run(); + } catch (...) { + command.fail(std::current_exception()); + } + } + if (engine_->HasPendingRequests()) { + RouteEvents(); + } + } + } catch (...) { + auto error = std::current_exception(); + std::deque commands; + { + std::lock_guard lock(command_mutex_); + fatal_error_ = error; + stopping_ = true; + commands.swap(commands_); + } + + FailAll(error); + for (auto& command : commands) { + command.fail(error); + } + } + + conversations_.clear(); + event_buffer_.reset(); + engine_.reset(); +} + +void OnnxChatEngine::RouteEvents() { + engine_->Run(*event_buffer_); + for (size_t i = 0; i < event_buffer_->Count(); ++i) { + const auto* event = event_buffer_->Get(i); + const auto request = event->Request(); + if (!request) { + continue; + } + + auto it = std::find_if(conversations_.begin(), conversations_.end(), [&](const auto& entry) { + return entry.second->request.get() == &request->get(); + }); + if (it == conversations_.end()) { + continue; + } + + auto& conversation = it->second->state; + const auto flags = event->Flags(); + { + std::lock_guard lock(conversation->mutex); + if ((flags & OgaEngineEventFlag_Token) != 0) { + conversation->tokens.push_back(event->Token()); + ++conversation->sequence_length; + } + if ((flags & OgaEngineEventFlag_TurnFinished) != 0) { + const auto& usage = event->Usage(); + conversation->result.prompt_tokens = usage.PromptTokens(); + conversation->result.generated_tokens = usage.GeneratedTokens(); + conversation->result.cached_prompt_tokens = usage.CachedPromptTokens(); + conversation->result.finish_reason = event->FinishReason(); + conversation->turn_finished = true; + } + if ((flags & OgaEngineEventFlag_Failed) != 0) { + conversation->error = std::make_exception_ptr( + std::runtime_error("ORT GenAI Engine request failed with error code " + + std::to_string(event->ErrorCode()))); + conversation->turn_finished = true; + } + } + conversation->cv.notify_all(); + } +} + +void OnnxChatEngine::FailAll(std::exception_ptr error) { + for (auto& [_, native] : conversations_) { + { + std::lock_guard lock(native->state->mutex); + native->state->error = error; + native->state->turn_finished = true; + } + native->state->cv.notify_all(); + } +} + +OnnxChatEngine::NativeConversation& OnnxChatEngine::FindNative( + const std::shared_ptr& conversation) { + auto it = conversations_.find(conversation.get()); + if (it == conversations_.end()) { + throw std::runtime_error("Engine conversation is closed or does not belong to this model."); + } + return *it->second; +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.h b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.h new file mode 100644 index 000000000..324ea0136 --- /dev/null +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.h @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "inferencing/generative/chat/search_options.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct OgaEngine; +struct OgaEngineEventBuffer; +struct OgaRequest; + +namespace fl { + +class GenAIModelInstance; +struct ToolCallContext; + +/// Owns one ORT GenAI Engine and serializes every Engine operation onto its owner thread. +class OnnxChatEngine { + public: + struct TurnResult { + uint64_t prompt_tokens = 0; + uint64_t generated_tokens = 0; + uint64_t cached_prompt_tokens = 0; + uint32_t finish_reason = 0; + }; + + class Conversation { + public: + Conversation(const Conversation&) = delete; + Conversation& operator=(const Conversation&) = delete; + + private: + friend class OnnxChatEngine; + Conversation() = default; + + std::mutex mutex; + std::condition_variable cv; + std::deque tokens; + std::exception_ptr error; + TurnResult result; + uint64_t turn_id = 0; + size_t sequence_length = 0; + bool turn_finished = true; + bool closed = false; + }; + + explicit OnnxChatEngine(GenAIModelInstance& model); + ~OnnxChatEngine(); + + OnnxChatEngine(const OnnxChatEngine&) = delete; + OnnxChatEngine& operator=(const OnnxChatEngine&) = delete; + + std::shared_ptr CreateConversation(const SearchOptions& options, + const ToolCallContext& tool_ctx, + int input_token_count); + uint64_t BeginTurn(const std::shared_ptr& conversation, + std::span input_ids, + std::optional max_output_tokens); + std::optional WaitForToken(const std::shared_ptr& conversation); + bool IsTurnFinished(const std::shared_ptr& conversation) const; + TurnResult GetTurnResult(const std::shared_ptr& conversation) const; + size_t SequenceLength(const std::shared_ptr& conversation) const; + void Cancel(const std::shared_ptr& conversation); + void Close(const std::shared_ptr& conversation); + + private: + struct NativeConversation; + struct PendingCommand { + std::function run; + std::function fail; + }; + + void Enqueue(std::function command, std::function fail); + void WorkerLoop(std::promise initialized); + void RouteEvents(); + void FailAll(std::exception_ptr error); + NativeConversation& FindNative(const std::shared_ptr& conversation); + + GenAIModelInstance& model_; + mutable std::mutex command_mutex_; + std::condition_variable command_cv_; + std::deque commands_; + std::exception_ptr fatal_error_; + bool stopping_ = false; + std::thread worker_; + + // Owner-thread-only state. WorkerLoop clears these before it exits. + std::unique_ptr engine_; + std::unique_ptr event_buffer_; + std::unordered_map> conversations_; +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc index 482e71493..d3749eab6 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc @@ -167,6 +167,7 @@ void OnnxChatGenerator::Cancel() { int OnnxChatGenerator::AppendMessages(const std::vector& new_messages, GenAIModelInstance& model, const std::string& tools_json, + const SearchOptions& /*options*/, const std::string& reasoning_start) { if (new_messages.empty()) { FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "new_messages must not be empty"); @@ -399,42 +400,8 @@ std::unique_ptr OnnxChatGenerator::CreateImpl(const std::vect ApplySearchOptions(options, input_token_count, model.GetGenAIConfig(), *gen_params, model.EP(), use_full_context, default_max_output); - // 5. Compute guidance for constrained decoding. - // Priority: user-specified guidance (from response_format) > auto-generated LARK grammar. - // Matches C# GetGuidance() — always compute, then guard application. - std::string guidance_type; - std::string guidance_data; - - if (!tool_ctx.guidance_type.empty() && !tool_ctx.guidance_data.empty()) { - // User specified guidance via response_format - guidance_type = tool_ctx.guidance_type; - guidance_data = tool_ctx.guidance_data; - } else { - // Auto-generate LARK grammar from tool definitions and reasoning state - std::string json_schema; - if (tool_ctx.HasTools()) { - json_schema = BuildToolJsonSchema(tool_ctx); - } - - guidance_data = BuildLarkGrammar(tool_ctx, json_schema); - if (!guidance_data.empty()) { - guidance_type = "lark_grammar"; - } - } - - // Apply auto-generated tool guidance for both auto and required tool choice. The grammar itself permits text when - // text_output is true. ChatSession invalidates the generator after guided turns because a completed finite grammar - // signals EOS and cannot be reused safely for continuous decoding. - bool tool_guidance_enabled = tool_ctx.tool_output && tool_ctx.HasTools(); - - if (!guidance_type.empty() && !guidance_data.empty() && tool_guidance_enabled) { - try { - gen_params->SetGuidance(guidance_type.c_str(), guidance_data.c_str()); - } catch (const std::runtime_error& e) { - // SetGuidance may not be supported by all models; continue without guidance - (void)e; - } - } + // 5. Apply constrained decoding for tool output when supported. + ApplyGuidanceOptions(tool_ctx, *gen_params); // 6. Create the Generator and feed it the prompt. // Text path: append the encoded token sequences. diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.h b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.h index 478949786..6b6eb0b32 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.h @@ -60,6 +60,7 @@ class OnnxChatGenerator : public ChatGenerator { int AppendMessages(const std::vector& new_messages, GenAIModelInstance& model, const std::string& tools_json, + const SearchOptions& options, const std::string& reasoning_start = {}); /// Recompute whether the appended prompt ends in an open reasoning block. @@ -70,7 +71,8 @@ class OnnxChatGenerator : public ChatGenerator { /// Rewind the generator to a previous token position. /// Used for error recovery — restores the KV cache to the state before the last turn. - void RewindTo(int token_count); + bool CanRewind() const override { return true; } + void RewindTo(int token_count) override; /// Factory: create a text-only chat generator. /// diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.cc new file mode 100644 index 000000000..1837d8b31 --- /dev/null +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.cc @@ -0,0 +1,154 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "inferencing/generative/chat/onnx_engine_chat_generator.h" + +#include "exception.h" +#include "inferencing/generative/chat/chat_template.h" +#include "inferencing/generative/genai_model_instance.h" + +#include + +#include + +namespace fl { + +OnnxEngineChatGenerator::OnnxEngineChatGenerator( + OnnxChatEngine& engine, + std::shared_ptr conversation, + std::unique_ptr stream, + std::unique_ptr stream_with_special, + GenAIModelInstance& model, + int prompt_token_count) + : engine_(engine), + conversation_(std::move(conversation)), + stream_(std::move(stream)), + stream_with_special_(std::move(stream_with_special)), + model_(model), + prompt_token_count_(prompt_token_count) {} + +OnnxEngineChatGenerator::~OnnxEngineChatGenerator() { + try { + engine_.Close(conversation_); + } catch (...) { + } +} + +bool OnnxEngineChatGenerator::IsDone() const { + return cancelled_ || engine_.IsTurnFinished(conversation_); +} + +void OnnxEngineChatGenerator::GenerateNextToken() { + if (cancelled_) { + return; + } + + try { + current_token_ = engine_.WaitForToken(conversation_); + } catch (const std::runtime_error& e) { + if (!cancelled_) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, std::string("Engine token generation failed: ") + e.what()); + } + } +} + +std::string OnnxEngineChatGenerator::Decode() { + if (!current_token_) { + return ""; + } + + const int32_t token_id = *current_token_; + current_token_.reset(); + const char* token_text = stream_->Decode(token_id); + const char* special_text = stream_with_special_->Decode(token_id); + std::string token = token_text ? token_text : ""; + + if (special_text != nullptr && token_text != nullptr && std::string(special_text) != token) { + const std::string special(special_text); + const bool surfaced_special = + special.find("tool_call") != std::string::npos || special.find("think") != std::string::npos; + const auto& eos_ids = model_.GetPreprocessor().GetEosTokenIds(); + const bool eos = std::find(eos_ids.begin(), eos_ids.end(), token_id) != eos_ids.end(); + if (surfaced_special && !eos) { + return special; + } + } + + return token; +} + +int OnnxEngineChatGenerator::TokenCount() const { + return static_cast(engine_.SequenceLength(conversation_)); +} + +int OnnxEngineChatGenerator::PromptTokenCount() const { + return prompt_token_count_; +} + +void OnnxEngineChatGenerator::Cancel() { + cancelled_ = true; + engine_.Cancel(conversation_); +} + +int OnnxEngineChatGenerator::AppendMessages(const std::vector& new_messages, + GenAIModelInstance& model, + const std::string& tools_json, + const SearchOptions& options, + const std::string& /*reasoning_start*/) { + if (new_messages.empty()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "new_messages must not be empty"); + } + + auto prompt = BuildChatContinuationPrompt(new_messages, model, tools_json); + auto sequences = EncodePrompt(prompt, model); + const int count = static_cast(sequences->SequenceCount(0)); + const auto* data = sequences->SequenceData(0); + engine_.BeginTurn(conversation_, std::span(data, static_cast(count)), + ResolveMaxOutputTokens(options)); + prompt_token_count_ = count; + cancelled_ = false; + return count; +} + +void OnnxEngineChatGenerator::RewindTo(int /*token_count*/) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "Engine request rewind is unavailable; recreate the request from retained conversation history"); +} + +std::optional OnnxEngineChatGenerator::GetTurnUsage() const { + const auto result = engine_.GetTurnResult(conversation_); + return ChatTurnUsage{ + static_cast(result.prompt_tokens + result.cached_prompt_tokens), + static_cast(result.generated_tokens), + }; +} + +std::unique_ptr OnnxEngineChatGenerator::Create( + const std::vector& messages, + const SearchOptions& options, + GenAIModelInstance& model, + const ToolCallContext& tool_ctx) { + if (messages.empty()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "messages must not be empty"); + } + + auto* engine = model.GetChatEngine(); + if (!engine) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "model does not own a chat Engine"); + } + + auto prompt = BuildChatPrompt(messages, model, tool_ctx.tools_json); + auto sequences = EncodePrompt(prompt, model); + const int prompt_token_count = static_cast(sequences->SequenceCount(0)); + auto conversation = engine->CreateConversation(options, tool_ctx, prompt_token_count); + const auto* data = sequences->SequenceData(0); + engine->BeginTurn(conversation, std::span(data, static_cast(prompt_token_count)), + ResolveMaxOutputTokens(options)); + + return std::unique_ptr( + new OnnxEngineChatGenerator(*engine, std::move(conversation), + model.GetPreprocessor().CreateTokenizerStream(), + model.GetPreprocessor().CreateSpecialTokenizerStream(), model, + prompt_token_count)); +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.h b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.h new file mode 100644 index 000000000..20d11e473 --- /dev/null +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.h @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "inferencing/generative/chat/chat_generator.h" +#include "inferencing/generative/chat/onnx_chat_engine.h" +#include "inferencing/generative/chat/search_options.h" +#include "inferencing/generative/toolcalling/tool_call_context.h" + +#include +#include +#include + +struct OgaTokenizerStream; + +namespace fl { + +class GenAIModelInstance; + +/// ChatGenerator adapter for a conversation scheduled by a model-owned ORT GenAI Engine. +class OnnxEngineChatGenerator final : public ChatGenerator { + public: + ~OnnxEngineChatGenerator() override; + + bool IsDone() const override; + void GenerateNextToken() override; + std::string Decode() override; + int TokenCount() const override; + int PromptTokenCount() const override; + void Cancel() override; + int AppendMessages(const std::vector& new_messages, + GenAIModelInstance& model, + const std::string& tools_json, + const SearchOptions& options, + const std::string& reasoning_start = {}) override; + bool CanRewind() const override { return false; } + void RewindTo(int token_count) override; + std::optional GetTurnUsage() const override; + + static std::unique_ptr Create( + const std::vector& messages, + const SearchOptions& options, + GenAIModelInstance& model, + const ToolCallContext& tool_ctx); + + private: + OnnxEngineChatGenerator(OnnxChatEngine& engine, + std::shared_ptr conversation, + std::unique_ptr stream, + std::unique_ptr stream_with_special, + GenAIModelInstance& model, + int prompt_token_count); + + OnnxChatEngine& engine_; + std::shared_ptr conversation_; + std::unique_ptr stream_; + std::unique_ptr stream_with_special_; + GenAIModelInstance& model_; + int prompt_token_count_ = 0; + std::optional current_token_; + std::atomic cancelled_{false}; +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/search_options.cc b/sdk_v2/cpp/src/inferencing/generative/chat/search_options.cc index 4ae6df6a8..f4352f936 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/search_options.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/search_options.cc @@ -2,6 +2,8 @@ // Licensed under the MIT License. #include "inferencing/generative/chat/search_options.h" #include "exception.h" +#include "inferencing/generative/toolcalling/grammar.h" +#include "inferencing/generative/toolcalling/tool_call_context.h" #include #include @@ -10,6 +12,44 @@ namespace fl { +int ResolveMaxOutputTokens(const SearchOptions& options, int default_max_output_tokens) { + const int max_output = options.max_output_tokens.value_or(default_max_output_tokens); + if (max_output < 1) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "max_output_tokens must be >= 1"); + } + + return max_output; +} + +void ApplyGuidanceOptions(const ToolCallContext& tool_ctx, OgaGeneratorParams& gen_params) { + std::string guidance_type; + std::string guidance_data; + + if (!tool_ctx.guidance_type.empty() && !tool_ctx.guidance_data.empty()) { + guidance_type = tool_ctx.guidance_type; + guidance_data = tool_ctx.guidance_data; + } else { + std::string json_schema; + if (tool_ctx.HasTools()) { + json_schema = BuildToolJsonSchema(tool_ctx); + } + + guidance_data = BuildLarkGrammar(tool_ctx, json_schema); + if (!guidance_data.empty()) { + guidance_type = "lark_grammar"; + } + } + + const bool tool_guidance_enabled = tool_ctx.tool_output && tool_ctx.HasTools(); + if (!guidance_type.empty() && !guidance_data.empty() && tool_guidance_enabled) { + try { + gen_params.SetGuidance(guidance_type.c_str(), guidance_data.c_str()); + } catch (const std::runtime_error&) { + // Some model/runtime combinations do not implement guidance. Preserve the existing unguided behavior. + } + } +} + int ApplySearchOptions(const SearchOptions& options, int input_token_count, const GenAIConfig& config, @@ -31,10 +71,7 @@ int ApplySearchOptions(const SearchOptions& options, // The catalog's maxOutputTokens is informational metadata only and is intentionally NOT used to clamp generation: // it is commonly a conservative 2048 that would wrongly cap larger contexts (e.g. the 3072 vision default). A // user-supplied max_output_tokens is honored as-is and only rejected if input+output exceeds max_length below. - int max_output = options.max_output_tokens.value_or(default_max_output_tokens); - if (max_output < 1) { - FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "max_output_tokens must be >= 1"); - } + const int max_output = ResolveMaxOutputTokens(options, default_max_output_tokens); // Validate token budget: input + output must not exceed model's max_length int total_required = input_token_count + max_output; @@ -106,14 +143,15 @@ int ApplySearchOptions(const SearchOptions& options, // Preserve a positive model setting. ORT GenAI reports both an absent setting and explicit zero as zero; Foundry // Local intentionally treats both as unset. ORT GenAI decides whether the model consumes the resulting option. - if (gen_params.GetSearchNumber("chunk_size") <= 0) { + if (config.GetChatBackendKind() != ChatBackendKind::kStaticEngine && + gen_params.GetSearchNumber("chunk_size") <= 0) { // The model's resolved EP is kDefault for the common load path, so use the provider declared in // genai_config.json. An empty provider means ORT's CPU fallback. ExecutionProvider effective_ep = ep; if (effective_ep == ExecutionProvider::kDefault) { std::string config_provider = config.DefaultProvider(); effective_ep = config_provider.empty() ? ExecutionProvider::kCPU - : EPUtils::StringtoEP(config_provider); + : EPUtils::StringtoEP(config_provider); } constexpr double kDefaultChunkSize = 2048.0; @@ -200,4 +238,11 @@ std::optional SearchOptions::ParseToolChoice(const KeyValuePairs& "Invalid value for tool_choice: '" + value + "'. Expected 'auto', 'none', or 'required'."); } +bool SearchOptions::HasSameRetainedGenerationSettings(const SearchOptions& other) const { + return temperature == other.temperature && top_p == other.top_p && top_k == other.top_k && + frequency_penalty == other.frequency_penalty && presence_penalty == other.presence_penalty && + seed == other.seed && do_sample == other.do_sample && early_stopping == other.early_stopping && + extra == other.extra; +} + } // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/search_options.h b/sdk_v2/cpp/src/inferencing/generative/chat/search_options.h index 7bc162c8a..231514aeb 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/search_options.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/search_options.h @@ -17,6 +17,8 @@ struct OgaGeneratorParams; namespace fl { +struct ToolCallContext; + /// Parameters extracted from a request that map to ORT GenAI search options. /// Decoupled from any specific request type so both C API and C++ API can use it. struct SearchOptions { @@ -47,8 +49,14 @@ struct SearchOptions { /// Returns std::nullopt when the key is absent. Throws fl::Exception when present /// with a value other than "auto", "none", or "required". static std::optional ParseToolChoice(const KeyValuePairs& params); + + /// Whether settings baked into retained generator/request state match another turn. + bool HasSameRetainedGenerationSettings(const SearchOptions& other) const; }; +/// Return the explicit or default output-token limit for a text generation turn. +int ResolveMaxOutputTokens(const SearchOptions& options, int default_max_output_tokens = 2048); + /// Apply search options to OgaGeneratorParams. /// Validates token budget (input + output vs model max_length from config). /// Returns the computed max_length that was set on the params. @@ -75,4 +83,7 @@ int ApplySearchOptions(const SearchOptions& options, bool use_full_context = false, int default_max_output_tokens = 2048); +/// Applies request-level grammar guidance to generator parameters when the tool context requires tool-only output. +void ApplyGuidanceOptions(const ToolCallContext& tool_ctx, OgaGeneratorParams& gen_params); + } // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_config.cc b/sdk_v2/cpp/src/inferencing/generative/genai_config.cc index 11d34d05a..379b06839 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_config.cc +++ b/sdk_v2/cpp/src/inferencing/generative/genai_config.cc @@ -4,9 +4,33 @@ #include "exception.h" #include +#include #include namespace fl { +namespace { + +size_t ParsePositiveSize(const nlohmann::json& object, const char* name, size_t default_value) { + if (!object.contains(name)) { + return default_value; + } + + const auto& value = object[name]; + if (!value.is_number_integer()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, + std::string("genai_config.json engine.") + name + " must be a positive integer"); + } + + const auto parsed = value.get(); + if (parsed <= 0 || static_cast(parsed) > std::numeric_limits::max()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, + std::string("genai_config.json engine.") + name + " must be a positive integer"); + } + + return static_cast(parsed); +} + +} // namespace bool GenAIConfig::OnnxModel::IsMultiModal() const { return type == "phi3v" || type == "whisper" || type == "phi4mm" || type == "fara" || @@ -33,6 +57,35 @@ std::string GenAIConfig::DefaultProvider() const { return first.begin()->first; } +ChatBackendKind GenAIConfig::GetChatBackendKind() const { + if (!engine) { + return ChatBackendKind::kGenerator; + } + + if (engine->dynamic_batching) { + return ChatBackendKind::kDynamicEngine; + } + + if (engine->static_batching) { + return ChatBackendKind::kStaticEngine; + } + + return ChatBackendKind::kGenerator; +} + +std::optional GenAIConfig::EngineMaxBatchSize() const { + switch (GetChatBackendKind()) { + case ChatBackendKind::kDynamicEngine: + return engine->dynamic_batching->max_batch_size; + case ChatBackendKind::kStaticEngine: + return engine->static_batching->max_batch_size; + case ChatBackendKind::kGenerator: + return std::nullopt; + } + + return std::nullopt; +} + GenAIConfig GenAIConfig::LoadFromFile(const std::string& path) { std::ifstream file(path); if (!file.is_open()) { @@ -113,6 +166,46 @@ GenAIConfig GenAIConfig::LoadFromFile(const std::string& path) { config.search = std::move(search); } + if (j.contains("engine") && j["engine"].is_object()) { + const auto& je = j["engine"]; + Engine engine; + + if (je.contains("dynamic_batching") && !je["dynamic_batching"].is_null()) { + if (!je["dynamic_batching"].is_object()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, + "genai_config.json engine.dynamic_batching must be an object"); + } + + const auto& batching = je["dynamic_batching"]; + Engine::DynamicBatching dynamic_batching; + dynamic_batching.max_batch_size = + ParsePositiveSize(batching, "max_batch_size", dynamic_batching.max_batch_size); + dynamic_batching.max_scheduled_tokens = + ParsePositiveSize(batching, "max_scheduled_tokens", dynamic_batching.max_scheduled_tokens); + engine.dynamic_batching = dynamic_batching; + } + + if (je.contains("static_batching") && !je["static_batching"].is_null()) { + if (!je["static_batching"].is_object()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, + "genai_config.json engine.static_batching must be an object"); + } + + const auto& batching = je["static_batching"]; + Engine::StaticBatching static_batching; + static_batching.max_batch_size = + ParsePositiveSize(batching, "max_batch_size", static_batching.max_batch_size); + engine.static_batching = static_batching; + } + + if (engine.dynamic_batching && engine.static_batching) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, + "genai_config.json cannot declare both engine.dynamic_batching and engine.static_batching"); + } + + config.engine = std::move(engine); + } + // hidden_size can appear at the top level or inside model if (j.contains("model") && j["model"].is_object()) { const auto& jm = j["model"]; diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_config.h b/sdk_v2/cpp/src/inferencing/generative/genai_config.h index b7e70dc3d..6a25d986a 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_config.h +++ b/sdk_v2/cpp/src/inferencing/generative/genai_config.h @@ -3,12 +3,19 @@ #pragma once #include +#include #include #include #include namespace fl { +enum class ChatBackendKind { + kGenerator, + kStaticEngine, + kDynamicEngine, +}; + /// Represents the parsed contents of a genai_config.json file. /// Maps the C# GenAIConfig / OnnxModel / OnnxDecoder types. struct GenAIConfig { @@ -36,14 +43,35 @@ struct GenAIConfig { int max_length = 0; }; + struct Engine { + struct DynamicBatching { + size_t max_batch_size = 16; + size_t max_scheduled_tokens = 2048; + }; + + struct StaticBatching { + size_t max_batch_size = 4; + }; + + std::optional dynamic_batching; + std::optional static_batching; + }; + std::optional model; std::optional search; + std::optional engine; std::optional hidden_size; // embedding dimension from genai_config.json /// Returns the first provider key from decoder.session_options.provider_options, /// or empty string if not found. std::string DefaultProvider() const; + /// Selects the chat inference backend declared by the model artifact. + ChatBackendKind GetChatBackendKind() const; + + /// Returns the configured Engine batch capacity, or nullopt for Generator models. + std::optional EngineMaxBatchSize() const; + /// Load and parse a genai_config.json file. Throws fl::Exception on failure. static GenAIConfig LoadFromFile(const std::string& path); }; diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc index 02d93f7af..262dadfa6 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc @@ -3,6 +3,7 @@ #include "inferencing/generative/genai_model_instance.h" #include "exception.h" #include "inferencing/execution_provider.h" +#include "inferencing/generative/chat/onnx_chat_engine.h" #include "util/key_value_pairs.h" #include "utils.h" @@ -69,11 +70,30 @@ GenAIModelInstance::GenAIModelInstance(std::string model_id, FL_LOG_AND_THROW(logger, FOUNDRY_LOCAL_ERROR_INTERNAL, "failed to create preprocessor for model ", model_id_, ": ", e.what()); } + + if (IsMultiModal() && genai_config_.GetChatBackendKind() != ChatBackendKind::kGenerator) { + FL_LOG_AND_THROW(logger, FOUNDRY_LOCAL_ERROR_INTERNAL, + "model ", model_id_, " declares an Engine backend, but Engine is not supported for multimodal models"); + } + + if (genai_config_.GetChatBackendKind() != ChatBackendKind::kGenerator) { +#ifdef FOUNDRY_LOCAL_HAS_OGA_ENGINE + try { + chat_engine_ = std::make_unique(*this); + } catch (const std::runtime_error& e) { + FL_LOG_AND_THROW(logger, FOUNDRY_LOCAL_ERROR_INTERNAL, + "failed to create chat engine for model ", model_id_, ": ", e.what()); + } +#else + FL_LOG_AND_THROW(logger, FOUNDRY_LOCAL_ERROR_INTERNAL, + "model ", model_id_, " requires the ORT GenAI Engine API, but this build does not provide it"); +#endif + } } // Destructor: unique_ptr members are destroyed in reverse declaration order. // OGA objects have custom operator delete that calls OgaDestroy* functions. -// Destruction order: preprocessor → oga_model (correct: dependents first). +// Destruction order: chat engine → preprocessor → OGA model (correct: dependents first). GenAIModelInstance::~GenAIModelInstance() = default; // --------------------------------------------------------------------------- diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h index 8a0f45c35..d8d9f8d3d 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h @@ -19,6 +19,8 @@ struct OgaModel; namespace fl { +class OnnxChatEngine; + /// A model that has been loaded into the ORT GenAI runtime. /// Owns the OgaModel and its preprocessing resources. /// Non-copyable, non-movable. Owned by ModelLoadManager via std::unique_ptr. @@ -53,6 +55,7 @@ class GenAIModelInstance { /// Access the underlying OGA objects. OgaModel& GetOgaModel(); Preprocessor& GetPreprocessor(); + OnnxChatEngine* GetChatEngine() { return chat_engine_.get(); } /// Get the last-activity timestamp. std::chrono::steady_clock::time_point LastActivity() const { return last_activity_; } @@ -81,6 +84,7 @@ class GenAIModelInstance { std::unique_ptr preprocessor_; TagInfo tag_info_; std::once_flag tag_info_init_flag_; + std::unique_ptr chat_engine_; std::chrono::steady_clock::time_point last_activity_; mutable std::atomic session_ref_count_{0}; }; diff --git a/sdk_v2/cpp/test/internal_api/chat/chat_session_test.cc b/sdk_v2/cpp/test/internal_api/chat/chat_session_test.cc index 281f8198a..7448c9e0e 100644 --- a/sdk_v2/cpp/test/internal_api/chat/chat_session_test.cc +++ b/sdk_v2/cpp/test/internal_api/chat/chat_session_test.cc @@ -21,6 +21,7 @@ #include +#include #include #include #include @@ -144,6 +145,28 @@ TEST_F(ChatSessionTest, RunBasic) { EXPECT_EQ(session.GetHistory()[1].GetSimpleText(), text); } +TEST_F(ChatSessionTest, ConcurrentIndependentSessions) { + auto run_request = [this](std::string prompt) { + ChatSession session(GetCatalogModel(), GetModel(), *logger_, null_telemetry_); + Request request; + request.AddOwnedItem(MakeMessage(FOUNDRY_LOCAL_ROLE_USER, prompt)); + request.options.Add("max_output_tokens", "32"); + request.options.Add("temperature", "0"); + + Response response; + session.ProcessRequest(request, response); + return GetAssistantText(response); + }; + + auto first = std::async(std::launch::async, run_request, "What is 2+2? Answer with just the number."); + auto second = std::async(std::launch::async, run_request, "What is 3+3? Answer with just the number."); + + const auto first_text = first.get(); + const auto second_text = second.get(); + EXPECT_NE(first_text.find("4"), std::string::npos) << first_text; + EXPECT_NE(second_text.find("6"), std::string::npos) << second_text; +} + TEST_F(ChatSessionTest, ChatCompletionRejectsAudioInput) { ChatSession session(GetCatalogModel(), GetModel(), *logger_, null_telemetry_); diff --git a/sdk_v2/cpp/test/internal_api/chat/chat_template_test.cc b/sdk_v2/cpp/test/internal_api/chat/chat_template_test.cc index a983da5e4..129819799 100644 --- a/sdk_v2/cpp/test/internal_api/chat/chat_template_test.cc +++ b/sdk_v2/cpp/test/internal_api/chat/chat_template_test.cc @@ -131,6 +131,17 @@ TEST_F(ChatTemplateTest, PromptEndsWithAssistantPrefix) { << "Prompt should end with assistant prefix for generation. Got: " << prompt; } +TEST_F(ChatTemplateTest, EngineContinuationIncludesAssistantTurnBoundary) { + std::vector messages = {{FOUNDRY_LOCAL_ROLE_USER, "What is the codeword?"}}; + + std::string prompt = BuildChatContinuationPrompt(messages, GetModel()); + + EXPECT_EQ(prompt.find("__foundry_engine_assistant_boundary__"), std::string::npos); + EXPECT_NE(prompt.find("<|im_end|>"), std::string::npos) << prompt; + EXPECT_NE(prompt.find("What is the codeword?"), std::string::npos) << prompt; + EXPECT_NE(prompt.find("assistant"), std::string::npos) << prompt; +} + // --------------------------------------------------------------------------- // EncodePrompt tests // --------------------------------------------------------------------------- diff --git a/sdk_v2/cpp/test/internal_api/chat/search_options_test.cc b/sdk_v2/cpp/test/internal_api/chat/search_options_test.cc index eb6ee5a32..3ae70c7b4 100644 --- a/sdk_v2/cpp/test/internal_api/chat/search_options_test.cc +++ b/sdk_v2/cpp/test/internal_api/chat/search_options_test.cc @@ -30,6 +30,30 @@ TEST(SearchOptionsParsingTest, TemperatureOutsideSupportedRangeThrows) { } } +TEST(SearchOptionsParsingTest, ResolvesDefaultAndExplicitOutputLimits) { + SearchOptions defaults; + EXPECT_EQ(ResolveMaxOutputTokens(defaults), 2048); + + SearchOptions explicit_limit; + explicit_limit.max_output_tokens = 64; + EXPECT_EQ(ResolveMaxOutputTokens(explicit_limit), 64); +} + +TEST(SearchOptionsParsingTest, RetainedGenerationSettingsIgnorePerTurnOptions) { + SearchOptions first; + first.temperature = 0.5f; + first.max_output_tokens = 16; + first.tool_choice = FOUNDRY_LOCAL_TOOL_CHOICE_AUTO; + + SearchOptions second = first; + second.max_output_tokens = 64; + second.tool_choice = FOUNDRY_LOCAL_TOOL_CHOICE_REQUIRED; + EXPECT_TRUE(first.HasSameRetainedGenerationSettings(second)); + + second.temperature = 1.0f; + EXPECT_FALSE(first.HasSameRetainedGenerationSettings(second)); +} + // --------------------------------------------------------------------------- // Test fixture: loads the shared test model once per suite // --------------------------------------------------------------------------- diff --git a/sdk_v2/cpp/test/internal_api/genai_config_test.cc b/sdk_v2/cpp/test/internal_api/genai_config_test.cc index fd420920e..74249a3a9 100644 --- a/sdk_v2/cpp/test/internal_api/genai_config_test.cc +++ b/sdk_v2/cpp/test/internal_api/genai_config_test.cc @@ -199,6 +199,89 @@ TEST_F(GenAIConfigTest, LoadMissingOptionalFields) { EXPECT_FALSE(config.model->decoder.has_value()); } +TEST_F(GenAIConfigTest, SelectsGeneratorWhenEngineBatchingIsAbsent) { + auto path = WriteFile("genai_config.json", R"({"engine": {}})"); + + auto config = GenAIConfig::LoadFromFile(path); + + EXPECT_EQ(config.GetChatBackendKind(), ChatBackendKind::kGenerator); + EXPECT_FALSE(config.EngineMaxBatchSize().has_value()); +} + +TEST_F(GenAIConfigTest, ParsesDynamicEngineConfiguration) { + auto path = WriteFile("genai_config.json", R"({ + "engine": { + "dynamic_batching": { + "max_batch_size": 8, + "max_scheduled_tokens": 1024 + } + } + })"); + + auto config = GenAIConfig::LoadFromFile(path); + + ASSERT_TRUE(config.engine.has_value()); + ASSERT_TRUE(config.engine->dynamic_batching.has_value()); + EXPECT_EQ(config.engine->dynamic_batching->max_batch_size, 8u); + EXPECT_EQ(config.engine->dynamic_batching->max_scheduled_tokens, 1024u); + EXPECT_EQ(config.GetChatBackendKind(), ChatBackendKind::kDynamicEngine); + EXPECT_EQ(config.EngineMaxBatchSize(), 8u); +} + +TEST_F(GenAIConfigTest, ParsesStaticEngineConfiguration) { + auto path = WriteFile("genai_config.json", R"({ + "engine": { + "static_batching": { + "max_batch_size": 2 + } + } + })"); + + auto config = GenAIConfig::LoadFromFile(path); + + ASSERT_TRUE(config.engine.has_value()); + ASSERT_TRUE(config.engine->static_batching.has_value()); + EXPECT_EQ(config.engine->static_batching->max_batch_size, 2u); + EXPECT_EQ(config.GetChatBackendKind(), ChatBackendKind::kStaticEngine); + EXPECT_EQ(config.EngineMaxBatchSize(), 2u); +} + +TEST_F(GenAIConfigTest, AppliesEngineDefaults) { + auto dynamic_path = WriteFile("dynamic.json", R"({"engine": {"dynamic_batching": {}}})"); + auto static_path = WriteFile("static.json", R"({"engine": {"static_batching": {}}})"); + + auto dynamic_config = GenAIConfig::LoadFromFile(dynamic_path); + auto static_config = GenAIConfig::LoadFromFile(static_path); + + EXPECT_EQ(dynamic_config.engine->dynamic_batching->max_batch_size, 16u); + EXPECT_EQ(dynamic_config.engine->dynamic_batching->max_scheduled_tokens, 2048u); + EXPECT_EQ(static_config.engine->static_batching->max_batch_size, 4u); +} + +TEST_F(GenAIConfigTest, RejectsBothEngineBatchingModes) { + auto path = WriteFile("genai_config.json", R"({ + "engine": { + "dynamic_batching": {}, + "static_batching": {} + } + })"); + + EXPECT_THROW(GenAIConfig::LoadFromFile(path), fl::Exception); +} + +TEST_F(GenAIConfigTest, RejectsInvalidEngineCapacity) { + auto zero_path = + WriteFile("zero.json", R"({"engine": {"dynamic_batching": {"max_batch_size": 0}}})"); + auto negative_path = + WriteFile("negative.json", R"({"engine": {"static_batching": {"max_batch_size": -1}}})"); + auto wrong_type_path = + WriteFile("wrong_type.json", R"({"engine": {"dynamic_batching": {"max_scheduled_tokens": "bad"}}})"); + + EXPECT_THROW(GenAIConfig::LoadFromFile(zero_path), fl::Exception); + EXPECT_THROW(GenAIConfig::LoadFromFile(negative_path), fl::Exception); + EXPECT_THROW(GenAIConfig::LoadFromFile(wrong_type_path), fl::Exception); +} + TEST_F(GenAIConfigTest, LoadThrowsForMissingFile) { EXPECT_THROW(GenAIConfig::LoadFromFile("/nonexistent/path/genai_config.json"), fl::Exception); diff --git a/sdk_v2/cpp/test/test_main.cc b/sdk_v2/cpp/test/test_main.cc index 2878ee757..129b4b956 100644 --- a/sdk_v2/cpp/test/test_main.cc +++ b/sdk_v2/cpp/test/test_main.cc @@ -3,6 +3,11 @@ #include +#if __has_include() +#include +#define FOUNDRY_LOCAL_TEST_HAS_OGA 1 +#endif + #include int main(int argc, char** argv) { @@ -13,5 +18,9 @@ int main(int argc, char** argv) { #endif ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); + const int result = RUN_ALL_TESTS(); +#ifdef FOUNDRY_LOCAL_TEST_HAS_OGA + OgaShutdown(); +#endif + return result; } diff --git a/sdk_v2/deps_versions.json b/sdk_v2/deps_versions.json index db735f041..0d733c880 100644 --- a/sdk_v2/deps_versions.json +++ b/sdk_v2/deps_versions.json @@ -1,6 +1,6 @@ { "_comment": "Single source of truth for native dependency versions in sdk_v2. Read by sdk_v2/cpp/cmake/Find*.cmake and sdk_v2/python/_build_backend/__init__.py. The .pipelines/foundry-local-packaging.yml literals must match; the 'Validate pinned versions' step fails the build on drift.", "onnxruntime": { "version": "1.28.0" }, - "onnxruntime-genai": { "version": "0.15.2" }, + "onnxruntime-genai": { "version": "0.15.3" }, "windows-ai-machinelearning": { "version": "2.1.70" } } From b43f3a38c8989bcc15ca557b7503d9ff2f40b4e8 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Thu, 3 Sep 2026 18:16:44 -0500 Subject: [PATCH 03/11] Require the OGA Engine API Remove compatibility branches for OGA releases without Engine support so missing APIs fail during compilation instead of at model load. Files changed: - sdk_v2/cpp/CMakeLists.txt - sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc - sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9b90e66e-af55-404d-b606-7899f6f73c54 --- sdk_v2/cpp/CMakeLists.txt | 21 ++----------------- .../generative/chat/chat_session.cc | 7 ------- .../generative/genai_model_instance.cc | 5 ----- 3 files changed, 2 insertions(+), 31 deletions(-) diff --git a/sdk_v2/cpp/CMakeLists.txt b/sdk_v2/cpp/CMakeLists.txt index 5487ebccd..f8b62ce62 100644 --- a/sdk_v2/cpp/CMakeLists.txt +++ b/sdk_v2/cpp/CMakeLists.txt @@ -109,15 +109,6 @@ endif() # ORT and ORT GenAI — acquired via FetchContent from nuget.org. list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") find_package(OnnxRuntimeGenAI REQUIRED) - -file(STRINGS "${ORT_GENAI_HEADER_DIR}/ort_genai_c.h" _OGA_ENGINE_DECLARATION REGEX "OgaCreateEngine") -if(_OGA_ENGINE_DECLARATION) - set(FOUNDRY_LOCAL_HAS_OGA_ENGINE ON) - message(STATUS "ORT GenAI Engine API: enabled") -else() - set(FOUNDRY_LOCAL_HAS_OGA_ENGINE OFF) - message(STATUS "ORT GenAI Engine API: unavailable") -endif() find_package(OnnxRuntime REQUIRED) # WinML EP Catalog — Windows-only, for hardware EP discovery and download. The @@ -222,6 +213,8 @@ set(FOUNDRY_LOCAL_SOURCES src/inferencing/generative/audio/pcm_utils.cc src/inferencing/generative/embeddings/embeddings_session.cc src/inferencing/generative/chat/chat_generator.cc + src/inferencing/generative/chat/onnx_chat_engine.cc + src/inferencing/generative/chat/onnx_engine_chat_generator.cc src/inferencing/session/session.cc src/inferencing/session/session_manager.cc src/inferencing/generative/chat/chat_session.cc @@ -286,13 +279,6 @@ set(FOUNDRY_LOCAL_SOURCES ${FOUNDRY_LOCAL_INTERNAL_HEADERS} ) -if(FOUNDRY_LOCAL_HAS_OGA_ENGINE) - list(APPEND FOUNDRY_LOCAL_SOURCES - src/inferencing/generative/chat/onnx_chat_engine.cc - src/inferencing/generative/chat/onnx_engine_chat_generator.cc - ) -endif() - # 1DS bridge — always compiled for Foundry Local Core. list(APPEND FOUNDRY_LOCAL_SOURCES src/telemetry/one_ds_telemetry.cc) @@ -417,9 +403,6 @@ configure_file( # static library targets re-use these object files, avoiding a double build. # -------------------------------------------------------------------------- add_library(foundry_local_objects OBJECT ${FOUNDRY_LOCAL_SOURCES}) -if(FOUNDRY_LOCAL_HAS_OGA_ENGINE) - target_compile_definitions(foundry_local_objects PRIVATE FOUNDRY_LOCAL_HAS_OGA_ENGINE=1) -endif() set_target_properties(foundry_local_objects PROPERTIES POSITION_INDEPENDENT_CODE ON) foundry_local_configure_target(foundry_local_objects PUBLIC) # FL_STATIC_LIBRARY makes FL_EXPORT empty on Windows so the two exported entry diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc index 9f93bec3d..17e7bbaf2 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc @@ -5,9 +5,7 @@ #include "contracts/chat_completions.h" #include "contracts/chat_completions_converter.h" -#ifdef FOUNDRY_LOCAL_HAS_OGA_ENGINE #include "inferencing/generative/chat/onnx_engine_chat_generator.h" -#endif #include "inferencing/generative/chat/onnx_chat_generator.h" #include "inferencing/generative/chat/reasoning_stream_splitter.h" #include "inferencing/generative/genai_model_instance.h" @@ -56,12 +54,7 @@ std::unique_ptr CreateTextChatGenerator(const std::vector(*this); } catch (const std::runtime_error& e) { FL_LOG_AND_THROW(logger, FOUNDRY_LOCAL_ERROR_INTERNAL, "failed to create chat engine for model ", model_id_, ": ", e.what()); } -#else - FL_LOG_AND_THROW(logger, FOUNDRY_LOCAL_ERROR_INTERNAL, - "model ", model_id_, " requires the ORT GenAI Engine API, but this build does not provide it"); -#endif } } From c2e424de9c198b21fa80983c213025fc82b780a0 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Thu, 3 Sep 2026 22:09:01 -0500 Subject: [PATCH 04/11] Keep the released OGA dependency Retain OGA 0.15.2 until the next Engine-capable stable package is published, while non-release CI continues using the selected nightly. Remove optional OGA hooks from the shared test entry point because not every test target consumes OGA. Files changed: - .pipelines/foundry-local-packaging.yml - .pipelines/v2/sdk_v2-pipeline-plan.md - sdk_v2/deps_versions.json - sdk_v2/cpp/test/test_main.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9b90e66e-af55-404d-b606-7899f6f73c54 --- .pipelines/foundry-local-packaging.yml | 2 +- .pipelines/v2/sdk_v2-pipeline-plan.md | 2 +- sdk_v2/cpp/test/test_main.cc | 13 ++----------- sdk_v2/deps_versions.json | 2 +- 4 files changed, 5 insertions(+), 14 deletions(-) diff --git a/.pipelines/foundry-local-packaging.yml b/.pipelines/foundry-local-packaging.yml index 684e7c1bc..2ee3d2e0b 100644 --- a/.pipelines/foundry-local-packaging.yml +++ b/.pipelines/foundry-local-packaging.yml @@ -60,7 +60,7 @@ variables: value: '1.28.0' - ${{ if eq(parameters.isRelease, true) }}: - name: cppGenaiVersion - value: '0.15.3' + value: '0.15.2' - ${{ else }}: - name: cppGenaiVersion value: '0.16.0-dev1001400138' diff --git a/.pipelines/v2/sdk_v2-pipeline-plan.md b/.pipelines/v2/sdk_v2-pipeline-plan.md index 4d646d7f7..44fc170f6 100644 --- a/.pipelines/v2/sdk_v2-pipeline-plan.md +++ b/.pipelines/v2/sdk_v2-pipeline-plan.md @@ -285,7 +285,7 @@ purposes: Versions are pipeline-level variables, currently: * `ortVersion` `1.28.0` (`Microsoft.ML.OnnxRuntime`) -* `genaiVersion` `0.15.3` for releases; selected ORT-Nightly version for non-release CI +* `genaiVersion` `0.15.2` for releases; selected ORT-Nightly version for non-release CI (`Microsoft.ML.OnnxRuntimeGenAI.Foundry`) * `winmlVersion` `2.1.70` (`Microsoft.Windows.AI.MachineLearning`, WinML 2.x reg-free) diff --git a/sdk_v2/cpp/test/test_main.cc b/sdk_v2/cpp/test/test_main.cc index 129b4b956..ef5ab8e58 100644 --- a/sdk_v2/cpp/test/test_main.cc +++ b/sdk_v2/cpp/test/test_main.cc @@ -1,12 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. - #include - -#if __has_include() -#include -#define FOUNDRY_LOCAL_TEST_HAS_OGA 1 -#endif +#include #include @@ -18,9 +13,5 @@ int main(int argc, char** argv) { #endif ::testing::InitGoogleTest(&argc, argv); - const int result = RUN_ALL_TESTS(); -#ifdef FOUNDRY_LOCAL_TEST_HAS_OGA - OgaShutdown(); -#endif - return result; + return RUN_ALL_TESTS(); } diff --git a/sdk_v2/deps_versions.json b/sdk_v2/deps_versions.json index 0d733c880..db735f041 100644 --- a/sdk_v2/deps_versions.json +++ b/sdk_v2/deps_versions.json @@ -1,6 +1,6 @@ { "_comment": "Single source of truth for native dependency versions in sdk_v2. Read by sdk_v2/cpp/cmake/Find*.cmake and sdk_v2/python/_build_backend/__init__.py. The .pipelines/foundry-local-packaging.yml literals must match; the 'Validate pinned versions' step fails the build on drift.", "onnxruntime": { "version": "1.28.0" }, - "onnxruntime-genai": { "version": "0.15.3" }, + "onnxruntime-genai": { "version": "0.15.2" }, "windows-ai-machinelearning": { "version": "2.1.70" } } From 2992e3b444fd42d415b07a96877883ef7c2f928f Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Thu, 3 Sep 2026 22:10:02 -0500 Subject: [PATCH 05/11] Keep conditional OGA test shutdown The shared test entry point is compiled by targets with and without OGA include paths. Retain header detection so OGA-linked tests shut down cleanly without imposing that dependency on cache-only tests. Files changed: - sdk_v2/cpp/test/test_main.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9b90e66e-af55-404d-b606-7899f6f73c54 --- sdk_v2/cpp/test/test_main.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sdk_v2/cpp/test/test_main.cc b/sdk_v2/cpp/test/test_main.cc index ef5ab8e58..9d70df882 100644 --- a/sdk_v2/cpp/test/test_main.cc +++ b/sdk_v2/cpp/test/test_main.cc @@ -2,6 +2,11 @@ // Licensed under the MIT License. #include #include +#include +#if __has_include() +#include +#define FOUNDRY_LOCAL_TEST_HAS_OGA 1 +#endif #include @@ -13,5 +18,9 @@ int main(int argc, char** argv) { #endif ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); + const int result = RUN_ALL_TESTS(); +#ifdef FOUNDRY_LOCAL_TEST_HAS_OGA + OgaShutdown(); +#endif + return result; } From 5d5299b41a5700447a682701c8d751964d404265 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Fri, 4 Sep 2026 23:52:25 -0500 Subject: [PATCH 06/11] Fix Engine chat lifecycle and limit handling Destroy retained generators before releasing model ownership, preserve undo correctness after generator rebuilds, and keep the Generator path's implicit limit behavior unchanged. Use bounded Generator context for one-shot OpenAI JSON requests and map the CI nightly's native StopSequence finish reason. Stage the shared chat model with dynamic Engine batching so concurrency and cancellation coverage exercise the dispatcher. Files changed: chat_session.cc, chat_session.h, onnx_engine_chat_generator.cc, chat_session_test.cc. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../generative/chat/chat_session.cc | 27 ++++--- .../generative/chat/chat_session.h | 5 +- .../chat/onnx_engine_chat_generator.cc | 21 ++++++ .../internal_api/chat/chat_session_test.cc | 70 ++++++++++++++++++- 4 files changed, 111 insertions(+), 12 deletions(-) diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc index 17e7bbaf2..f8c2b025b 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc @@ -52,12 +52,13 @@ void ApplyToolChoiceToContext(std::optional tool_choice, ToolCallC std::unique_ptr CreateTextChatGenerator(const std::vector& messages, const SearchOptions& options, GenAIModelInstance& model, - const ToolCallContext& tool_ctx) { + const ToolCallContext& tool_ctx, + bool use_full_context) { if (model.GetGenAIConfig().GetChatBackendKind() != ChatBackendKind::kGenerator) { return OnnxEngineChatGenerator::Create(messages, options, model, tool_ctx); } - return OnnxChatGenerator::Create(messages, options, model, tool_ctx, /*use_full_context=*/true); + return OnnxChatGenerator::Create(messages, options, model, tool_ctx, use_full_context); } } // namespace @@ -71,6 +72,7 @@ ChatSession::ChatSession(const fl::Model& catalog_model, GenAIModelInstance& mod ChatSession::~ChatSession() { if (owns_session_) { + cached_generator_.reset(); model_.ReleaseSession(); } } @@ -478,6 +480,7 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) int prompt_tokens = 0; int pre_turn_token_count = 0; + bool can_rewind_to_pre_turn = true; if (cached_generator_ && !cached_search_options_.HasSameRetainedGenerationSettings(effective_options)) { @@ -517,6 +520,7 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) if (!cached_generator_) { // First request (or cache invalidated): create the generator from scratch. // Combine existing history with new messages for the full context. + can_rewind_to_pre_turn = history_.empty(); auto tool_ctx = BuildToolCallContext(request); std::vector all_messages; @@ -535,7 +539,8 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) generator = OnnxChatGenerator::CreateWithMedia(all_messages, effective_options, Model(), images, audios, tool_ctx, /*use_full_context*/ false); } else { - generator = CreateTextChatGenerator(all_messages, effective_options, Model(), tool_ctx); + generator = CreateTextChatGenerator(all_messages, effective_options, Model(), tool_ctx, + /*use_full_context=*/true); } prompt_tokens = generator->PromptTokenCount(); @@ -544,7 +549,7 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) cached_search_options_ = effective_options; } - const int max_output = ResolveMaxOutputTokens(effective_options); + const int max_output = effective_options.max_output_tokens.value_or(0); // Generate token-by-token with optional streaming. // Check request.canceled each iteration — a streaming callback returning @@ -689,7 +694,8 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) cached_tool_ctx_ = {}; } - CommitTurn(std::move(new_messages), response, pre_turn_token_count, total_tokens); + CommitTurn(std::move(new_messages), response, pre_turn_token_count, total_tokens, + can_rewind_to_pre_turn); // After a media turn, drop the cached generator so any text follow-up // rebuilds from history. AppendMessages cannot extend a media-decoded @@ -765,7 +771,8 @@ void ChatSession::ProcessChatCompletionsJson(const std::string& request_json, co } // Create generator - auto generator = CreateTextChatGenerator(messages, options, Model(), tool_ctx); + auto generator = CreateTextChatGenerator(messages, options, Model(), tool_ctx, + /*use_full_context=*/false); int prompt_tokens = generator->PromptTokenCount(); auto streaming_callback = CreateCallbackHandler(original_request); @@ -921,7 +928,8 @@ const std::vector& ChatSession::GetHistory() const { } void ChatSession::CommitTurn(std::vector&& new_messages, const Response& response, - int pre_turn_token_count, int post_turn_token_count) { + int pre_turn_token_count, int post_turn_token_count, + bool can_rewind_to_pre_turn) { size_t history_start = history_.size(); size_t input_count = new_messages.size(); @@ -955,7 +963,8 @@ void ChatSession::CommitTurn(std::vector&& new_messages, const Resp history_.push_back(std::move(*assistant_reply)); } - turns_.push_back({history_start, input_count, pre_turn_token_count, post_turn_token_count}); + turns_.push_back( + {history_start, input_count, pre_turn_token_count, post_turn_token_count, can_rewind_to_pre_turn}); } size_t ChatSession::TurnCount() const { @@ -985,7 +994,7 @@ void ChatSession::UndoTurns(size_t count) { // Undoing all turns — destroy the generator entirely cached_generator_.reset(); cached_tool_ctx_ = {}; - } else if (cached_generator_->CanRewind()) { + } else if (cached_generator_->CanRewind() && target.can_rewind_to_pre_turn) { cached_generator_->RewindTo(target.pre_turn_token_count); } else { cached_generator_.reset(); diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h index d7811930b..302a09afd 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h @@ -40,6 +40,7 @@ class ChatSession : public Session { size_t input_count; // number of input messages (user + tool results) in this turn int pre_turn_token_count; // generator sequence length before this turn's input was appended int post_turn_token_count; // generator sequence length after generation completed + bool can_rewind_to_pre_turn; // The assistant reply is at history_[history_start + input_count] }; @@ -103,8 +104,8 @@ class ChatSession : public Session { Response& response); /// Commit input messages and assistant reply to history after a successful turn. - void CommitTurn(std::vector&& new_messages, std::string assistant_history, - int pre_turn_token_count, int post_turn_token_count); + void CommitTurn(std::vector&& new_messages, const Response& response, + int pre_turn_token_count, int post_turn_token_count, bool can_rewind_to_pre_turn); GenAIModelInstance& Model() { return model_; } const GenAIModelInstance& Model() const { return model_; } diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.cc index 1837d8b31..1fb1a95a3 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.cc @@ -12,6 +12,27 @@ namespace fl { +namespace { + +std::optional MapFinishReason(OgaFinishReason reason) { + switch (reason) { + case OgaFinishReason_Eos: + case OgaFinishReason_StopSequence: + return FOUNDRY_LOCAL_FINISH_STOP; + case OgaFinishReason_MaxGeneratedTokens: + case OgaFinishReason_MaxSessionTokens: + return FOUNDRY_LOCAL_FINISH_LENGTH; + case OgaFinishReason_Cancelled: + return FOUNDRY_LOCAL_FINISH_NONE; + case OgaFinishReason_Failed: + return FOUNDRY_LOCAL_FINISH_ERROR; + default: + return std::nullopt; + } +} + +} // namespace + OnnxEngineChatGenerator::OnnxEngineChatGenerator( OnnxChatEngine& engine, std::shared_ptr conversation, diff --git a/sdk_v2/cpp/test/internal_api/chat/chat_session_test.cc b/sdk_v2/cpp/test/internal_api/chat/chat_session_test.cc index 7448c9e0e..2536a457f 100644 --- a/sdk_v2/cpp/test/internal_api/chat/chat_session_test.cc +++ b/sdk_v2/cpp/test/internal_api/chat/chat_session_test.cc @@ -18,9 +18,13 @@ #include "internal_api/test_helpers.h" #include "internal_api/test_model_cache.h" #include "utils/string_utils.h" +#include "utils/temp_path.h" #include +#include +#include +#include #include #include #include @@ -28,6 +32,59 @@ using namespace fl; +namespace { + +class EngineModelStaging { + public: + explicit EngineModelStaging(const std::filesystem::path& source) + : path_(source.parent_path() / + ("engine-chat-test-" + std::to_string(fl::test::CurrentPid()))) { + std::error_code ec; + std::filesystem::remove_all(path_, ec); + std::filesystem::create_directories(path_); + + try { + for (const auto& entry : std::filesystem::recursive_directory_iterator(source)) { + const auto relative = std::filesystem::relative(entry.path(), source); + const auto destination = path_ / relative; + if (entry.is_directory()) { + std::filesystem::create_directories(destination); + } else if (entry.path().filename() == "genai_config.json") { + std::filesystem::copy_file(entry.path(), destination); + } else { + std::filesystem::create_hard_link(entry.path(), destination); + } + } + + const auto config_path = path_ / "genai_config.json"; + std::ifstream input(config_path); + auto config = nlohmann::json::parse(input); + config["engine"] = { + {"dynamic_batching", {{"max_batch_size", 2}, {"max_scheduled_tokens", 2048}}}, + }; + std::ofstream(config_path) << config.dump(2); + } catch (...) { + std::filesystem::remove_all(path_, ec); + throw; + } + } + + ~EngineModelStaging() { + std::error_code ec; + std::filesystem::remove_all(path_, ec); + } + + EngineModelStaging(const EngineModelStaging&) = delete; + EngineModelStaging& operator=(const EngineModelStaging&) = delete; + + const std::filesystem::path& path() const { return path_; } + + private: + std::filesystem::path path_; +}; + +} // namespace + // =========================================================================== // Integration test fixture: loads the shared test model once per suite // =========================================================================== @@ -36,12 +93,13 @@ class ChatSessionTest : public ::testing::Test { protected: static void SetUpTestSuite() { auto model_path = fl::test::GetTestModelPath(fl::test::kTestChatModelAlias); + engine_model_ = std::make_unique(model_path); logger_ = std::make_unique(); ep_detector_ = std::make_unique(); load_manager_ = std::make_unique(*ep_detector_, *logger_); auto result = load_manager_->LoadModel( - model_path.string(), + engine_model_->path().string(), fl::test::kTestChatModelAlias); ASSERT_EQ(result.status, ModelLoadManager::LoadStatus::kSuccess) @@ -58,12 +116,14 @@ class ChatSessionTest : public ::testing::Test { load_manager_.reset(); ep_detector_.reset(); model_ = nullptr; + engine_model_.reset(); } GenAIModelInstance& GetModel() { return *model_; } const Model& GetCatalogModel() { return catalog_model_; } static inline std::unique_ptr logger_; + static inline std::unique_ptr engine_model_; static inline std::unique_ptr ep_detector_; static inline std::unique_ptr load_manager_; static inline GenAIModelInstance* model_ = nullptr; @@ -146,6 +206,10 @@ TEST_F(ChatSessionTest, RunBasic) { } TEST_F(ChatSessionTest, ConcurrentIndependentSessions) { + ASSERT_NE(GetModel().GetGenAIConfig().GetChatBackendKind(), ChatBackendKind::kGenerator) + << "The concurrency test model must declare an Engine backend"; + ASSERT_NE(GetModel().GetChatEngine(), nullptr); + auto run_request = [this](std::string prompt) { ChatSession session(GetCatalogModel(), GetModel(), *logger_, null_telemetry_); Request request; @@ -340,6 +404,10 @@ TEST_F(ChatSessionTest, RunMultiTurn) { } TEST_F(ChatSessionTest, RunStreamingCancellation) { + ASSERT_NE(GetModel().GetGenAIConfig().GetChatBackendKind(), ChatBackendKind::kGenerator) + << "The cancellation test model must declare an Engine backend"; + ASSERT_NE(GetModel().GetChatEngine(), nullptr); + ChatSession session(GetCatalogModel(), GetModel(), *logger_, null_telemetry_); Request request; From 17e86088a1926c2047f4ebf90a347e93199d9fc8 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sat, 5 Sep 2026 00:04:03 -0500 Subject: [PATCH 07/11] Harden Engine request setup and guidance reuse Invalidate retained requests when explicit guidance changes, and close newly-created Engine conversations if turn setup or adapter construction fails. Accept both StopSequence and StopString finish-reason symbols while compiling against the exact non-release CI nightly. Files changed: chat_session.cc, onnx_engine_chat_generator.cc. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../generative/chat/chat_session.cc | 5 +++- .../chat/onnx_engine_chat_generator.cc | 28 +++++++++++++------ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc index f8c2b025b..8557ae7ca 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc @@ -496,10 +496,13 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) bool prev_needs_guidance = cached_tool_ctx_.tool_output && cached_tool_ctx_.HasTools(); bool curr_needs_guidance = turn_tool_ctx.tool_output && turn_tool_ctx.HasTools(); + const bool guidance_changed = + cached_tool_ctx_.guidance_type != turn_tool_ctx.guidance_type || + cached_tool_ctx_.guidance_data != turn_tool_ctx.guidance_data; const bool static_engine = Model().GetGenAIConfig().GetChatBackendKind() == ChatBackendKind::kStaticEngine; - if (prev_needs_guidance != curr_needs_guidance || static_engine) { + if (prev_needs_guidance != curr_needs_guidance || guidance_changed || static_engine) { // Guidance requirements changed — invalidate. The branch below will rebuild from full history. cached_generator_.reset(); cached_tool_ctx_ = {}; diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.cc index 1fb1a95a3..e421d2988 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.cc @@ -17,7 +17,11 @@ namespace { std::optional MapFinishReason(OgaFinishReason reason) { switch (reason) { case OgaFinishReason_Eos: +#if defined(OgaFinishReason_StopSequence) case OgaFinishReason_StopSequence: +#elif defined(OgaFinishReason_StopString) + case OgaFinishReason_StopString: +#endif return FOUNDRY_LOCAL_FINISH_STOP; case OgaFinishReason_MaxGeneratedTokens: case OgaFinishReason_MaxSessionTokens: @@ -160,16 +164,24 @@ std::unique_ptr OnnxEngineChatGenerator::Create( auto prompt = BuildChatPrompt(messages, model, tool_ctx.tools_json); auto sequences = EncodePrompt(prompt, model); const int prompt_token_count = static_cast(sequences->SequenceCount(0)); + auto stream = model.GetPreprocessor().CreateTokenizerStream(); + auto stream_with_special = model.GetPreprocessor().CreateSpecialTokenizerStream(); auto conversation = engine->CreateConversation(options, tool_ctx, prompt_token_count); - const auto* data = sequences->SequenceData(0); - engine->BeginTurn(conversation, std::span(data, static_cast(prompt_token_count)), - ResolveMaxOutputTokens(options)); + try { + const auto* data = sequences->SequenceData(0); + engine->BeginTurn(conversation, std::span(data, static_cast(prompt_token_count)), + ResolveMaxOutputTokens(options)); - return std::unique_ptr( - new OnnxEngineChatGenerator(*engine, std::move(conversation), - model.GetPreprocessor().CreateTokenizerStream(), - model.GetPreprocessor().CreateSpecialTokenizerStream(), model, - prompt_token_count)); + return std::unique_ptr( + new OnnxEngineChatGenerator(*engine, std::move(conversation), std::move(stream), + std::move(stream_with_special), model, prompt_token_count)); + } catch (...) { + try { + engine->Close(conversation); + } catch (...) { + } + throw; + } } } // namespace fl From b0cb5f6e17c5258038047425b6e74e1abb560cea Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sat, 5 Sep 2026 00:14:06 -0500 Subject: [PATCH 08/11] Align Engine decoding and static coverage Decode model-configured BOT/EOT/BOR/EOR token IDs exactly like the Generator backend instead of matching English token spellings. Stage a static-batching variant of the shared model and verify that a rebuilt second turn retains committed conversation history. Files changed: onnx_engine_chat_generator.cc/.h, chat_session_test.cc. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../chat/onnx_engine_chat_generator.cc | 40 ++++++------ .../chat/onnx_engine_chat_generator.h | 2 - .../internal_api/chat/chat_session_test.cc | 63 ++++++++++++++++--- 3 files changed, 77 insertions(+), 28 deletions(-) diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.cc index e421d2988..fa06a8933 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.cc @@ -41,13 +41,11 @@ OnnxEngineChatGenerator::OnnxEngineChatGenerator( OnnxChatEngine& engine, std::shared_ptr conversation, std::unique_ptr stream, - std::unique_ptr stream_with_special, GenAIModelInstance& model, int prompt_token_count) : engine_(engine), conversation_(std::move(conversation)), stream_(std::move(stream)), - stream_with_special_(std::move(stream_with_special)), model_(model), prompt_token_count_(prompt_token_count) {} @@ -83,22 +81,27 @@ std::string OnnxEngineChatGenerator::Decode() { const int32_t token_id = *current_token_; current_token_.reset(); - const char* token_text = stream_->Decode(token_id); - const char* special_text = stream_with_special_->Decode(token_id); - std::string token = token_text ? token_text : ""; - - if (special_text != nullptr && token_text != nullptr && std::string(special_text) != token) { - const std::string special(special_text); - const bool surfaced_special = - special.find("tool_call") != std::string::npos || special.find("think") != std::string::npos; - const auto& eos_ids = model_.GetPreprocessor().GetEosTokenIds(); - const bool eos = std::find(eos_ids.begin(), eos_ids.end(), token_id) != eos_ids.end(); - if (surfaced_special && !eos) { - return special; - } + + const auto& tag_info = model_.GetTagInfo(); + if (tag_info.bot_id.has_value() && token_id == *tag_info.bot_id) { + stream_->Decode(token_id); + return tag_info.bot_str; + } + if (tag_info.eot_id.has_value() && token_id == *tag_info.eot_id) { + stream_->Decode(token_id); + return tag_info.eot_str; + } + if (tag_info.bor_id.has_value() && token_id == *tag_info.bor_id) { + stream_->Decode(token_id); + return tag_info.bor_str; + } + if (tag_info.eor_id.has_value() && token_id == *tag_info.eor_id) { + stream_->Decode(token_id); + return tag_info.eor_str; } - return token; + const char* token_text = stream_->Decode(token_id); + return token_text ? std::string(token_text) : ""; } int OnnxEngineChatGenerator::TokenCount() const { @@ -165,7 +168,6 @@ std::unique_ptr OnnxEngineChatGenerator::Create( auto sequences = EncodePrompt(prompt, model); const int prompt_token_count = static_cast(sequences->SequenceCount(0)); auto stream = model.GetPreprocessor().CreateTokenizerStream(); - auto stream_with_special = model.GetPreprocessor().CreateSpecialTokenizerStream(); auto conversation = engine->CreateConversation(options, tool_ctx, prompt_token_count); try { const auto* data = sequences->SequenceData(0); @@ -173,8 +175,8 @@ std::unique_ptr OnnxEngineChatGenerator::Create( ResolveMaxOutputTokens(options)); return std::unique_ptr( - new OnnxEngineChatGenerator(*engine, std::move(conversation), std::move(stream), - std::move(stream_with_special), model, prompt_token_count)); + new OnnxEngineChatGenerator(*engine, std::move(conversation), std::move(stream), model, + prompt_token_count)); } catch (...) { try { engine->Close(conversation); diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.h b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.h index 20d11e473..702024115 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.h @@ -47,14 +47,12 @@ class OnnxEngineChatGenerator final : public ChatGenerator { OnnxEngineChatGenerator(OnnxChatEngine& engine, std::shared_ptr conversation, std::unique_ptr stream, - std::unique_ptr stream_with_special, GenAIModelInstance& model, int prompt_token_count); OnnxChatEngine& engine_; std::shared_ptr conversation_; std::unique_ptr stream_; - std::unique_ptr stream_with_special_; GenAIModelInstance& model_; int prompt_token_count_ = 0; std::optional current_token_; diff --git a/sdk_v2/cpp/test/internal_api/chat/chat_session_test.cc b/sdk_v2/cpp/test/internal_api/chat/chat_session_test.cc index 2536a457f..adc54e789 100644 --- a/sdk_v2/cpp/test/internal_api/chat/chat_session_test.cc +++ b/sdk_v2/cpp/test/internal_api/chat/chat_session_test.cc @@ -36,9 +36,9 @@ namespace { class EngineModelStaging { public: - explicit EngineModelStaging(const std::filesystem::path& source) - : path_(source.parent_path() / - ("engine-chat-test-" + std::to_string(fl::test::CurrentPid()))) { + EngineModelStaging(const std::filesystem::path& source, ChatBackendKind backend) + : path_(source.parent_path() / ("engine-chat-test-" + BackendName(backend) + "-" + + std::to_string(fl::test::CurrentPid()))) { std::error_code ec; std::filesystem::remove_all(path_, ec); std::filesystem::create_directories(path_); @@ -59,9 +59,15 @@ class EngineModelStaging { const auto config_path = path_ / "genai_config.json"; std::ifstream input(config_path); auto config = nlohmann::json::parse(input); - config["engine"] = { - {"dynamic_batching", {{"max_batch_size", 2}, {"max_scheduled_tokens", 2048}}}, - }; + if (backend == ChatBackendKind::kDynamicEngine) { + config["engine"] = { + {"dynamic_batching", {{"max_batch_size", 2}, {"max_scheduled_tokens", 2048}}}, + }; + } else { + config["engine"] = { + {"static_batching", {{"max_batch_size", 2}}}, + }; + } std::ofstream(config_path) << config.dump(2); } catch (...) { std::filesystem::remove_all(path_, ec); @@ -80,6 +86,10 @@ class EngineModelStaging { const std::filesystem::path& path() const { return path_; } private: + static std::string BackendName(ChatBackendKind backend) { + return backend == ChatBackendKind::kDynamicEngine ? "dynamic" : "static"; + } + std::filesystem::path path_; }; @@ -93,7 +103,8 @@ class ChatSessionTest : public ::testing::Test { protected: static void SetUpTestSuite() { auto model_path = fl::test::GetTestModelPath(fl::test::kTestChatModelAlias); - engine_model_ = std::make_unique(model_path); + engine_model_ = + std::make_unique(model_path, ChatBackendKind::kDynamicEngine); logger_ = std::make_unique(); ep_detector_ = std::make_unique(); load_manager_ = std::make_unique(*ep_detector_, *logger_); @@ -403,6 +414,44 @@ TEST_F(ChatSessionTest, RunMultiTurn) { EXPECT_EQ(session.MessageCount(), 4u); } +TEST_F(ChatSessionTest, StaticEngineReconstructsMultiTurnHistory) { + constexpr const char* kStaticModelAlias = "static-engine-chat-test"; + auto model_path = fl::test::GetTestModelPath(fl::test::kTestChatModelAlias); + EngineModelStaging static_model(model_path, ChatBackendKind::kStaticEngine); + test::CpuOnlyEpDetector ep_detector; + ModelLoadManager load_manager(ep_detector, *logger_); + auto result = load_manager.LoadModel(static_model.path().string(), kStaticModelAlias); + ASSERT_EQ(result.status, ModelLoadManager::LoadStatus::kSuccess); + ASSERT_NE(result.model, nullptr); + ASSERT_EQ(result.model->GetGenAIConfig().GetChatBackendKind(), ChatBackendKind::kStaticEngine); + + { + ChatSession session(GetCatalogModel(), *result.model, *logger_, null_telemetry_); + + Request first_request; + first_request.AddOwnedItem( + MakeMessage(FOUNDRY_LOCAL_ROLE_USER, "Remember the word sapphire. Reply OK.")); + first_request.options.Add("max_output_tokens", "32"); + first_request.options.Add("temperature", "0"); + Response first_response; + session.ProcessRequest(first_request, first_response); + + Request second_request; + second_request.AddOwnedItem( + MakeMessage(FOUNDRY_LOCAL_ROLE_USER, "What word did I ask you to remember?")); + second_request.options.Add("max_output_tokens", "32"); + second_request.options.Add("temperature", "0"); + Response second_response; + session.ProcessRequest(second_request, second_response); + + EXPECT_NE(fl::test::ToLower(GetAssistantText(second_response)).find("sapphire"), + std::string::npos); + EXPECT_EQ(session.TurnCount(), 2u); + } + + EXPECT_TRUE(load_manager.UnloadModel(kStaticModelAlias)); +} + TEST_F(ChatSessionTest, RunStreamingCancellation) { ASSERT_NE(GetModel().GetGenAIConfig().GetChatBackendKind(), ChatBackendKind::kGenerator) << "The cancellation test model must declare an Engine backend"; From d9d981ae917646aad8483bcdb7314525f3e269c9 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sat, 5 Sep 2026 00:34:18 -0500 Subject: [PATCH 09/11] Recover Engine capacity without invalid test models Evict dormant dynamic conversations under capacity pressure so blocked turns can progress and rebuild evicted state from committed chat history. Stage the shared CPU model in a writable temp directory and exercise only its supported static Engine configuration. Files changed: chat_session.cc, onnx_chat_engine.cc, onnx_chat_engine.h, chat_session_test.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../generative/chat/chat_session.cc | 21 +-- .../generative/chat/onnx_chat_engine.cc | 44 +++++- .../generative/chat/onnx_chat_engine.h | 8 ++ .../internal_api/chat/chat_session_test.cc | 133 +++++++----------- 4 files changed, 112 insertions(+), 94 deletions(-) diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc index 8557ae7ca..acab8917a 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc @@ -509,14 +509,19 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) } else { // Continuous decoding: append only the new messages to the existing generator. pre_turn_token_count = cached_generator_->TokenCount(); - const std::string reasoning_start_marker = - cached_tool_ctx_.reasoning_start.empty() ? std::string("") : cached_tool_ctx_.reasoning_start; - prompt_tokens = cached_generator_->AppendMessages( - new_messages, Model(), cached_tool_ctx_.tools_json, effective_options, - cached_tool_ctx_.supports_reasoning ? reasoning_start_marker : std::string{}); - - // Refresh per-turn fields (tool_choice, guidance) while keeping session-level definitions stable. - UpdateToolContextForTurn(request, cached_tool_ctx_); + try { + const std::string reasoning_start_marker = + cached_tool_ctx_.reasoning_start.empty() ? std::string("") : cached_tool_ctx_.reasoning_start; + prompt_tokens = cached_generator_->AppendMessages( + new_messages, Model(), cached_tool_ctx_.tools_json, effective_options, + cached_tool_ctx_.supports_reasoning ? reasoning_start_marker : std::string{}); + + // Refresh per-turn fields (tool_choice, guidance) while keeping session-level definitions stable. + UpdateToolContextForTurn(request, cached_tool_ctx_); + } catch (const OnnxChatEngine::ConversationEvictedError&) { + cached_generator_.reset(); + cached_tool_ctx_ = {}; + } } } diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc index 6cb803f16..8d3390a47 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc @@ -285,10 +285,31 @@ void OnnxChatEngine::RouteEvents() { engine_->Run(*event_buffer_); for (size_t i = 0; i < event_buffer_->Count(); ++i) { const auto* event = event_buffer_->Get(i); + const auto flags = event->Flags(); const auto request = event->Request(); if (!request) { - continue; + if ((flags & OgaEngineEventFlag_Failed) != 0) { + throw std::runtime_error("ORT GenAI Engine failed with error code " + + std::to_string(event->ErrorCode())); + } + if ((flags & OgaEngineEventFlag_CapacityBlocked) != 0 && EvictDormantConversation()) { + consecutive_retry_events_ = 0; + continue; + } + if ((flags & (OgaEngineEventFlag_CapacityBlocked | OgaEngineEventFlag_Retryable)) != 0) { + constexpr size_t kMaxConsecutiveRetries = 100; + if (++consecutive_retry_events_ > kMaxConsecutiveRetries) { + throw std::runtime_error("ORT GenAI Engine made no progress after " + + std::to_string(kMaxConsecutiveRetries) + + " retryable events; last error code " + + std::to_string(event->ErrorCode())); + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + continue; + } + throw std::runtime_error("ORT GenAI Engine returned an invalid request-less event"); } + consecutive_retry_events_ = 0; auto it = std::find_if(conversations_.begin(), conversations_.end(), [&](const auto& entry) { return entry.second->request.get() == &request->get(); @@ -324,6 +345,25 @@ void OnnxChatEngine::RouteEvents() { } } +bool OnnxChatEngine::EvictDormantConversation() { + for (auto it = conversations_.begin(); it != conversations_.end(); ++it) { + auto conversation = it->second->state; + { + std::lock_guard lock(conversation->mutex); + if (!conversation->turn_finished) { + continue; + } + conversation->closed = true; + } + + it->second->request->Close(); + conversations_.erase(it); + conversation->cv.notify_all(); + return true; + } + return false; +} + void OnnxChatEngine::FailAll(std::exception_ptr error) { for (auto& [_, native] : conversations_) { { @@ -339,7 +379,7 @@ OnnxChatEngine::NativeConversation& OnnxChatEngine::FindNative( const std::shared_ptr& conversation) { auto it = conversations_.find(conversation.get()); if (it == conversations_.end()) { - throw std::runtime_error("Engine conversation is closed or does not belong to this model."); + throw ConversationEvictedError(); } return *it->second; } diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.h b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.h index 324ea0136..921c380a8 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,11 @@ struct ToolCallContext; /// Owns one ORT GenAI Engine and serializes every Engine operation onto its owner thread. class OnnxChatEngine { public: + class ConversationEvictedError : public std::runtime_error { + public: + ConversationEvictedError() : std::runtime_error("Engine conversation was evicted for capacity") {} + }; + struct TurnResult { uint64_t prompt_tokens = 0; uint64_t generated_tokens = 0; @@ -87,6 +93,7 @@ class OnnxChatEngine { void Enqueue(std::function command, std::function fail); void WorkerLoop(std::promise initialized); void RouteEvents(); + bool EvictDormantConversation(); void FailAll(std::exception_ptr error); NativeConversation& FindNative(const std::shared_ptr& conversation); @@ -102,6 +109,7 @@ class OnnxChatEngine { std::unique_ptr engine_; std::unique_ptr event_buffer_; std::unordered_map> conversations_; + size_t consecutive_retry_events_ = 0; }; } // namespace fl diff --git a/sdk_v2/cpp/test/internal_api/chat/chat_session_test.cc b/sdk_v2/cpp/test/internal_api/chat/chat_session_test.cc index adc54e789..7d43b45ae 100644 --- a/sdk_v2/cpp/test/internal_api/chat/chat_session_test.cc +++ b/sdk_v2/cpp/test/internal_api/chat/chat_session_test.cc @@ -36,61 +36,43 @@ namespace { class EngineModelStaging { public: - EngineModelStaging(const std::filesystem::path& source, ChatBackendKind backend) - : path_(source.parent_path() / ("engine-chat-test-" + BackendName(backend) + "-" + - std::to_string(fl::test::CurrentPid()))) { - std::error_code ec; - std::filesystem::remove_all(path_, ec); - std::filesystem::create_directories(path_); - - try { - for (const auto& entry : std::filesystem::recursive_directory_iterator(source)) { - const auto relative = std::filesystem::relative(entry.path(), source); - const auto destination = path_ / relative; - if (entry.is_directory()) { - std::filesystem::create_directories(destination); - } else if (entry.path().filename() == "genai_config.json") { - std::filesystem::copy_file(entry.path(), destination); - } else { - std::filesystem::create_hard_link(entry.path(), destination); - } - } - - const auto config_path = path_ / "genai_config.json"; - std::ifstream input(config_path); - auto config = nlohmann::json::parse(input); - if (backend == ChatBackendKind::kDynamicEngine) { - config["engine"] = { - {"dynamic_batching", {{"max_batch_size", 2}, {"max_scheduled_tokens", 2048}}}, - }; + explicit EngineModelStaging(const std::filesystem::path& source) + : root_(fl::test::TempPath::CreateTempDir("engine-chat-test-static-")) { + for (const auto& entry : std::filesystem::recursive_directory_iterator(source)) { + const auto relative = std::filesystem::relative(entry.path(), source); + const auto destination = root_.path() / relative; + if (entry.is_directory()) { + std::filesystem::create_directories(destination); } else { - config["engine"] = { - {"static_batching", {{"max_batch_size", 2}}}, - }; + std::filesystem::copy_file(entry.path(), destination); + std::filesystem::permissions(destination, std::filesystem::perms::owner_write, + std::filesystem::perm_options::add); } - std::ofstream(config_path) << config.dump(2); - } catch (...) { - std::filesystem::remove_all(path_, ec); - throw; } - } - ~EngineModelStaging() { - std::error_code ec; - std::filesystem::remove_all(path_, ec); + const auto config_path = root_.path() / "genai_config.json"; + std::ifstream input(config_path); + if (!input) { + throw std::runtime_error("Failed to open staged genai_config.json"); + } + auto config = nlohmann::json::parse(input); + input.close(); + config["engine"] = { + {"static_batching", {{"max_batch_size", 2}}}, + }; + std::ofstream output(config_path, std::ios::trunc); + if (!output || !(output << config.dump(2))) { + throw std::runtime_error("Failed to write staged genai_config.json"); + } } EngineModelStaging(const EngineModelStaging&) = delete; EngineModelStaging& operator=(const EngineModelStaging&) = delete; - const std::filesystem::path& path() const { return path_; } + const std::filesystem::path& path() const { return root_.path(); } private: - static std::string BackendName(ChatBackendKind backend) { - return backend == ChatBackendKind::kDynamicEngine ? "dynamic" : "static"; - } - - std::filesystem::path path_; + fl::test::TempPath root_; }; } // namespace @@ -103,8 +85,7 @@ class ChatSessionTest : public ::testing::Test { protected: static void SetUpTestSuite() { auto model_path = fl::test::GetTestModelPath(fl::test::kTestChatModelAlias); - engine_model_ = - std::make_unique(model_path, ChatBackendKind::kDynamicEngine); + engine_model_ = std::make_unique(model_path); logger_ = std::make_unique(); ep_detector_ = std::make_unique(); load_manager_ = std::make_unique(*ep_detector_, *logger_); @@ -217,8 +198,7 @@ TEST_F(ChatSessionTest, RunBasic) { } TEST_F(ChatSessionTest, ConcurrentIndependentSessions) { - ASSERT_NE(GetModel().GetGenAIConfig().GetChatBackendKind(), ChatBackendKind::kGenerator) - << "The concurrency test model must declare an Engine backend"; + ASSERT_EQ(GetModel().GetGenAIConfig().GetChatBackendKind(), ChatBackendKind::kStaticEngine); ASSERT_NE(GetModel().GetChatEngine(), nullptr); auto run_request = [this](std::string prompt) { @@ -415,46 +395,31 @@ TEST_F(ChatSessionTest, RunMultiTurn) { } TEST_F(ChatSessionTest, StaticEngineReconstructsMultiTurnHistory) { - constexpr const char* kStaticModelAlias = "static-engine-chat-test"; - auto model_path = fl::test::GetTestModelPath(fl::test::kTestChatModelAlias); - EngineModelStaging static_model(model_path, ChatBackendKind::kStaticEngine); - test::CpuOnlyEpDetector ep_detector; - ModelLoadManager load_manager(ep_detector, *logger_); - auto result = load_manager.LoadModel(static_model.path().string(), kStaticModelAlias); - ASSERT_EQ(result.status, ModelLoadManager::LoadStatus::kSuccess); - ASSERT_NE(result.model, nullptr); - ASSERT_EQ(result.model->GetGenAIConfig().GetChatBackendKind(), ChatBackendKind::kStaticEngine); - - { - ChatSession session(GetCatalogModel(), *result.model, *logger_, null_telemetry_); - - Request first_request; - first_request.AddOwnedItem( - MakeMessage(FOUNDRY_LOCAL_ROLE_USER, "Remember the word sapphire. Reply OK.")); - first_request.options.Add("max_output_tokens", "32"); - first_request.options.Add("temperature", "0"); - Response first_response; - session.ProcessRequest(first_request, first_response); - - Request second_request; - second_request.AddOwnedItem( - MakeMessage(FOUNDRY_LOCAL_ROLE_USER, "What word did I ask you to remember?")); - second_request.options.Add("max_output_tokens", "32"); - second_request.options.Add("temperature", "0"); - Response second_response; - session.ProcessRequest(second_request, second_response); - - EXPECT_NE(fl::test::ToLower(GetAssistantText(second_response)).find("sapphire"), - std::string::npos); - EXPECT_EQ(session.TurnCount(), 2u); - } + ASSERT_EQ(GetModel().GetGenAIConfig().GetChatBackendKind(), ChatBackendKind::kStaticEngine); + ChatSession session(GetCatalogModel(), GetModel(), *logger_, null_telemetry_); - EXPECT_TRUE(load_manager.UnloadModel(kStaticModelAlias)); + Request first_request; + first_request.AddOwnedItem( + MakeMessage(FOUNDRY_LOCAL_ROLE_USER, "Remember the word sapphire. Reply OK.")); + first_request.options.Add("max_output_tokens", "32"); + first_request.options.Add("temperature", "0"); + Response first_response; + session.ProcessRequest(first_request, first_response); + + Request second_request; + second_request.AddOwnedItem( + MakeMessage(FOUNDRY_LOCAL_ROLE_USER, "What word did I ask you to remember?")); + second_request.options.Add("max_output_tokens", "32"); + second_request.options.Add("temperature", "0"); + Response second_response; + session.ProcessRequest(second_request, second_response); + + EXPECT_NE(fl::test::ToLower(GetAssistantText(second_response)).find("sapphire"), std::string::npos); + EXPECT_EQ(session.TurnCount(), 2u); } TEST_F(ChatSessionTest, RunStreamingCancellation) { - ASSERT_NE(GetModel().GetGenAIConfig().GetChatBackendKind(), ChatBackendKind::kGenerator) - << "The cancellation test model must declare an Engine backend"; + ASSERT_EQ(GetModel().GetGenAIConfig().GetChatBackendKind(), ChatBackendKind::kStaticEngine); ASSERT_NE(GetModel().GetChatEngine(), nullptr); ChatSession session(GetCatalogModel(), GetModel(), *logger_, null_telemetry_); From 90e7f777feffd23564676ea0d93bbe61acd2615c Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sat, 5 Sep 2026 00:44:58 -0500 Subject: [PATCH 10/11] Protect new Engine conversations from eviction Only completed conversations with a nonzero turn ID are dormant candidates, preventing capacity pressure from closing a request between creation and its first BeginTurn. Files changed: onnx_chat_engine.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc index 8d3390a47..166e6bc46 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc @@ -350,7 +350,7 @@ bool OnnxChatEngine::EvictDormantConversation() { auto conversation = it->second->state; { std::lock_guard lock(conversation->mutex); - if (!conversation->turn_finished) { + if (!conversation->turn_finished || conversation->turn_id == 0) { continue; } conversation->closed = true; From 5f37c2865fbc94d312e739ab5cf34cd0e4fc476e Mon Sep 17 00:00:00 2001 From: David Fan Date: Mon, 7 Sep 2026 19:49:43 -0700 Subject: [PATCH 11/11] Resolve PR 1060 and 1071 integration conflicts --- sdk_v2/cpp/src/catalog/azure_model_catalog.h | 4 +- .../generative/chat/chat_generator.h | 3 + .../generative/chat/chat_session.cc | 124 ++++--- .../generative/chat/chat_session.h | 6 +- .../generative/chat/onnx_chat_engine.cc | 1 - .../generative/chat/onnx_chat_generator.cc | 25 +- .../generative/chat/onnx_chat_generator.h | 4 +- .../chat/onnx_engine_chat_generator.h | 2 + .../chat/reasoning_stream_splitter.h | 320 +++++++++++++++--- .../tool_call_stream_accumulator.h | 65 +++- .../generative/toolcalling/tool_call_utils.cc | 28 +- 11 files changed, 436 insertions(+), 146 deletions(-) diff --git a/sdk_v2/cpp/src/catalog/azure_model_catalog.h b/sdk_v2/cpp/src/catalog/azure_model_catalog.h index df75f70f2..9e584a774 100644 --- a/sdk_v2/cpp/src/catalog/azure_model_catalog.h +++ b/sdk_v2/cpp/src/catalog/azure_model_catalog.h @@ -60,8 +60,8 @@ class AzureModelCatalog : public BaseModelCatalog { static constexpr const char* kDefaultCatalogFilter = "''"; CatalogResult GetLiveCatalogOrLocalSnapshot(const std::vector& cached_model_ids) const; - std::vector CreateModelsWithLocalPaths(const std::vector& model_infos, - const LocalModels& local_models) const; + std::vector AddLocalModels(std::vector& model_infos, + const LocalModels& local_models) const; std::vector>> catalog_urls_; std::string cache_dir_; diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.h b/sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.h index 9f23ed191..fdeb8d1e8 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.h @@ -42,6 +42,9 @@ class ChatGenerator { /// Get the most recently generated token ID before Decode consumes it. virtual std::optional CurrentTokenId() const = 0; + /// Return whether the rendered prompt ends with an open reasoning marker. + virtual bool PromptEndsInReasoning() const = 0; + /// Get the total number of tokens (input + generated) so far. virtual int TokenCount() const = 0; diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc index acab8917a..d686f0ee1 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc @@ -61,6 +61,25 @@ std::unique_ptr CreateTextChatGenerator(const std::vector{*tag_info.bor_id} : std::vector{}; + auto end_token_ids = + tag_info.eor_id.has_value() ? std::vector{*tag_info.eor_id} : std::vector{}; + auto ignored_token_ids = model.GetPreprocessor().GetEosTokenIds(); + return {std::move(start), std::move(end), std::move(start_token_ids), std::move(end_token_ids), + std::move(ignored_token_ids), start_inside_reasoning}; +} + } // namespace ChatSession::ChatSession(const fl::Model& catalog_model, GenAIModelInstance& model, ILogger& logger, ITelemetry& telemetry) @@ -568,14 +587,8 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) // Splitter: only active for reasoning models. For non-reasoning models start_marker is empty and the splitter // degrades to a passthrough (every token becomes one DEFAULT segment), so the streaming path stays uniform. - ReasoningStreamSplitter splitter( - cached_tool_ctx_.supports_reasoning ? (cached_tool_ctx_.reasoning_start.empty() ? std::string("") - : cached_tool_ctx_.reasoning_start) - : std::string(), - cached_tool_ctx_.supports_reasoning ? (cached_tool_ctx_.reasoning_end.empty() ? std::string("") - : cached_tool_ctx_.reasoning_end) - : std::string(), - cached_generator_ && cached_generator_->PromptEndsInReasoning()); + auto splitter = CreateReasoningSplitter( + cached_tool_ctx_, Model(), cached_generator_ && cached_generator_->PromptEndsInReasoning()); // Accumulator: separates visible text from tool-call blocks in the DEFAULT-segment stream. For models without // tool-call markers configured, both marker strings are empty and the accumulator degrades to passthrough. @@ -591,6 +604,24 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) // which is fine because the IDs only need to be stable when a client is observing the stream. std::vector streamed_tool_calls; + auto emit_tool_output = [&](ToolCallStreamAccumulator::Output output) { + for (auto& event : output.events) { + if (auto* visible_text = std::get_if(&event)) { + if (streaming_callback) { + streaming_callback->PushItem( + std::make_unique(std::move(*visible_text), FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT)); + } + } else { + auto parsed_call = std::move(std::get(event)); + if (streaming_callback) { + streaming_callback->PushItem( + std::make_unique(parsed_call.id, parsed_call.name, parsed_call.arguments)); + } + streamed_tool_calls.push_back(std::move(parsed_call)); + } + } + }; + auto emit_segments = [&](const std::vector& segments) { for (const auto& seg : segments) { if (seg.type == FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING) { @@ -601,45 +632,24 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) continue; } - auto out = tool_accumulator.Push(seg.text); - - if (streaming_callback && !out.visible_text.empty()) { - streaming_callback->PushItem( - std::make_unique(std::move(out.visible_text), FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT)); - } - - for (auto& pc : out.ready_calls) { - if (streaming_callback) { - streaming_callback->PushItem(std::make_unique(pc.id, pc.name, pc.arguments)); - } - streamed_tool_calls.push_back(std::move(pc)); - } + emit_tool_output(tool_accumulator.Push(seg.text)); } }; - auto flush_accumulator = [&]() { - auto out = tool_accumulator.Flush(); - - if (streaming_callback && !out.visible_text.empty()) { - streaming_callback->PushItem( - std::make_unique(std::move(out.visible_text), FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT)); - } - - for (auto& pc : out.ready_calls) { - if (streaming_callback) { - streaming_callback->PushItem(std::make_unique(pc.id, pc.name, pc.arguments)); - } - streamed_tool_calls.push_back(std::move(pc)); - } - }; + auto flush_accumulator = [&]() { emit_tool_output(tool_accumulator.Flush()); }; while (!cached_generator_->IsDone() && !request.canceled) { cached_generator_->GenerateNextToken(); + const auto token_id = cached_generator_->CurrentTokenId(); std::string token = cached_generator_->Decode(); ++output_tokens; if (!token.empty()) { text += token; + } + if (token_id.has_value()) { + emit_segments(splitter.Push(*token_id, std::move(token))); + } else if (!token.empty()) { emit_segments(splitter.Push(token)); } @@ -811,14 +821,7 @@ void ChatSession::ProcessChatCompletionsJson(const std::string& request_json, co // Chat Completions stream — the OpenAI Chat Completions spec has no reasoning-delta concept; reasoning is exposed // via the Responses API path in Stage 4. The non-streaming response already excludes reasoning text from // `delta.content` via the typed-MessageItem build in ProcessGeneratedOutput. - ReasoningStreamSplitter splitter( - tool_ctx.supports_reasoning ? (tool_ctx.reasoning_start.empty() ? std::string("") - : tool_ctx.reasoning_start) - : std::string(), - tool_ctx.supports_reasoning ? (tool_ctx.reasoning_end.empty() ? std::string("") - : tool_ctx.reasoning_end) - : std::string(), - generator->PromptEndsInReasoning()); + auto splitter = CreateReasoningSplitter(tool_ctx, Model(), generator->PromptEndsInReasoning()); auto emit_visible_text = [&](std::string visible) { if (visible.empty() || !is_streaming) { @@ -860,6 +863,24 @@ void ChatSession::ProcessChatCompletionsJson(const std::string& request_json, co } }; + auto process_tool_output = [&](ToolCallStreamAccumulator::Output output) { + std::vector ready_calls; + auto flush_ready_calls = [&]() { + emit_ready_calls(ready_calls); + ready_calls.clear(); + }; + + for (auto& event : output.events) { + if (auto* visible_text = std::get_if(&event)) { + flush_ready_calls(); + emit_visible_text(std::move(*visible_text)); + } else { + ready_calls.push_back(std::move(std::get(event))); + } + } + flush_ready_calls(); + }; + auto process_segments = [&](const std::vector& segments) { for (const auto& seg : segments) { // REASONING segments: intentionally dropped from the Chat Completions stream. Never feed reasoning text to @@ -868,9 +889,7 @@ void ChatSession::ProcessChatCompletionsJson(const std::string& request_json, co continue; } - auto out = tool_accumulator.Push(seg.text); - emit_visible_text(std::move(out.visible_text)); - emit_ready_calls(out.ready_calls); + process_tool_output(tool_accumulator.Push(seg.text)); } }; @@ -878,10 +897,15 @@ void ChatSession::ProcessChatCompletionsJson(const std::string& request_json, co std::string text; while (!generator->IsDone() && !original_request.canceled) { generator->GenerateNextToken(); + const auto token_id = generator->CurrentTokenId(); std::string token = generator->Decode(); if (!token.empty()) { text += token; + } + if (token_id.has_value()) { + process_segments(splitter.Push(*token_id, std::move(token))); + } else if (!token.empty()) { process_segments(splitter.Push(token)); } } @@ -889,11 +913,7 @@ void ChatSession::ProcessChatCompletionsJson(const std::string& request_json, co // Drain any buffered partial-marker bytes at end-of-stream. Reasoning splitter first so any final DEFAULT bytes // feed into the tool accumulator; then drain the tool accumulator. process_segments(splitter.Flush()); - { - auto out = tool_accumulator.Flush(); - emit_visible_text(std::move(out.visible_text)); - emit_ready_calls(out.ready_calls); - } + process_tool_output(tool_accumulator.Flush()); if (original_request.canceled) { generator->Cancel(); diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h index 302a09afd..dbd446ded 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h @@ -90,11 +90,11 @@ class ChatSession : public Session { /// while keeping session-level tool definitions and marker tokens stable. void UpdateToolContextForTurn(const Request& request, ToolCallContext& tool_ctx) const; - /// Build final response items from the typed segments and tool calls produced during generation. - void ProcessGeneratedOutput(std::vector events, + /// Build final response items from the generated text and tool calls produced during generation. + void ProcessGeneratedOutput(std::string text, const ToolCallContext& tool_ctx, const SearchOptions& effective_options, bool canceled, Response& response, int prompt_tokens, int total_tokens, - int reasoning_tokens); + std::vector pre_parsed_calls = {}); /// Process a request whose first item is a TextItem tagged OPENAI_JSON containing an OpenAI chat completions /// request. Parses the JSON, converts to internal items, runs generation, and produces an OPENAI_JSON-tagged diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc index 166e6bc46..3ecd04c56 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_engine.cc @@ -319,7 +319,6 @@ void OnnxChatEngine::RouteEvents() { } auto& conversation = it->second->state; - const auto flags = event->Flags(); { std::lock_guard lock(conversation->mutex); if ((flags & OgaEngineEventFlag_Token) != 0) { diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc index d3749eab6..b6e32acf1 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc @@ -82,11 +82,19 @@ bool OnnxChatGenerator::IsDone() const { void OnnxChatGenerator::GenerateNextToken() { if (cancelled_) { + current_token_.reset(); return; } + current_token_.reset(); + try { generator_->GenerateNextToken(); + + const auto next_tokens = generator_->GetNextTokens(); + if (!next_tokens.empty()) { + current_token_ = next_tokens[0]; + } } catch (const std::runtime_error& e) { // If cancelled while generating, the OGA engine throws when the session is terminated. // This is expected — not an error. @@ -99,19 +107,12 @@ void OnnxChatGenerator::GenerateNextToken() { } std::string OnnxChatGenerator::Decode() { - if (cancelled_) { - return ""; - } - - // Get the most recently generated token ID. - // GetNextTokens returns the batch of next tokens; we use index 0 (batch size = 1). - auto next_tokens = generator_->GetNextTokens(); - - if (next_tokens.empty()) { + if (cancelled_ || !current_token_.has_value()) { return ""; } - int32_t token_id = next_tokens[0]; + const int32_t token_id = *current_token_; + current_token_.reset(); // Decode through the normal tokenizer stream const char* token_text = stream_->Decode(token_id); @@ -140,6 +141,10 @@ std::string OnnxChatGenerator::Decode() { return token_str; } +std::optional OnnxChatGenerator::CurrentTokenId() const { + return current_token_; +} + int OnnxChatGenerator::TokenCount() const { return static_cast(generator_->GetSequenceCount(0)); } diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.h b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.h index 6b6eb0b32..b2f70145c 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.h @@ -38,6 +38,7 @@ class OnnxChatGenerator : public ChatGenerator { bool IsDone() const override; void GenerateNextToken() override; std::string Decode() override; + std::optional CurrentTokenId() const override; int TokenCount() const override; int PromptTokenCount() const override; void Cancel() override; @@ -47,7 +48,7 @@ class OnnxChatGenerator : public ChatGenerator { /// Reasoning-model chat templates pre-fill this marker so the model's first /// generated token is reasoning content rather than the marker itself. The /// stream splitter consumes this flag to start in the reasoning state. - bool PromptEndsInReasoning() const { return prompt_ends_in_reasoning_; } + bool PromptEndsInReasoning() const override { return prompt_ends_in_reasoning_; } /// Encode new messages and append their tokens to the generator's sequence. /// Used for continuous decoding — only the new turn's messages are encoded and appended. @@ -145,6 +146,7 @@ class OnnxChatGenerator : public ChatGenerator { GenAIModelInstance& model_; // non-owning reference — model outlives generator int prompt_token_count_ = 0; bool prompt_ends_in_reasoning_ = false; + std::optional current_token_; std::atomic cancelled_{false}; }; diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.h b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.h index 702024115..280234620 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_engine_chat_generator.h @@ -25,6 +25,8 @@ class OnnxEngineChatGenerator final : public ChatGenerator { bool IsDone() const override; void GenerateNextToken() override; std::string Decode() override; + std::optional CurrentTokenId() const override { return current_token_; } + bool PromptEndsInReasoning() const override { return false; } int TokenCount() const override; int PromptTokenCount() const override; void Cancel() override; diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/reasoning_stream_splitter.h b/sdk_v2/cpp/src/inferencing/generative/chat/reasoning_stream_splitter.h index d01af8a64..1b6564ae8 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/reasoning_stream_splitter.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/reasoning_stream_splitter.h @@ -4,25 +4,23 @@ #include "foundry_local/foundry_local_c.h" +#include +#include +#include #include #include #include namespace fl { -/// Token-level state machine that splits a stream of generated text chunks around reasoning markers -/// (e.g. `` / ``) into typed segments. +/// Token-aware state machine that splits generated output around reasoning markers into typed segments. /// -/// The streaming code calls `Push(token)` for every decoded token and forwards the returned segments to the -/// caller as typed `TextItem`s. At end-of-generation, `Flush()` drains any buffered bytes (e.g. a trailing -/// partial marker that turned out not to be one). +/// Marker token IDs come from ORT GenAI's model metadata. Matching IDs before inspecting decoded text is required +/// for special tokens, whose decoded chunks can be empty when the tokenizer skips special tokens. Token-prefix +/// buffering also supports compatibility callers that provide markers composed of multiple token IDs. /// -/// Why a state machine: the marker can straddle multiple tokens (a tokenizer might split `` into -/// ``). We must not emit the partial prefix as visible text and then realize on the next -/// token that it was actually a marker. The buffer holds the suffix that could still grow into the marker. -/// -/// When `start_marker` is empty, the splitter degrades to a passthrough that always emits DEFAULT segments — -/// non-reasoning models share this code without a behavior change. +/// The text-only Push overload preserves the prior decoded-marker behavior for callers without token IDs. When +/// `start_marker` is empty, both modes degrade to a DEFAULT passthrough for non-reasoning models. class ReasoningStreamSplitter { public: struct Segment { @@ -30,46 +28,58 @@ class ReasoningStreamSplitter { flTextItemType type; }; - /// @param start_inside_reasoning True when the chat template pre-fills the reasoning - /// open marker (e.g. `\n`) before the model's first token. In that case - /// the model's first byte is reasoning content and the splitter must start in - /// the reasoning state instead of waiting for a start marker that will never - /// appear. - ReasoningStreamSplitter(std::string start_marker, std::string end_marker, + ReasoningStreamSplitter(std::string start_marker, + std::string end_marker, + std::vector start_token_ids = {}, + std::vector end_token_ids = {}, + std::vector ignored_token_ids = {}, bool start_inside_reasoning = false) : start_marker_(std::move(start_marker)), end_marker_(std::move(end_marker)), + start_token_ids_(std::move(start_token_ids)), + end_token_ids_(std::move(end_token_ids)), + ignored_token_ids_(std::move(ignored_token_ids)), inside_reasoning_(start_inside_reasoning) {} - /// Feed a token into the splitter. Returns zero or more segments to emit. - std::vector Push(const std::string& token) { - std::vector out; + /// Feed one generated token into the splitter. Marker IDs are consumed even when decoded_text is empty. + std::vector Push(int32_t token_id, std::string decoded_text) { + if (!HasTextMarkers()) { + if (decoded_text.empty() || IsIgnoredToken(token_id)) { + return {}; + } - if (token.empty()) { - return out; + return {{std::move(decoded_text), FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT}}; } - if (start_marker_.empty()) { - out.push_back({token, FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT}); - return out; + if (!HasTokenMarkers()) { + return PushText(decoded_text, IsIgnoredToken(token_id)); } - buffer_ += token; - Drain(out, /*flushing=*/false); - + std::vector out; + pending_tokens_.push_back({token_id, std::move(decoded_text)}); + DrainTokens(out, /*flushing=*/false); return out; } - /// Drain any remaining buffered bytes at end-of-stream. Buffered bytes that looked like a partial marker - /// turn out not to be — emit them with the current type. + /// Feed a decoded token into the text-only fallback. + std::vector Push(const std::string& token) { + return PushText(token, false); + } + + /// Drain pending content at end-of-generation. A partial marker is content in the current reasoning state. std::vector Flush() { std::vector out; - if (start_marker_.empty()) { + if (!HasTextMarkers()) { return out; } - Drain(out, /*flushing=*/true); + if (HasTokenMarkers()) { + DrainTokens(out, /*flushing=*/true); + DrainText(out, /*flushing=*/true); + } else { + DrainText(out, /*flushing=*/true); + } return out; } @@ -78,8 +88,167 @@ class ReasoningStreamSplitter { /// downstream decisions (e.g. suppressing chunks) without inspecting segment types. bool InsideReasoning() const noexcept { return inside_reasoning_; } + /// Number of generated content tokens classified as reasoning. Boundary marker tokens are excluded. + int ReasoningTokenCount() const noexcept { return reasoning_token_count_; } + private: - void Drain(std::vector& out, bool flushing) { + struct PendingToken { + int32_t id; + std::string text; + }; + + struct PendingTextToken { + std::string text; + bool reasoning_counted = false; + bool ignored = false; + }; + + bool HasTextMarkers() const noexcept { + return !start_marker_.empty() && !end_marker_.empty(); + } + + bool HasTokenMarkers() const noexcept { + return !start_token_ids_.empty() && !end_token_ids_.empty(); + } + + std::vector PushText(const std::string& token, bool ignored) { + std::vector out; + + if (token.empty()) { + return out; + } + + if (!HasTextMarkers()) { + if (!ignored) { + out.push_back({token, FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT}); + } + return out; + } + + pending_text_tokens_.push_back({token, false, ignored}); + text_buffer_ += token; + DrainText(out, /*flushing=*/false); + return out; + } + + void DrainTokens(std::vector& out, bool flushing) { + while (!pending_tokens_.empty()) { + const auto& marker = inside_reasoning_ ? end_token_ids_ : start_token_ids_; + const auto found = FindTokenSequence(pending_tokens_, marker); + + if (found < pending_tokens_.size()) { + const auto state_before_prefix = inside_reasoning_; + EmitPendingTokens(out, found); + if (inside_reasoning_ != state_before_prefix) { + continue; + } + + // A decoded-marker prefix buffered before this ID marker is ordinary content because the complete boundary + // is represented by the IDs below. + DrainText(out, /*flushing=*/true); + if (inside_reasoning_ != state_before_prefix) { + continue; + } + + pending_tokens_.erase( + pending_tokens_.begin(), + pending_tokens_.begin() + static_cast(marker.size())); + inside_reasoning_ = !inside_reasoning_; + trim_default_prefix_ = !inside_reasoning_; + continue; + } + + if (flushing) { + const auto state_before_flush = inside_reasoning_; + EmitPendingTokens(out, pending_tokens_.size()); + if (inside_reasoning_ == state_before_flush) { + return; + } + + continue; + } + + const auto hold = LongestTokenSuffixThatIsPrefixOf(pending_tokens_, marker); + const auto safe = pending_tokens_.size() - hold; + const auto state_before_safe_tokens = inside_reasoning_; + EmitPendingTokens(out, safe); + if (inside_reasoning_ != state_before_safe_tokens) { + continue; + } + + return; + } + } + + void EmitPendingTokens(std::vector& out, size_t count) { + if (count == 0) { + return; + } + + for (size_t i = 0; i < count; ++i) { + auto token = std::move(pending_tokens_.front()); + pending_tokens_.erase(pending_tokens_.begin()); + const auto was_inside_reasoning = inside_reasoning_; + + if (IsIgnoredToken(token.id)) { + // EOS and configured control tokens are neither reasoning content nor visible output, regardless of how + // the tokenizer chooses to decode them. + } else if (token.text.empty()) { + if (inside_reasoning_) { + ++reasoning_token_count_; + } + } else { + pending_text_tokens_.push_back({token.text}); + text_buffer_ += token.text; + DrainText(out, /*flushing=*/false); + } + + if (inside_reasoning_ != was_inside_reasoning) { + return; + } + } + } + + static size_t FindTokenSequence(const std::vector& tokens, + const std::vector& marker) { + if (marker.empty() || tokens.size() < marker.size()) { + return tokens.size(); + } + + for (size_t pos = 0; pos + marker.size() <= tokens.size(); ++pos) { + const auto matches = std::equal( + marker.begin(), marker.end(), tokens.begin() + static_cast(pos), + [](int32_t marker_id, const PendingToken& token) { return marker_id == token.id; }); + if (matches) { + return pos; + } + } + + return tokens.size(); + } + + bool IsIgnoredToken(int32_t token_id) const { + return std::find(ignored_token_ids_.begin(), ignored_token_ids_.end(), token_id) != + ignored_token_ids_.end(); + } + + static size_t LongestTokenSuffixThatIsPrefixOf(const std::vector& tokens, + const std::vector& marker) { + const auto max_length = std::min(tokens.size(), marker.size()); + for (size_t length = max_length; length > 0; --length) { + const auto token_start = tokens.end() - static_cast(length); + const auto matches = std::equal( + marker.begin(), marker.begin() + static_cast(length), token_start, + [](int32_t marker_id, const PendingToken& token) { return marker_id == token.id; }); + if (matches) { + return length; + } + } + + return 0; + } + + void DrainText(std::vector& out, bool flushing) { while (true) { const std::string& marker = inside_reasoning_ ? end_marker_ : start_marker_; flTextItemType current_type = inside_reasoning_ ? FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING @@ -88,51 +257,89 @@ class ReasoningStreamSplitter { // Marker may be empty (e.g. end_marker not configured). With no end marker we can never close a // reasoning block — drain the buffer with the current type and stop. if (marker.empty()) { - EmitSegment(out, std::move(buffer_), current_type); - buffer_.clear(); + EmitTextSegment(out, ConsumeText(text_buffer_.size(), current_type, /*is_content=*/true), current_type); return; } - size_t found = buffer_.find(marker); + size_t found = text_buffer_.find(marker); if (found != std::string::npos) { // Emit prefix with current type, consume marker, flip state. - EmitSegment(out, buffer_.substr(0, found), current_type); - - size_t after = found + marker.size(); + EmitTextSegment(out, ConsumeText(found, current_type, /*is_content=*/true), current_type); + ConsumeText(marker.size(), current_type, /*is_content=*/false); + const auto closed_reasoning = inside_reasoning_; + inside_reasoning_ = !inside_reasoning_; + trim_default_prefix_ = !inside_reasoning_; - // Drop a single trailing newline immediately after the closing marker — matches the non-streaming - // SplitReasoningContent behavior so callers see the same visible text either way. - if (inside_reasoning_ && after < buffer_.size() && buffer_[after] == '\n') { - ++after; + // Preserve the established behavior of dropping a newline immediately after a closed reasoning block. + if (closed_reasoning && !text_buffer_.empty() && text_buffer_.front() == '\n') { + ConsumeText(1, FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT, /*is_content=*/false); + trim_default_prefix_ = false; } - buffer_.erase(0, after); - inside_reasoning_ = !inside_reasoning_; - continue; // re-scan the remaining buffer for the next marker } // No full marker. If we're flushing, emit everything and stop. Otherwise hold back the longest suffix // of buffer_ that could still grow into the marker. if (flushing) { - EmitSegment(out, std::move(buffer_), current_type); - buffer_.clear(); + EmitTextSegment(out, ConsumeText(text_buffer_.size(), current_type, /*is_content=*/true), current_type); return; } - size_t hold = LongestSuffixThatIsPrefixOf(buffer_, marker); - size_t safe = buffer_.size() - hold; + size_t hold = LongestSuffixThatIsPrefixOf(text_buffer_, marker); + size_t safe = text_buffer_.size() - hold; if (safe > 0) { - EmitSegment(out, buffer_.substr(0, safe), current_type); - buffer_.erase(0, safe); + EmitTextSegment(out, ConsumeText(safe, current_type, /*is_content=*/true), current_type); } return; } } + std::string ConsumeText(size_t length, flTextItemType type, bool is_content) { + std::string text; + text.reserve(length); + text_buffer_.erase(0, length); + + auto remaining = length; + while (remaining > 0 && !pending_text_tokens_.empty()) { + auto& token = pending_text_tokens_.front(); + const auto consumed = std::min(remaining, token.text.size()); + + // Ignored token bytes participate in marker matching but are never emitted or counted. + if (is_content && !token.ignored) { + text.append(token.text, 0, consumed); + if (type == FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING && !token.reasoning_counted) { + ++reasoning_token_count_; + token.reasoning_counted = true; + } + } + + token.text.erase(0, consumed); + remaining -= consumed; + if (token.text.empty()) { + pending_text_tokens_.erase(pending_text_tokens_.begin()); + } + } + + return text; + } + + void EmitTextSegment(std::vector& out, std::string text, flTextItemType type) { + if (type == FOUNDRY_LOCAL_TEXT_ITEM_TYPE_DEFAULT && trim_default_prefix_ && !text.empty()) { + if (text.starts_with("\r\n")) { + text.erase(0, 2); + } else if (text.starts_with('\n')) { + text.erase(0, 1); + } + trim_default_prefix_ = false; + } + + EmitSegment(out, std::move(text), type); + } + static void EmitSegment(std::vector& out, std::string text, flTextItemType type) { if (text.empty()) { return; @@ -156,8 +363,15 @@ class ReasoningStreamSplitter { std::string start_marker_; std::string end_marker_; - std::string buffer_; + std::vector start_token_ids_; + std::vector end_token_ids_; + std::vector ignored_token_ids_; + std::vector pending_tokens_; + std::vector pending_text_tokens_; + std::string text_buffer_; bool inside_reasoning_ = false; + bool trim_default_prefix_ = false; + int reasoning_token_count_ = 0; }; } // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_stream_accumulator.h b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_stream_accumulator.h index 95a1de8b7..d4e11b129 100644 --- a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_stream_accumulator.h +++ b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_stream_accumulator.h @@ -7,6 +7,7 @@ #include #include #include +#include #include namespace fl { @@ -19,10 +20,10 @@ namespace fl { /// across tokens, parse it once the closing marker arrives, and surface the structured `ParsedToolCall`s. /// /// `Push(chunk)` accepts any text chunk (a single decoded token, or a multi-token segment produced by the upstream -/// `ReasoningStreamSplitter`) and returns: -/// - `visible_text`: text that is safe to emit to the caller (everything outside a tool-call block, minus any -/// pending suffix that could still grow into the start marker). -/// - `ready_calls`: zero or more fully parsed tool calls whose closing marker arrived in this chunk. +/// `ReasoningStreamSplitter`) and returns ordered events containing: +/// - text that is safe to emit to the caller (everything outside a tool-call block, minus any pending suffix that +/// could still grow into the start marker); +/// - fully parsed tool calls whose closing marker arrived in this chunk. /// /// Marker matching is buffered, mirroring `ReasoningStreamSplitter`: a marker can straddle multiple tokens, so the /// accumulator holds back the longest suffix of its scan buffer that could still extend into the marker rather than @@ -32,15 +33,18 @@ namespace fl { /// returned as visible text — they turned out not to be a tool call, so the caller still sees what the model /// produced. Matches `ReasoningStreamSplitter::Flush()`. /// -/// When either marker is empty, the accumulator degrades to a passthrough: `Push` returns its input verbatim as -/// `visible_text` with no `ready_calls`. This keeps the call site uniform for non-tool-calling models. +/// When either marker is empty, the accumulator degrades to a passthrough and returns its input as a text event. +/// This keeps the call site uniform for non-tool-calling models. /// /// Callers must not feed REASONING-tagged content into `Push` — reasoning is the model's scratchpad and any /// tool-call-shaped text inside `...` is not a real tool call. The upstream `ReasoningStreamSplitter` /// already routes REASONING segments through a separate path; this accumulator sits below the DEFAULT-segment branch. class ToolCallStreamAccumulator { public: + using Event = std::variant; + struct Output { + std::vector events; std::string visible_text; std::vector ready_calls; }; @@ -50,7 +54,7 @@ class ToolCallStreamAccumulator { end_marker_(std::move(end_marker)), tools_json_(std::move(tools_json)) {} - /// Feed a chunk into the accumulator. Returns visible text and any tool calls completed by this chunk. + /// Feed a chunk into the accumulator. Returns ordered visible-text and completed-tool-call events. Output Push(const std::string& chunk) { Output out; @@ -60,7 +64,7 @@ class ToolCallStreamAccumulator { if (start_marker_.empty() || end_marker_.empty()) { // Passthrough mode — no tool-call detection. - out.visible_text = chunk; + EmitVisible(out, chunk); return out; } @@ -70,8 +74,8 @@ class ToolCallStreamAccumulator { return out; } - /// Drain at end-of-stream. A complete or narrowly repairable JSON call is recovered even if the model omitted the - /// closing marker; genuinely truncated blocks become visible text. + /// Drain at end-of-stream. An unterminated tool-call block becomes visible text — it turned out not to be a real + /// tool call (no closing marker arrived), so the caller still sees what the model produced. Output Flush() { Output out; @@ -88,6 +92,25 @@ class ToolCallStreamAccumulator { bool InsideToolCall() const noexcept { return inside_tool_call_; } private: + static void EmitVisible(Output& out, std::string text) { + if (text.empty()) { + return; + } + out.visible_text += text; + if (!out.events.empty()) { + if (auto* previous = std::get_if(&out.events.back())) { + *previous += text; + return; + } + } + out.events.emplace_back(std::move(text)); + } + + static void EmitToolCall(Output& out, ParsedToolCall parsed_call) { + out.ready_calls.push_back(parsed_call); + out.events.emplace_back(std::move(parsed_call)); + } + void Drain(Output& out, bool flushing) { while (true) { const std::string& marker = inside_tool_call_ ? end_marker_ : start_marker_; @@ -102,8 +125,14 @@ class ToolCallStreamAccumulator { buffer_.erase(0, found + marker.size()); auto parsed = ParseToolCalls(tool_call_buffer_, start_marker_, end_marker_, tools_json_); - for (auto& pc : parsed) { - out.ready_calls.push_back(std::move(pc)); + if (parsed.empty()) { + // A marker-shaped block that cannot be parsed is model text, not a tool call. Preserve it rather than + // silently dropping generated output. + EmitVisible(out, tool_call_buffer_); + } else { + for (auto& pc : parsed) { + EmitToolCall(out, std::move(pc)); + } } tool_call_buffer_.clear(); @@ -112,7 +141,7 @@ class ToolCallStreamAccumulator { // Opening marker: emit prefix as visible text, then start buffering the tool-call block (including the // marker — ParseToolCalls expects the full `...` substring). if (found > 0) { - out.visible_text.append(buffer_, 0, found); + EmitVisible(out, buffer_.substr(0, found)); } tool_call_buffer_ = buffer_.substr(found, marker.size()); buffer_.erase(0, found + marker.size()); @@ -134,16 +163,16 @@ class ToolCallStreamAccumulator { candidate += end_marker_; auto parsed = ParseToolCalls(candidate, start_marker_, end_marker_, tools_json_); if (parsed.empty()) { - out.visible_text += incomplete; + EmitVisible(out, std::move(incomplete)); } else { - for (auto& pc : parsed) { - out.ready_calls.push_back(std::move(pc)); + for (auto& parsed_call : parsed) { + EmitToolCall(out, std::move(parsed_call)); } } tool_call_buffer_.clear(); inside_tool_call_ = false; } else { - out.visible_text.append(buffer_); + EmitVisible(out, buffer_); } buffer_.clear(); return; @@ -165,7 +194,7 @@ class ToolCallStreamAccumulator { size_t safe = buffer_.size() - hold; if (safe > 0) { - out.visible_text.append(buffer_, 0, safe); + EmitVisible(out, buffer_.substr(0, safe)); buffer_.erase(0, safe); } } diff --git a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_utils.cc b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_utils.cc index cdd35c6d5..daac57011 100644 --- a/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_utils.cc +++ b/sdk_v2/cpp/src/inferencing/generative/toolcalling/tool_call_utils.cc @@ -220,7 +220,7 @@ std::vector DeserializeToolCalls(const std::string& json_text, auto parse_one = [&](const nlohmann::json& call) { if (!call.is_object()) { - return; + return false; } ParsedToolCall tc; @@ -232,19 +232,29 @@ std::vector DeserializeToolCalls(const std::string& json_text, tc.name = call["function"].get(); } else if (call.size() == 1) { const auto& [name, arguments] = *call.items().begin(); + if (name == "name" || name == "function" || name == "arguments" || + name == "parameters" || name == "args") { + return false; + } tc.name = NormalizeToolName(name, advertised_tools); + if (tc.name.empty()) { + return false; + } if (name == "cmd" && tc.name == "shell") { tc.arguments = nlohmann::json({{"cmd", arguments}}).dump(); } else { tc.arguments = arguments.is_string() ? arguments.get() : arguments.dump(); } results.push_back(std::move(tc)); - return; + return true; } else { - return; + return false; } tc.name = NormalizeToolName(std::move(tc.name), advertised_tools); + if (tc.name.empty()) { + return false; + } // Arguments can be under "arguments", "parameters", or "args". if (call.contains("arguments")) { @@ -275,17 +285,23 @@ std::vector DeserializeToolCalls(const std::string& json_text, } results.push_back(std::move(tc)); + return true; }; if (json.is_array()) { for (const auto& item : json) { - parse_one(item); + if (!parse_one(item)) { + results.clear(); + return results; + } } } else if (json.is_object()) { - parse_one(json); + if (!parse_one(json)) { + results.clear(); + } } } catch (const nlohmann::json::exception&) { - // Invalid tool-call shape — return whatever we have so far (may be empty) + results.clear(); } return results;