Integrate tool calling with Engine chat batching

docs/README.md

@@ -2,6 +2,7 @@
 
 Documentation for Foundry Local can be found in the following resources:
 
+- [MSBench Tool Calling Fixes](MSBench%20Tool%20Calling%20Fixes.md): Engineering notes for the Responses API and tool-calling changes that enabled a local Qwen model to produce SWE-bench patches through MSBench.
 - [Microsoft Learn](https://learn.microsoft.com/azure/foundry-local/): This is the official documentation for Foundry Local, providing comprehensive guides, tutorials, and reference materials to help you get started and make the most of Foundry Local.
 - SDK Reference:
     - [C# SDK Reference](../sdk/cs/README.md): This documentation provides detailed information about the C# SDK for Foundry Local, including API references, usage examples, and best practices for integrating Foundry Local into your applications.

sdk_v2/cpp/CMakeLists.txt

@@ -213,6 +213,8 @@ set(FOUNDRY_LOCAL_SOURCES
     src/inferencing/generative/audio/pcm_utils.cc
     src/inferencing/generative/embeddings/embeddings_session.cc
     src/inferencing/generative/chat/chat_generator.cc
+    src/inferencing/generative/chat/onnx_chat_engine.cc
+    src/inferencing/generative/chat/onnx_engine_chat_generator.cc
     src/inferencing/session/session.cc
     src/inferencing/session/session_manager.cc
     src/inferencing/generative/chat/chat_session.cc

sdk_v2/cpp/src/catalog/azure_model_catalog.cc

@@ -6,19 +6,87 @@
 #include "catalog/local_model_scanner.h"
 #include "model.h"
 #include "model_info.h"
+#include "utils.h"
 
 #include <foundry_local/foundry_local_c.h>
 #include <fmt/format.h>
+#include <nlohmann/json.hpp>
 
 #include <algorithm>
+#include <filesystem>
+#include <fstream>
 #include <iterator>
+#include <system_error>
 #include <unordered_set>
 #include <utility>
 
 namespace fl {
 
 namespace {
 
+// Merge selected fields from the model directory's inference_model.json into
+// a BYOM ModelInfo. The scanner only extracts the model name, so tool-calling
+// and reasoning tags declared by the model author would otherwise be dropped.
+void MergeInferenceModelJson(ModelInfo& info, const std::string& local_path) {
+  if (local_path.empty()) {
+    return;
+  }
+
+  auto json_path = std::filesystem::path(local_path) / "inference_model.json";
+  std::error_code ec;
+  if (!std::filesystem::exists(json_path, ec)) {
+    return;
+  }
+
+  try {
+    std::ifstream file(json_path);
+    if (!file.is_open()) {
+      return;
+    }
+    auto j = nlohmann::json::parse(file, /*cb=*/nullptr, /*allow_exceptions=*/false);
+    if (j.is_discarded() || !j.is_object()) {
+      return;
+    }
+
+    auto read_string = [&](const char* key, const char* prop) {
+      if (j.contains(key) && j[key].is_string()) {
+        info.string_properties[prop] = j[key].get<std::string>();
+      }
+    };
+    auto read_bool_as_int = [&](const char* key, const char* prop) {
+      if (j.contains(key) && j[key].is_boolean()) {
+        info.int_properties[prop] = j[key].get<bool>() ? 1 : 0;
+      }
+    };
+
+    read_bool_as_int("supportsToolCalling", FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT);
+    read_bool_as_int("supportsReasoning", FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT);
+    read_string("toolCallStart", FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_START_STR);
+    read_string("toolCallEnd", FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_END_STR);
+    read_string("reasoningStart", FOUNDRY_LOCAL_MODEL_PROP_REASONING_START_STR);
+    read_string("reasoningEnd", FOUNDRY_LOCAL_MODEL_PROP_REASONING_END_STR);
+  } catch (...) {
+    // inference_model.json is best-effort metadata; ignore parse failures.
+  }
+}
+
+ModelInfo MakeByomModelInfo(const std::string& model_id, const std::string& local_path) {
+  auto [name, version] = Utils::SplitModelNameAndVersion(model_id);
+
+  ModelInfo info;
+  info.model_id = model_id;
+  info.name = name;
+  info.alias = name;
+  info.uri = "local://" + name;
+  info.version = version;
+  info.string_properties[FOUNDRY_LOCAL_MODEL_PROP_MODEL_PROVIDER_STR] = "Local";
+  info.string_properties[FOUNDRY_LOCAL_MODEL_PROP_MODEL_TYPE_STR] = "ONNX";
+
+  MergeInferenceModelJson(info, local_path);
+
+  return info;
+}
+
 std::vector<ModelInfo> DeduplicateByModelId(std::vector<ModelInfo> model_infos) {
   std::vector<ModelInfo> deduplicated;
   deduplicated.reserve(model_infos.size());
@@ -33,13 +101,6 @@ std::vector<ModelInfo> DeduplicateByModelId(std::vector<ModelInfo> model_infos)
   return deduplicated;
 }
 
-void RemoveLegacyLocalEntries(std::vector<ModelInfo>& model_infos) {
-  std::erase_if(model_infos, [](const auto& info) {
-    const auto* provider = info.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_MODEL_PROVIDER_STR);
-    return provider && *provider == "Local";
-  });
-}
-
 }  // namespace
 
 AzureModelCatalog::AzureModelCatalog(std::vector<std::pair<std::string, std::optional<std::string>>> catalog_urls,
@@ -107,26 +168,37 @@ AzureModelCatalog::CatalogResult AzureModelCatalog::GetLiveCatalogOrLocalSnapsho
   CatalogCache cache(cache_dir_, logger_);
   cache.Load();
   auto cached = cache.GetCachedModels();
-  auto snapshot_model_infos = cached ? std::move(*cached) : std::vector<ModelInfo>{};
-  RemoveLegacyLocalEntries(snapshot_model_infos);
 
   return {
-      .model_infos = DeduplicateByModelId(std::move(snapshot_model_infos)),
+      .model_infos = cached ? DeduplicateByModelId(std::move(*cached)) : std::vector<ModelInfo>{},
       .source = CatalogSource::kSnapshot,
   };
 }
 
-std::vector<Model> AzureModelCatalog::CreateModelsWithLocalPaths(const std::vector<ModelInfo>& model_infos,
-                                                                const LocalModels& local_models) const {
+std::vector<Model> AzureModelCatalog::AddLocalModels(std::vector<ModelInfo>& model_infos,
+                                                     const LocalModels& local_models) const {
   std::vector<Model> models;
-  models.reserve(model_infos.size());
+  models.reserve(model_infos.size() + local_models.size());
 
+  std::unordered_set<std::string> model_ids;
+  model_ids.reserve(model_infos.size() + local_models.size());
   for (const auto& info : model_infos) {
+    model_ids.insert(info.model_id);
+
     auto local_model = local_models.find(info.model_id);
     auto local_path = local_model != local_models.end() ? local_model->second : std::string{};
     models.push_back(model_factory_(ModelInfo(info), std::move(local_path)));
   }
 
+  for (const auto& [model_id, local_path] : local_models) {
+    if (!model_ids.insert(model_id).second) {
+      continue;
+    }
+
+    model_infos.push_back(MakeByomModelInfo(model_id, local_path));
+    models.push_back(model_factory_(ModelInfo(model_infos.back()), local_path));
+  }
+
   return models;
 }
 
@@ -143,7 +215,7 @@ std::vector<Model> AzureModelCatalog::FetchModels() const {
   logger_.Log(LogLevel::Information, fmt::format("Found {} locally cached models.", cached_model_ids.size()));
 
   auto catalog_result = GetLiveCatalogOrLocalSnapshot(cached_model_ids);
-  auto models = CreateModelsWithLocalPaths(catalog_result.model_infos, local_models);
+  auto models = AddLocalModels(catalog_result.model_infos, local_models);
 
   logger_.Log(LogLevel::Information, fmt::format("Populated model info for {} models.", models.size()));
 

sdk_v2/cpp/src/catalog/azure_model_catalog.h

@@ -60,8 +60,8 @@ class AzureModelCatalog : public BaseModelCatalog {
   static constexpr const char* kDefaultCatalogFilter = "''";
 
   CatalogResult GetLiveCatalogOrLocalSnapshot(const std::vector<std::string>& cached_model_ids) const;
-  std::vector<Model> CreateModelsWithLocalPaths(const std::vector<ModelInfo>& model_infos,
-                                                const LocalModels& local_models) const;
+  std::vector<Model> AddLocalModels(std::vector<ModelInfo>& model_infos,
+                                    const LocalModels& local_models) const;
 
   std::vector<std::pair<std::string, std::optional<std::string>>> catalog_urls_;
   std::string cache_dir_;

sdk_v2/cpp/src/contracts/responses.h

@@ -74,7 +74,15 @@ struct FunctionCallResultInputItem {
   std::string output;
 };
 
-using InputItem = std::variant<InputMessage, FunctionCallResultInputItem>;
+struct FunctionCallInputItem {
+  std::string type = "function_call";
+  std::string call_id;
+  std::string name;
+  std::string arguments;
+};
+
+using InputItem = std::variant<InputMessage, FunctionCallInputItem,
+                               FunctionCallResultInputItem>;
 
 // ---------------------------------------------------------------------------
 // Tool calling types (AD-010)
@@ -336,6 +344,7 @@ void from_json(const nlohmann::json& j, InputImageContent& c);
 void from_json(const nlohmann::json& j, InputFileContent& c);
 void from_json(const nlohmann::json& j, InputAudioContent& c);
 void from_json(const nlohmann::json& j, InputMessage& m);
+void from_json(const nlohmann::json& j, FunctionCallInputItem& f);
 void from_json(const nlohmann::json& j, FunctionCallResultInputItem& f);
 
 // --- Tool types from_json ---

sdk_v2/cpp/src/contracts/responses_json.cc

@@ -164,6 +164,13 @@ void from_json(const nlohmann::json& j, FunctionCallResultInputItem& f) {
   f.output = j.at("output").get<std::string>();
 }
 
+void from_json(const nlohmann::json& j, FunctionCallInputItem& f) {
+  f.type = j.value("type", "function_call");
+  f.call_id = j.at("call_id").get<std::string>();
+  f.name = j.at("name").get<std::string>();
+  f.arguments = j.at("arguments").get<std::string>();
+}
+
 // ========================================================================
 // Tool types from_json
 // ========================================================================
@@ -248,7 +255,9 @@ void from_json(const nlohmann::json& j, ResponseCreateParams& p) {
       for (const auto& entry : input) {
         std::string type = entry.value("type", "");
 
-        if (type == "function_call_output") {
+        if (type == "function_call") {
+          items.push_back(entry.get<FunctionCallInputItem>());
+        } else if (type == "function_call_output") {
           items.push_back(entry.get<FunctionCallResultInputItem>());
         } else {
           // Default: message item

sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.cc

@@ -4,6 +4,10 @@
 
 namespace fl {
 
+std::optional<ChatTurnUsage> ChatGenerator::GetTurnUsage() const {
+  return std::nullopt;
+}
+
 std::string ChatGenerator::GenerateAll() {
   std::string result;
 

sdk_v2/cpp/src/inferencing/generative/chat/chat_generator.h

@@ -5,9 +5,19 @@
 #include <cstdint>
 #include <optional>
 #include <string>
+#include <vector>
 
 namespace fl {
 
+class GenAIModelInstance;
+struct MessageItem;
+struct SearchOptions;
+
+struct ChatTurnUsage {
+  int prompt_tokens = 0;
+  int generated_tokens = 0;
+};
+
 /// Abstract interface for token-by-token text generation.
 /// One generator per request — not reusable, not thread-safe.
 /// Follows the classic pull-based iterator pattern:
@@ -32,6 +42,9 @@ class ChatGenerator {
   /// Get the most recently generated token ID before Decode consumes it.
   virtual std::optional<int32_t> CurrentTokenId() const = 0;
 
+  /// Return whether the rendered prompt ends with an open reasoning marker.
+  virtual bool PromptEndsInReasoning() const = 0;
+
   /// Get the total number of tokens (input + generated) so far.
   virtual int TokenCount() const = 0;
 
@@ -46,6 +59,22 @@ class ChatGenerator {
   /// After cancellation, IsDone() should return true on the next check.
   virtual void Cancel() = 0;
 
+  /// Append a new conversational turn to retained model state.
+  virtual int AppendMessages(const std::vector<MessageItem>& new_messages,
+                             GenAIModelInstance& model,
+                             const std::string& tools_json,
+                             const SearchOptions& options,
+                             const std::string& reasoning_start = {}) = 0;
+
+  /// Returns whether this backend can rewind retained model state directly.
+  virtual bool CanRewind() const = 0;
+
+  /// Rewind retained model state to a prior token position.
+  virtual void RewindTo(int token_count) = 0;
+
+  /// Return exact usage for the most recently completed turn when the backend exposes it.
+  virtual std::optional<ChatTurnUsage> GetTurnUsage() const;
+
  protected:
   ChatGenerator() = default;
 };