From 12e97169eaad49c29e8768a6bda91eba3ca5efe2 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Wed, 4 Nov 2026 10:50:55 -0500 Subject: [PATCH 1/8] feat(ai): limit GLM tool-calling turns to 10 A LanguageModelSession keeps calling tools for as long as the model returns function calls. With `toolCallingMode: .required`, or with a response schema, gemini-3.5-flash-lite can keep calling the same tool indefinitely, which runs up charges and hangs CI. - Count the tool-calling turns since the most recent prompt. After 10, send the next request with function calling mode `NONE`. - Gemini doesn't always follow `NONE`, so if it returns a function call anyway, throw `GeminiLanguageModel.Error.invalidResponse` instead of passing the call to the session. - Document the limit on `GeminiLanguageModel.capabilities` and in the changelog. The limit isn't configurable yet, since that needs a public API review. # Conflicts: # FirebaseAI/CHANGELOG.md --- FirebaseAI/CHANGELOG.md | 6 + .../GeminiLanguageModel.swift | 17 +++ .../GeminiLanguageModelExecutor.swift | 40 +++++- .../GeminiRequestTranslator.swift | 48 ++++++- .../GeminiLanguageModelTests.swift | 119 ++++++++++++++++++ .../GeminiRequestTranslatorTests.swift | 100 +++++++++++++++ .../ToolCallingIntegrationTests.swift | 57 +++++++++ 7 files changed, 380 insertions(+), 7 deletions(-) diff --git a/FirebaseAI/CHANGELOG.md b/FirebaseAI/CHANGELOG.md index 87e1d5869cf..1405e48338b 100644 --- a/FirebaseAI/CHANGELOG.md +++ b/FirebaseAI/CHANGELOG.md @@ -8,6 +8,12 @@ `start` and `end`, removed `VoiceActivityType.unknown` from the public API, and made `LiveServerVoiceActivity.audioOffset` a non-optional `TimeInterval` (defaulting to `0`). +- [changed] `GeminiLanguageModel` now allows at most 10 tool-calling turns in + response to each prompt, so that a session can't keep calling tools + indefinitely. After 10 turns, the next request asks Gemini for a final + response; if Gemini returns a function call anyway, the session throws + `GeminiLanguageModel.Error.invalidResponse`. The limit also applies when the + tool calling mode is `required`. - [fixed] Fixed a crash at launch on iOS 17 and earlier, and the corresponding macOS, tvOS, watchOS, and visionOS versions, in apps built with Xcode 27 when the app or a library (such as Realm) enumerated Objective-C classes with diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift index 2ec41890b80..ade8bebc071 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift @@ -107,6 +107,23 @@ /// images in the session are re-sent with every request. /// - Image files are read again for every request in the session, so a file must remain /// available for as long as the session that references it is used. + /// + /// ### Tool calling + /// + /// With the `toolCalling` capability, the model can call the tools that you provide to the + /// session. To limit cost when the model keeps calling tools, the model can make at most 10 + /// tool-calling turns in response to each prompt. A tool-calling turn is one response that + /// calls one or more tools. + /// + /// - After 10 tool-calling turns, the next request tells Gemini not to call tools, so that + /// the model gives a final response. + /// - If Gemini returns a function call anyway, the session throws + /// ``GeminiLanguageModel/Error/invalidResponse(_:)``. + /// + /// The limit also applies when the tool calling mode is `required`, which otherwise keeps + /// calling tools until a tool throws an error or the mode changes. To stop sooner, change the + /// tool calling mode from a `LanguageModelSession.DynamicProfile`, for example after the first + /// tool call. public var capabilities: LanguageModelCapabilities { LanguageModelCapabilities([ .guidedGeneration, diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift index f5446a27bfa..f68418e9c43 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift @@ -46,6 +46,17 @@ /// The `URLSessionConfiguration` to use. let sessionConfiguration: URLSessionConfiguration + /// The maximum number of tool-calling turns allowed in response to a single prompt. + /// + /// A tool-calling turn is a model response that calls one or more tools. When this limit + /// is reached, the next request tells Gemini not to call functions; if it returns a + /// function call anyway, the executor throws `GeminiLanguageModel.Error.invalidResponse`. + let maximumToolCallingTurns: Int + + /// The default maximum number of tool-calling turns allowed in response to a single + /// prompt. + static let defaultMaximumToolCallingTurns = 10 + /// Initializes an executor configuration. /// /// - Parameters: @@ -53,16 +64,21 @@ /// - endpointConfiguration: The network endpoint configuration. /// - headerProvider: An optional async provider for dynamic headers. /// - sessionConfiguration: The `URLSessionConfiguration` to use. + /// - maximumToolCallingTurns: The maximum number of tool-calling turns allowed in + /// response to a single prompt. Must be greater than zero. init( modelResource: ModelResource, endpointConfiguration: EndpointConfiguration, headerProvider: HeaderProvider?, - sessionConfiguration: URLSessionConfiguration + sessionConfiguration: URLSessionConfiguration, + maximumToolCallingTurns: Int = Configuration.defaultMaximumToolCallingTurns ) { + precondition(maximumToolCallingTurns > 0, "maximumToolCallingTurns must be positive.") self.modelResource = modelResource self.endpointConfiguration = endpointConfiguration self.headerProvider = headerProvider self.sessionConfiguration = sessionConfiguration + self.maximumToolCallingTurns = maximumToolCallingTurns } } @@ -88,7 +104,15 @@ model: GeminiLanguageModel, streamingInto channel: LanguageModelExecutorGenerationChannel ) async throws { - let generateRequest = try GeminiRequestTranslator.translate(request) + // The session keeps calling tools for as long as the model returns function calls, so + // limit the number of tool-calling turns to bound the number of requests. + let isAtToolCallingTurnLimit = + GeminiRequestTranslator.toolCallingTurns(in: request.transcript) + >= configuration.maximumToolCallingTurns + let generateRequest = try GeminiRequestTranslator.translate( + request, + isAtToolCallingTurnLimit: isAtToolCallingTurnLimit + ) let client = GeminiAPIClient( modelResource: configuration.modelResource, @@ -167,6 +191,18 @@ } if case .functionCall(let call) = part.data { + // Gemini doesn't always follow a function calling mode of `NONE`, so never pass a + // function call to the session once the limit is reached. + guard !isAtToolCallingTurnLimit else { + throw GeminiLanguageModel.Error.invalidResponse( + GeminiLanguageModel.Error.InvalidResponse( + debugDescription: """ + Gemini returned a function call after the maximum of \ + \(configuration.maximumToolCallingTurns) tool-calling turns was reached. + """ + ) + ) + } guard let name = call.name else { throw GeminiLanguageModel.Error.invalidResponse( GeminiLanguageModel.Error.InvalidResponse( diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift index ac6be97891a..4fa8c6829ca 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift @@ -27,12 +27,17 @@ enum GeminiRequestTranslator { /// Translates a `LanguageModelExecutorGenerationRequest` into a `GenerateContentRequest`. /// - /// - Parameter request: The generation request from the Foundation Models session. + /// - Parameters: + /// - request: The generation request from the Foundation Models session. + /// - isAtToolCallingTurnLimit: Whether the session has reached the maximum number of + /// tool-calling turns for the current prompt. If `true`, the request tells Gemini not to + /// call functions. /// - Returns: A `GenerateContentRequest` configured for the Gemini API. /// - Throws: An error if transcript or schema translation fails, or if the request specifies /// an unrecognized tool calling mode. static func translate( - _ request: LanguageModelExecutorGenerationRequest + _ request: LanguageModelExecutorGenerationRequest, + isAtToolCallingTurnLimit: Bool = false ) throws -> GenerateContentRequest { let (contents, systemInstruction) = try GeminiTranscriptTranslator.translate( request.transcript @@ -47,12 +52,36 @@ $0.tools = tools $0.toolConfig = try translateToolConfig( toolCallingMode: request.generationOptions.toolCallingMode, - hasFunctionDeclarations: hasFunctionDeclarations + hasFunctionDeclarations: hasFunctionDeclarations, + isAtToolCallingTurnLimit: isAtToolCallingTurnLimit ) $0.generationConfig = try translateGenerationConfig(schema: request.schema) } } + /// Returns the number of tool-calling turns since the most recent prompt in a transcript. + /// + /// A tool-calling turn is a model response that calls one or more tools. The session records + /// each one as a single `Transcript.Entry.toolCalls` entry, so parallel tool calls count as + /// one turn. Each prompt starts a new count. + /// + /// - Parameter transcript: The transcript of the session. + /// - Returns: The number of tool-calling turns since the most recent prompt. + static func toolCallingTurns(in transcript: Transcript) -> Int { + var turns = 0 + for entry in transcript { + switch entry { + case .prompt: + turns = 0 + case .toolCalls(let toolCalls) where !toolCalls.isEmpty: + turns += 1 + default: + break + } + } + return turns + } + /// Translates an optional `GenerationSchema` into a Gemini `GenerationConfig`. /// /// - Parameter schema: An optional generation schema specifying structured output constraints. @@ -97,12 +126,16 @@ /// - hasFunctionDeclarations: Whether the request declares any functions. /// `functionCallingConfig` only governs function calling, so it is omitted entirely when /// there are none. + /// - isAtToolCallingTurnLimit: Whether the session has reached the maximum number of + /// tool-calling turns for the current prompt. If `true`, function calling is disabled + /// whatever `toolCallingMode` is, so that the model gives a final response. /// - Returns: A `ToolConfig`, or `nil` if no configuration is needed. /// - Throws: `LanguageModelError.unsupportedCapability` if `toolCallingMode` is a mode that /// this version of the SDK does not recognize. static func translateToolConfig( toolCallingMode: GenerationOptions.ToolCallingMode?, - hasFunctionDeclarations: Bool + hasFunctionDeclarations: Bool, + isAtToolCallingTurnLimit: Bool = false ) throws -> ToolConfig? { // Computed independently of the `ToolConfig` check below to allow future options (such // as `includeServerSideToolInvocations`) when built-in tools are present without function @@ -114,7 +147,12 @@ // `AUTO`, so `nil` maps to the allowed mode. let allowedMode = FunctionCallingConfig.Mode.validated let callingMode: FunctionCallingConfig.Mode - if let mode = toolCallingMode { + if isAtToolCallingTurnLimit { + // Applies to every mode, including `required`, which would otherwise keep calling tools + // indefinitely. Written out in full because `mode` is optional, so `.none` would mean + // `nil`. + callingMode = FunctionCallingConfig.Mode.none + } else if let mode = toolCallingMode { switch mode.kind { case .allowed: callingMode = allowedMode diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift index 2e77766fd4e..3848278ba7f 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift @@ -878,8 +878,127 @@ #expect(fr.response?["result"] == .string("12:00 PM")) } + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func sessionRespondDisablesFunctionCallingAtToolCallingTurnLimit() async throws { + defer { MockHTTPURLProtocol.reset() } + let maximumTurns = GeminiLanguageModel.Executor.Configuration.defaultMaximumToolCallingTurns + let recordedRequests = try Self.mockToolCallingLoop(functionCallResponseCount: maximumTurns) + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + + let response = try await session.respond( + to: "What is the weather in Paris?", + options: GenerationOptions(toolCallingMode: .required) + ) + + #expect(maximumTurns == 10) + #expect(response.content == Self.weatherResponseText) + let requests = recordedRequests.all + #expect(requests.count == maximumTurns + 1) + #expect(requests.dropLast().allSatisfy { $0.toolConfig?.functionCallingConfig?.mode == .any }) + #expect( + requests.last?.toolConfig?.functionCallingConfig?.mode == FunctionCallingConfig.Mode.none) + #expect(requests.last?.tools?.first?.functionDeclarations?.first?.name == "get_weather") + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func functionCallAfterToolCallingTurnLimitThrowsInvalidResponse() async throws { + defer { MockHTTPURLProtocol.reset() } + let maximumTurns = GeminiLanguageModel.Executor.Configuration.defaultMaximumToolCallingTurns + let recordedRequests = try Self.mockToolCallingLoop(functionCallResponseCount: .max) + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + + do { + _ = try await session.respond(to: "What is the weather in Paris?") + Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + #expect(invalidResponse.debugDescription.contains("maximum of 10 tool-calling turns")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + + #expect(recordedRequests.all.count == maximumTurns + 1) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func newPromptResetsToolCallingTurnCount() async throws { + defer { MockHTTPURLProtocol.reset() } + let maximumTurns = GeminiLanguageModel.Executor.Configuration.defaultMaximumToolCallingTurns + let recordedRequests = try Self.mockToolCallingLoop(functionCallResponseCount: maximumTurns) + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + _ = try await session.respond(to: "What is the weather in Paris?") + + let response = try await session.respond(to: "Thanks! Anything else?") + + #expect(response.content == Self.weatherResponseText) + let requests = recordedRequests.all + #expect(requests.count == maximumTurns + 2) + #expect(requests.last?.toolConfig?.functionCallingConfig?.mode == .validated) + } + // MARK: - Helper Methods + /// The text of the final response returned by `mockToolCallingLoop(functionCallResponseCount:)`. + private static let weatherResponseText = "The weather in Paris is sunny, 72°F." + + /// Records the Gemini requests intercepted by `MockHTTPURLProtocol`. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + private final class RecordedRequests: Sendable { + private let requests = Mutex<[GenerateContentRequest]>([]) + + /// The recorded requests, in the order they were sent. + var all: [GenerateContentRequest] { requests.withLock { $0 } } + + /// Records a request and returns the number of requests recorded so far. + /// + /// - Parameter request: The request to record. + /// - Returns: The number of requests recorded, including `request`. + func append(_ request: GenerateContentRequest) -> Int { + requests.withLock { + $0.append(request) + return $0.count + } + } + } + + /// Mocks a model that calls `MockWeatherTool` and then responds with text. + /// + /// - Parameter functionCallResponseCount: The number of requests that receive a function call + /// in response; later requests receive `weatherResponseText`. + /// - Returns: The requests that the mock receives. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + private static func mockToolCallingLoop( + functionCallResponseCount: Int + ) throws -> RecordedRequests { + let expectedURL = try makeExpectedStreamURL() + let httpResponse = try HTTPURLResponse.mock( + url: expectedURL, + headerFields: ["Content-Type": "text/event-stream"] + ) + let functionCallSSE = """ + data: {"candidates": [{"content": {"parts": [{"functionCall": {"id": "call-1", "name": "get_weather", "args": {"location": "Paris"}}}], "role": "model"}, "finishReason": "STOP", "index": 0}]} + + """ + let textSSE = """ + data: {"candidates": [{"content": {"parts": [{"text": "\(weatherResponseText)"}], "role": "model"}, "finishReason": "STOP", "index": 0}]} + + """ + let recordedRequests = RecordedRequests() + MockHTTPURLProtocol.setHandler(for: expectedURL) { request, proto in + let body = try #require(request.httpBodyData) + let requestNumber = recordedRequests.append( + try JSONDecoder().decode(GenerateContentRequest.self, from: body) + ) + let payload = requestNumber <= functionCallResponseCount ? functionCallSSE : textSSE + proto.client?.urlProtocol(proto, didReceive: httpResponse, cacheStoragePolicy: .notAllowed) + proto.client?.urlProtocol(proto, didLoad: Data(payload.utf8)) + proto.client?.urlProtocolDidFinishLoading(proto) + } + return recordedRequests + } + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) private static func makeMockModel( diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiRequestTranslatorTests.swift b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiRequestTranslatorTests.swift index 5e861ed975c..49012c8410d 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiRequestTranslatorTests.swift +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiRequestTranslatorTests.swift @@ -338,5 +338,105 @@ let functionCallingConfig = try #require(toolConfig.functionCallingConfig) #expect(functionCallingConfig.mode == FunctionCallingConfig.Mode.validated) } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func toolCallingTurnsCountsToolCallsSinceMostRecentPrompt() throws { + let arguments = try GeneratedContent(json: #"{"city": "Paris"}"#) + let parisCall = Transcript.ToolCall( + id: "call-1", toolName: "getWeather", arguments: arguments) + let londonCall = Transcript.ToolCall( + id: "call-2", toolName: "getWeather", arguments: arguments) + let tokyoCall = Transcript.ToolCall( + id: "call-3", toolName: "getWeather", arguments: arguments) + let transcript = Transcript( + entries: [ + .prompt(Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Paris?"))])), + .toolCalls(Transcript.ToolCalls(id: "calls-1", [parisCall])), + .toolOutput(Self.makeToolOutput(id: "call-1")), + .response( + Transcript.Response(segments: [.text(Transcript.TextSegment(content: "Sunny."))])), + .prompt(Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Others?"))])), + .toolCalls(Transcript.ToolCalls(id: "calls-2", [londonCall, tokyoCall])), + .toolOutput(Self.makeToolOutput(id: "call-2")), + .toolOutput(Self.makeToolOutput(id: "call-3")), + .toolCalls(Transcript.ToolCalls(id: "calls-3", [])), + .toolCalls(Transcript.ToolCalls(id: "calls-4", [parisCall])), + .toolOutput(Self.makeToolOutput(id: "call-1")), + ] + ) + + let turns = GeminiRequestTranslator.toolCallingTurns(in: transcript) + let emptyTranscriptTurns = GeminiRequestTranslator.toolCallingTurns( + in: Transcript(entries: [])) + + #expect(turns == 2) + #expect(emptyTranscriptTurns == 0) + } + + @Test(arguments: [ + GenerationOptions.ToolCallingMode?.none, + .allowed, + .required, + .disallowed, + ]) + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesToolConfigAtToolCallingTurnLimitToNone( + toolCallingMode: GenerationOptions.ToolCallingMode? + ) throws { + let encoder = JSONEncoder() + encoder.outputFormatting = .sortedKeys + + let toolConfig = try GeminiRequestTranslator.translateToolConfig( + toolCallingMode: toolCallingMode, + hasFunctionDeclarations: true, + isAtToolCallingTurnLimit: true + ) + + let mode = try #require(toolConfig?.functionCallingConfig?.mode) + #expect(mode == FunctionCallingConfig.Mode.none) + let json = String(decoding: try encoder.encode(toolConfig), as: UTF8.self) + #expect(json == #"{"functionCallingConfig":{"mode":"NONE"}}"#) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesRequestAtToolCallingTurnLimitKeepsToolDeclarations() throws { + let toolDefinition = Transcript.ToolDefinition( + name: "get_current_weather", + description: "Get the current weather for a city.", + parameters: LocationArguments.generationSchema + ) + let request = LanguageModelExecutorGenerationRequest( + id: UUID(), + transcript: Transcript( + entries: [ + .prompt(Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Paris?"))])) + ] + ), + enabledTools: [toolDefinition], + schema: nil, + generationOptions: GenerationOptions(toolCallingMode: .required), + contextOptions: ContextOptions(), + metadata: [:] + ) + + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: true) + + let functionDeclarations = try #require(result.tools?.first?.functionDeclarations) + #expect(functionDeclarations.map(\.name) == ["get_current_weather"]) + #expect(result.toolConfig?.functionCallingConfig?.mode == FunctionCallingConfig.Mode.none) + } + + // MARK: - Helper Methods + + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + private static func makeToolOutput(id: String) -> Transcript.ToolOutput { + Transcript.ToolOutput( + id: id, + toolName: "getWeather", + segments: [.text(Transcript.TextSegment(content: "22°C and sunny"))] + ) + } } #endif // compiler(>=6.4) && (FoundationModelsFake || canImport(FoundationModels)) diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ToolCallingIntegrationTests.swift b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ToolCallingIntegrationTests.swift index 01e0de73f82..a201881e1ef 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ToolCallingIntegrationTests.swift +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ToolCallingIntegrationTests.swift @@ -21,6 +21,7 @@ #endif import GeminiAPIClient import GeminiTestUtilities + import Synchronization import Testing @testable import GeminiLanguageModel @@ -274,5 +275,61 @@ #expect(hasToolOutput) #expect(response.usage.totalTokenCount > 0) } + + /// A weather tool that counts how many times the model calls it. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + @available(tvOS, unavailable) + struct CountingWeatherTool: FoundationModels.Tool { + /// The number of calls, shared by all copies of the tool. + final class CallCount: Sendable { + let value = Mutex(0) + } + + let name = "getWeather" + let description = "Get the current weather for a city." + let callCount = CallCount() + + @Generable + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + @available(tvOS, unavailable) + struct Arguments { + @Guide(description: "The city name to look up weather for") + var city: String + } + + func call(arguments: Arguments) async throws -> String { + callCount.value.withLock { $0 += 1 } + return "The current weather in \(arguments.city) is 22°C and sunny." + } + } + + @Test( + .requireIntegrationTestingBackend, + .timeLimit(.minutes(2)), + arguments: IntegrationTestingBackend.availableBackends + ) + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func sessionRespondRequiredToolCallingStopsAtToolCallingTurnLimit( + backend: IntegrationTestingBackend + ) async throws { + let maximumTurns = GeminiLanguageModel.Executor.Configuration.defaultMaximumToolCallingTurns + let model = try await backend.makeModel() + let tool = CountingWeatherTool() + let session = LanguageModelSession(model: model, tools: [tool]) + + // With `required`, the model calls a tool in every response until the limit is reached. + do { + let response = try await session.respond( + to: "What is the weather in Paris right now?", + options: GenerationOptions(toolCallingMode: .required) + ) + #expect(!response.content.isEmpty) + } catch GeminiLanguageModel.Error.invalidResponse { + // Gemini sometimes returns a function call after being told not to, which ends the turn + // with this error instead of a response. + } + + #expect(tool.callCount.value.withLock { $0 } == maximumTurns) + } } #endif // compiler(>=6.4) && (FoundationModelsFake || canImport(FoundationModels)) From dd5cfa90f149bf1129b08463917435bde41a0585 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Tue, 10 Nov 2026 16:52:52 -0500 Subject: [PATCH 2/8] test(ai): use FoundationModelsTool for CountingWeatherTool CountingWeatherTool conformed to FoundationModels.Tool, which doesn't exist when building with the FoundationModelsFake trait (the default on Linux), so the GeminiLanguageModelTests target failed to compile in the unit-tests-linux and unit-tests-macos-fake jobs. Use the FoundationModelsTool typealias, like the other test tools. --- .../IntegrationTests/ToolCallingIntegrationTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ToolCallingIntegrationTests.swift b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ToolCallingIntegrationTests.swift index a201881e1ef..b8e7a6ccf26 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ToolCallingIntegrationTests.swift +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ToolCallingIntegrationTests.swift @@ -279,7 +279,7 @@ /// A weather tool that counts how many times the model calls it. @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) @available(tvOS, unavailable) - struct CountingWeatherTool: FoundationModels.Tool { + struct CountingWeatherTool: FoundationModelsTool { /// The number of calls, shared by all copies of the tool. final class CallCount: Sendable { let value = Mutex(0) From 01f0e768f93c949acc1ca2e05a93b72c2c1d2f9e Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Tue, 10 Nov 2026 18:22:22 -0500 Subject: [PATCH 3/8] fix(ai): reject GLM function calls when function calling is disabled LanguageModelSession runs every tool call that the executor sends it, even when the tool calling mode is `disallowed` (tested on macOS 27.2). Gemini sometimes returns a function call anyway, including for tools that the request doesn't declare, so the session could run a tool that the app disallowed. Throw `GeminiLanguageModel.Error.invalidResponse` instead of passing a function call to the session when the tool calling mode is `disallowed`, when no tools are enabled, or when the tool-calling turn limit is reached, with a message for each case. Also document that the turn limit is counted from the transcript that the session sends to the model, so a `historyTransform(_:)` that removes earlier tool calls or prompts changes the count. --- FirebaseAI/CHANGELOG.md | 4 ++ .../GeminiLanguageModel.swift | 10 ++++ .../GeminiLanguageModelExecutor.swift | 52 ++++++++++++++++--- .../GeminiLanguageModelTests.swift | 43 ++++++++++++++- 4 files changed, 101 insertions(+), 8 deletions(-) diff --git a/FirebaseAI/CHANGELOG.md b/FirebaseAI/CHANGELOG.md index 1405e48338b..d45c99b1c27 100644 --- a/FirebaseAI/CHANGELOG.md +++ b/FirebaseAI/CHANGELOG.md @@ -14,6 +14,10 @@ response; if Gemini returns a function call anyway, the session throws `GeminiLanguageModel.Error.invalidResponse`. The limit also applies when the tool calling mode is `required`. +- [fixed] When the tool calling mode is `disallowed` or no tools are enabled, + `GeminiLanguageModel` now throws `GeminiLanguageModel.Error.invalidResponse` + if Gemini returns a function call, instead of running the tool or failing + with a parsing error. - [fixed] Fixed a crash at launch on iOS 17 and earlier, and the corresponding macOS, tvOS, watchOS, and visionOS versions, in apps built with Xcode 27 when the app or a library (such as Realm) enumerated Objective-C classes with diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift index ade8bebc071..3c2bad7b51b 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift @@ -124,6 +124,16 @@ /// calling tools until a tool throws an error or the mode changes. To stop sooner, change the /// tool calling mode from a `LanguageModelSession.DynamicProfile`, for example after the first /// tool call. + /// + /// The limit counts the tool-calling turns since the most recent prompt in the transcript that + /// the session sends to the model, which reflects any `historyTransform(_:)` from a + /// `LanguageModelSession.DynamicProfile`. A transform that removes earlier tool calls allows + /// more than 10 tool-calling turns per prompt, and a transform that removes prompts counts + /// tool-calling turns across prompts. + /// + /// If Gemini returns a function call when the tool calling mode is `disallowed` or when no + /// tools are enabled, the session also throws ``GeminiLanguageModel/Error/invalidResponse(_:)`` + /// instead of running the tool. public var capabilities: LanguageModelCapabilities { LanguageModelCapabilities([ .guidedGeneration, diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift index f68418e9c43..53207b1d9a7 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift @@ -109,6 +109,11 @@ let isAtToolCallingTurnLimit = GeminiRequestTranslator.toolCallingTurns(in: request.transcript) >= configuration.maximumToolCallingTurns + let functionCallRejectionReason = Self.functionCallRejectionReason( + for: request, + isAtToolCallingTurnLimit: isAtToolCallingTurnLimit, + maximumToolCallingTurns: configuration.maximumToolCallingTurns + ) let generateRequest = try GeminiRequestTranslator.translate( request, isAtToolCallingTurnLimit: isAtToolCallingTurnLimit @@ -191,15 +196,13 @@ } if case .functionCall(let call) = part.data { - // Gemini doesn't always follow a function calling mode of `NONE`, so never pass a - // function call to the session once the limit is reached. - guard !isAtToolCallingTurnLimit else { + // Gemini doesn't always follow a function calling mode of `NONE`, and the session + // runs every tool call that it receives, so never pass a function call to the + // session when function calling is disabled for this request. + if let functionCallRejectionReason { throw GeminiLanguageModel.Error.invalidResponse( GeminiLanguageModel.Error.InvalidResponse( - debugDescription: """ - Gemini returned a function call after the maximum of \ - \(configuration.maximumToolCallingTurns) tool-calling turns was reached. - """ + debugDescription: functionCallRejectionReason ) ) } @@ -268,6 +271,41 @@ throw GeminiErrorMapper.map(error) } } + + /// Returns why a function call in the response to a request must be rejected, or `nil` if + /// the request allows function calls. + /// + /// `LanguageModelSession` runs every tool call that the executor sends it, even when the + /// tool calling mode is `disallowed` (tested on macOS 27.2). Gemini sometimes returns a + /// function call when function calling is disabled, including for tools that the request + /// doesn't declare, so the executor rejects function calls in these cases instead. + /// + /// - Parameters: + /// - request: The generation request from the Foundation Models session. + /// - isAtToolCallingTurnLimit: Whether the session has reached the maximum number of + /// tool-calling turns for the current prompt. + /// - maximumToolCallingTurns: The maximum number of tool-calling turns per prompt. + /// - Returns: A description of why a function call must be rejected, or `nil` if function + /// calls are allowed. + static func functionCallRejectionReason( + for request: LanguageModelExecutorGenerationRequest, + isAtToolCallingTurnLimit: Bool, + maximumToolCallingTurns: Int + ) -> String? { + if request.generationOptions.toolCallingMode?.kind == .disallowed { + return "Gemini returned a function call when the tool calling mode was `disallowed`." + } + if request.enabledToolDefinitions.isEmpty { + return "Gemini returned a function call when no tools were enabled." + } + if isAtToolCallingTurnLimit { + return """ + Gemini returned a function call after the maximum of \(maximumToolCallingTurns) \ + tool-calling turns was reached. + """ + } + return nil + } } } #endif // compiler(>=6.4) && (FoundationModelsFake || canImport(FoundationModels)) diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift index 3848278ba7f..15fd2599613 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift @@ -691,7 +691,7 @@ proto.client?.urlProtocol(proto, didLoad: Data(ssePayload.utf8)) proto.client?.urlProtocolDidFinishLoading(proto) } - let session = LanguageModelSession(model: model) + let session = LanguageModelSession(model: model, tools: [MockWeatherTool()]) do { _ = try await session.respond(to: "What's the weather?") @@ -938,6 +938,47 @@ #expect(requests.last?.toolConfig?.functionCallingConfig?.mode == .validated) } + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func functionCallWithToolCallingModeDisallowedThrowsInvalidResponse() async throws { + defer { MockHTTPURLProtocol.reset() } + let recordedRequests = try Self.mockToolCallingLoop(functionCallResponseCount: .max) + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + + do { + _ = try await session.respond( + to: "What is the weather in Paris?", + options: GenerationOptions(toolCallingMode: .disallowed) + ) + Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + #expect(invalidResponse.debugDescription.contains("tool calling mode was `disallowed`")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + + #expect(recordedRequests.all.count == 1) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func functionCallWithoutToolsThrowsInvalidResponse() async throws { + defer { MockHTTPURLProtocol.reset() } + let recordedRequests = try Self.mockToolCallingLoop(functionCallResponseCount: .max) + let session = LanguageModelSession(model: Self.makeMockModel()) + + do { + _ = try await session.respond(to: "What is the weather in Paris?") + Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + #expect(invalidResponse.debugDescription.contains("no tools were enabled")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + + #expect(recordedRequests.all.count == 1) + } + // MARK: - Helper Methods /// The text of the final response returned by `mockToolCallingLoop(functionCallResponseCount:)`. From 619f12effe0548d5d13277de0f4c2c889a6a8f6d Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Tue, 10 Nov 2026 19:24:36 -0500 Subject: [PATCH 4/8] fix(ai): only pass GLM function calls to enabled tools LanguageModelSession runs any tool call that names one of the session's tools, without checking the tools enabled for the request. Check each function call against the request's enabled tools, in addition to the `disallowed` mode and the tool-calling turn limit, so that a call to a tool that isn't enabled throws `GeminiLanguageModel.Error.invalidResponse` instead of running the tool or throwing `GeneratedContent.ParsingError`. Validate every function call in a chunk before streaming any of its parts, so that a rejected call doesn't leave a partial response behind. With the `disallowed` mode, the session enables no tools, so a request after earlier tool calls sent function calls to undeclared functions. Declare the tools from the transcript's instructions with function calling mode `NONE` in that case, matching the final request at the turn limit. Also: - Validate an unknown tool calling mode before applying the turn limit. - Rename `toolCallingTurns(in:)` to `toolCallingTurnCount(in:)`, count from the end of the transcript, and remove the `= false` defaults. - Document the transcript reversion on errors, recommend switching from `required` to `allowed`, and clarify the `historyTransform` caveat. - Add PR numbers to the changelog entries. - Strengthen the tests: assert tool runs from the transcript, prove the turn count resets, cover parallel calls and the rejection order, add a time limit to the suite, and count tool-calling turns rather than tool calls in the integration test. --- FirebaseAI/CHANGELOG.md | 19 +- .../GeminiLanguageModel.swift | 33 ++- .../GeminiLanguageModelExecutor.swift | 124 ++++++--- .../GeminiRequestTranslator.swift | 97 +++++-- .../GeminiLanguageModelTests.swift | 262 ++++++++++++++++-- .../GeminiRequestTranslatorTests.swift | 205 ++++++++++++-- .../ToolCallingIntegrationTests.swift | 39 ++- 7 files changed, 630 insertions(+), 149 deletions(-) diff --git a/FirebaseAI/CHANGELOG.md b/FirebaseAI/CHANGELOG.md index d45c99b1c27..07ccab593ee 100644 --- a/FirebaseAI/CHANGELOG.md +++ b/FirebaseAI/CHANGELOG.md @@ -8,16 +8,15 @@ `start` and `end`, removed `VoiceActivityType.unknown` from the public API, and made `LiveServerVoiceActivity.audioOffset` a non-optional `TimeInterval` (defaulting to `0`). -- [changed] `GeminiLanguageModel` now allows at most 10 tool-calling turns in - response to each prompt, so that a session can't keep calling tools - indefinitely. After 10 turns, the next request asks Gemini for a final - response; if Gemini returns a function call anyway, the session throws - `GeminiLanguageModel.Error.invalidResponse`. The limit also applies when the - tool calling mode is `required`. -- [fixed] When the tool calling mode is `disallowed` or no tools are enabled, - `GeminiLanguageModel` now throws `GeminiLanguageModel.Error.invalidResponse` - if Gemini returns a function call, instead of running the tool or failing - with a parsing error. +- [changed] `GeminiLanguageModel` now limits each prompt to 10 tool-calling + turns, including when `toolCallingMode` is `.required`. After 10 turns, the + next request asks Gemini for a final response without calling tools; if + Gemini calls a tool anyway, the session throws + `GeminiLanguageModel.Error.invalidResponse`. (#17043) +- [fixed] Fixed `GeminiLanguageModel` running a tool, or throwing a + `GeneratedContent.ParsingError`, when Gemini called a tool that wasn't enabled + for the request, including when `toolCallingMode` was `.disallowed`. It now + throws `GeminiLanguageModel.Error.invalidResponse` instead. (#17043) - [fixed] Fixed a crash at launch on iOS 17 and earlier, and the corresponding macOS, tvOS, watchOS, and visionOS versions, in apps built with Xcode 27 when the app or a library (such as Realm) enumerated Objective-C classes with diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift index 3c2bad7b51b..5420ff75c03 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift @@ -117,23 +117,30 @@ /// /// - After 10 tool-calling turns, the next request tells Gemini not to call tools, so that /// the model gives a final response. - /// - If Gemini returns a function call anyway, the session throws + /// - If Gemini calls a tool anyway, the session throws /// ``GeminiLanguageModel/Error/invalidResponse(_:)``. /// - /// The limit also applies when the tool calling mode is `required`, which otherwise keeps - /// calling tools until a tool throws an error or the mode changes. To stop sooner, change the - /// tool calling mode from a `LanguageModelSession.DynamicProfile`, for example after the first - /// tool call. + /// The limit also applies when the tool calling mode is `required`. With `required`, the model + /// otherwise keeps calling tools until a tool throws an error or the mode changes. To stop + /// sooner, change the mode with the [`toolCallingMode(_:)`](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/dynamicprofile/toolcallingmode%28_%29) + /// modifier of a `LanguageModelSession.DynamicProfile`, for example from `required` to + /// `allowed` after the first tool call, as shown in [`GenerationOptions.ToolCallingMode`](https://developer.apple.com/documentation/foundationmodels/generationoptions/toolcallingmode-swift.struct). /// - /// The limit counts the tool-calling turns since the most recent prompt in the transcript that - /// the session sends to the model, which reflects any `historyTransform(_:)` from a - /// `LanguageModelSession.DynamicProfile`. A transform that removes earlier tool calls allows - /// more than 10 tool-calling turns per prompt, and a transform that removes prompts counts - /// tool-calling turns across prompts. + /// If Gemini calls a tool when the tool calling mode is `disallowed`, or calls a tool that + /// isn't enabled for the request, the session throws + /// ``GeminiLanguageModel/Error/invalidResponse(_:)`` and the tool doesn't run. Gemini doesn't + /// always follow `disallowed` after several tool calls in response to a prompt, so prefer + /// `allowed` when you change the mode after a tool call. /// - /// If Gemini returns a function call when the tool calling mode is `disallowed` or when no - /// tools are enabled, the session also throws ``GeminiLanguageModel/Error/invalidResponse(_:)`` - /// instead of running the tool. + /// By default, when the session throws an error, it reverts its transcript to how it was before + /// the prompt, even if tools have already run. To keep the tool calls and their output, set the + /// session's [`transcriptErrorHandlingPolicy`](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/transcripterrorhandlingpolicy) + /// to `preserveTranscript`. + /// + /// The limit is counted from the transcript that the session sends to the model, after any + /// `historyTransform(_:)` of a `LanguageModelSession.DynamicProfile`. A transform that removes + /// earlier tool calls can stop the limit from being reached, and a transform that removes + /// prompts can make the limit apply across prompts. public var capabilities: LanguageModelCapabilities { LanguageModelCapabilities([ .guidedGeneration, diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift index 53207b1d9a7..1c2f9adfc2a 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift @@ -107,13 +107,8 @@ // The session keeps calling tools for as long as the model returns function calls, so // limit the number of tool-calling turns to bound the number of requests. let isAtToolCallingTurnLimit = - GeminiRequestTranslator.toolCallingTurns(in: request.transcript) + GeminiRequestTranslator.toolCallingTurnCount(in: request.transcript) >= configuration.maximumToolCallingTurns - let functionCallRejectionReason = Self.functionCallRejectionReason( - for: request, - isAtToolCallingTurnLimit: isAtToolCallingTurnLimit, - maximumToolCallingTurns: configuration.maximumToolCallingTurns - ) let generateRequest = try GeminiRequestTranslator.translate( request, isAtToolCallingTurnLimit: isAtToolCallingTurnLimit @@ -160,6 +155,16 @@ if let candidates = chunk.candidates, let candidate = candidates.first, let parts = candidate.content?.parts { + // Check every function call in the chunk before streaming any of its parts, so that + // a rejected function call doesn't leave a partial response (such as its thought + // signature) in the transcript. + try Self.validateFunctionCalls( + in: parts, + for: request, + isAtToolCallingTurnLimit: isAtToolCallingTurnLimit, + maximumToolCallingTurns: configuration.maximumToolCallingTurns + ) + for part in parts { if let thoughtSignature = part.thoughtSignature, !thoughtSignature.isEmpty { let signatureData = Data(thoughtSignature.utf8) @@ -195,24 +200,9 @@ } } - if case .functionCall(let call) = part.data { - // Gemini doesn't always follow a function calling mode of `NONE`, and the session - // runs every tool call that it receives, so never pass a function call to the - // session when function calling is disabled for this request. - if let functionCallRejectionReason { - throw GeminiLanguageModel.Error.invalidResponse( - GeminiLanguageModel.Error.InvalidResponse( - debugDescription: functionCallRejectionReason - ) - ) - } - guard let name = call.name else { - throw GeminiLanguageModel.Error.invalidResponse( - GeminiLanguageModel.Error.InvalidResponse( - debugDescription: "Gemini returned a function call without a name." - ) - ) - } + // `validateFunctionCalls` has already checked that every function call has a name + // and is allowed for this request. + if case .functionCall(let call) = part.data, let name = call.name { let callID = call.id ?? UUID().uuidString let argsString: String if let args = call.args, !args.isEmpty { @@ -272,38 +262,88 @@ } } - /// Returns why a function call in the response to a request must be rejected, or `nil` if - /// the request allows function calls. + /// Checks that the function calls in a response can be passed to the session. /// - /// `LanguageModelSession` runs every tool call that the executor sends it, even when the - /// tool calling mode is `disallowed` (tested on macOS 27.2). Gemini sometimes returns a - /// function call when function calling is disabled, including for tools that the request - /// doesn't declare, so the executor rejects function calls in these cases instead. + /// - Parameters: + /// - parts: The parts of a response chunk from Gemini. + /// - request: The generation request from the Foundation Models session. + /// - isAtToolCallingTurnLimit: Whether the session has reached the maximum number of + /// tool-calling turns for the current prompt. + /// - maximumToolCallingTurns: The maximum number of tool-calling turns per prompt. + /// - Throws: `GeminiLanguageModel.Error.invalidResponse` if a function call has no name or + /// if the request doesn't allow it (see `functionCallRejectionReason`). + static func validateFunctionCalls( + in parts: [Part], + for request: LanguageModelExecutorGenerationRequest, + isAtToolCallingTurnLimit: Bool, + maximumToolCallingTurns: Int + ) throws { + for part in parts { + guard case .functionCall(let call) = part.data else { continue } + guard let name = call.name else { + throw GeminiLanguageModel.Error.invalidResponse( + GeminiLanguageModel.Error.InvalidResponse( + debugDescription: "Gemini returned a function call without a name." + ) + ) + } + if let reason = functionCallRejectionReason( + forFunctionNamed: name, + in: request, + isAtToolCallingTurnLimit: isAtToolCallingTurnLimit, + maximumToolCallingTurns: maximumToolCallingTurns + ) { + throw GeminiLanguageModel.Error.invalidResponse( + GeminiLanguageModel.Error.InvalidResponse(debugDescription: reason) + ) + } + } + } + + /// Returns why a function call in the response to a request must not be passed to the + /// session, or `nil` if it can be. + /// + /// `LanguageModelSession` runs any tool call that names one of the session's tools, even + /// when the tool calling mode is `disallowed` or the tool isn't enabled for the request + /// (tested on macOS 27.2). Gemini sometimes returns a function call when it's told not to + /// call functions, or for a function that the request doesn't declare, so the executor only + /// passes on function calls to the request's enabled tools while function calling is + /// enabled. /// /// - Parameters: + /// - name: The name of the function that Gemini called. /// - request: The generation request from the Foundation Models session. /// - isAtToolCallingTurnLimit: Whether the session has reached the maximum number of /// tool-calling turns for the current prompt. /// - maximumToolCallingTurns: The maximum number of tool-calling turns per prompt. - /// - Returns: A description of why a function call must be rejected, or `nil` if function - /// calls are allowed. + /// - Returns: A description of why the function call must not be passed to the session, or + /// `nil` if it can be. static func functionCallRejectionReason( - for request: LanguageModelExecutorGenerationRequest, + forFunctionNamed name: String, + in request: LanguageModelExecutorGenerationRequest, isAtToolCallingTurnLimit: Bool, maximumToolCallingTurns: Int ) -> String? { - if request.generationOptions.toolCallingMode?.kind == .disallowed { - return "Gemini returned a function call when the tool calling mode was `disallowed`." - } - if request.enabledToolDefinitions.isEmpty { - return "Gemini returned a function call when no tools were enabled." - } - if isAtToolCallingTurnLimit { + let toolCallingMode = request.generationOptions.toolCallingMode + if GeminiRequestTranslator.isFunctionCallingDisabled( + toolCallingMode: toolCallingMode, + isAtToolCallingTurnLimit: isAtToolCallingTurnLimit + ) { + if toolCallingMode?.kind == .disallowed { + return """ + Gemini returned a function call to `\(name)` when the tool calling mode was \ + `disallowed`. + """ + } return """ - Gemini returned a function call after the maximum of \(maximumToolCallingTurns) \ - tool-calling turns was reached. + Gemini returned a function call to `\(name)` after function calling was disabled \ + because the limit of \(maximumToolCallingTurns) tool-calling turns per prompt was \ + reached. """ } + if !request.enabledToolDefinitions.contains(where: { $0.name == name }) { + return "Gemini returned a function call to `\(name)`, which isn't an enabled tool." + } return nil } } diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift index 4fa8c6829ca..12aa96cf16b 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift @@ -37,12 +37,17 @@ /// an unrecognized tool calling mode. static func translate( _ request: LanguageModelExecutorGenerationRequest, - isAtToolCallingTurnLimit: Bool = false + isAtToolCallingTurnLimit: Bool ) throws -> GenerateContentRequest { let (contents, systemInstruction) = try GeminiTranscriptTranslator.translate( request.transcript ) - let tools = try translateTools(request.enabledToolDefinitions) + let tools = try translateTools( + functionDeclarationToolDefinitions( + for: request, + isAtToolCallingTurnLimit: isAtToolCallingTurnLimit + ) + ) let hasFunctionDeclarations = tools?.contains { $0.functionDeclarations?.isEmpty == false } ?? false @@ -59,6 +64,60 @@ } } + /// Returns whether a request must not call functions. + /// + /// Function calling is disabled when the tool calling mode is `disallowed` or when the session + /// has reached the maximum number of tool-calling turns for the current prompt. + /// + /// - Parameters: + /// - toolCallingMode: The tool calling mode from the request. + /// - isAtToolCallingTurnLimit: Whether the session has reached the maximum number of + /// tool-calling turns for the current prompt. + /// - Returns: `true` if the request must not call functions. + static func isFunctionCallingDisabled( + toolCallingMode: GenerationOptions.ToolCallingMode?, + isAtToolCallingTurnLimit: Bool + ) -> Bool { + isAtToolCallingTurnLimit || toolCallingMode?.kind == .disallowed + } + + /// Returns the tool definitions to declare as functions in the Gemini request. + /// + /// This is usually the request's enabled tools. With the `disallowed` tool calling mode, the + /// session enables no tools, so a transcript with earlier tool calls would refer to functions + /// that the request doesn't declare. In that case, this returns the tool definitions from the + /// transcript's instructions instead, and the request disables function calling, matching the + /// final request after the tool-calling turn limit is reached. + /// + /// - Parameters: + /// - request: The generation request from the Foundation Models session. + /// - isAtToolCallingTurnLimit: Whether the session has reached the maximum number of + /// tool-calling turns for the current prompt. + /// - Returns: The tool definitions to declare as functions. + static func functionDeclarationToolDefinitions( + for request: LanguageModelExecutorGenerationRequest, + isAtToolCallingTurnLimit: Bool + ) -> [Transcript.ToolDefinition] { + guard request.enabledToolDefinitions.isEmpty, + isFunctionCallingDisabled( + toolCallingMode: request.generationOptions.toolCallingMode, + isAtToolCallingTurnLimit: isAtToolCallingTurnLimit + ), + request.transcript.contains(where: { entry in + guard case .toolCalls(let toolCalls) = entry else { return false } + return !toolCalls.isEmpty + }) + else { + return request.enabledToolDefinitions + } + for entry in request.transcript.reversed() { + if case .instructions(let instructions) = entry, !instructions.toolDefinitions.isEmpty { + return instructions.toolDefinitions + } + } + return [] + } + /// Returns the number of tool-calling turns since the most recent prompt in a transcript. /// /// A tool-calling turn is a model response that calls one or more tools. The session records @@ -67,19 +126,19 @@ /// /// - Parameter transcript: The transcript of the session. /// - Returns: The number of tool-calling turns since the most recent prompt. - static func toolCallingTurns(in transcript: Transcript) -> Int { - var turns = 0 - for entry in transcript { + static func toolCallingTurnCount(in transcript: Transcript) -> Int { + var turnCount = 0 + for entry in transcript.reversed() { switch entry { case .prompt: - turns = 0 + return turnCount case .toolCalls(let toolCalls) where !toolCalls.isEmpty: - turns += 1 + turnCount += 1 default: break } } - return turns + return turnCount } /// Translates an optional `GenerationSchema` into a Gemini `GenerationConfig`. @@ -135,7 +194,7 @@ static func translateToolConfig( toolCallingMode: GenerationOptions.ToolCallingMode?, hasFunctionDeclarations: Bool, - isAtToolCallingTurnLimit: Bool = false + isAtToolCallingTurnLimit: Bool ) throws -> ToolConfig? { // Computed independently of the `ToolConfig` check below to allow future options (such // as `includeServerSideToolInvocations`) when built-in tools are present without function @@ -146,20 +205,15 @@ // caller. Omitting `functionCallingConfig` in that case causes the backend to default to // `AUTO`, so `nil` maps to the allowed mode. let allowedMode = FunctionCallingConfig.Mode.validated - let callingMode: FunctionCallingConfig.Mode - if isAtToolCallingTurnLimit { - // Applies to every mode, including `required`, which would otherwise keep calling tools - // indefinitely. Written out in full because `mode` is optional, so `.none` would mean - // `nil`. - callingMode = FunctionCallingConfig.Mode.none - } else if let mode = toolCallingMode { + let requestedMode: FunctionCallingConfig.Mode + if let mode = toolCallingMode { switch mode.kind { case .allowed: - callingMode = allowedMode + requestedMode = allowedMode case .required: - callingMode = .any + requestedMode = .any case .disallowed: - callingMode = FunctionCallingConfig.Mode.none + requestedMode = FunctionCallingConfig.Mode.none @unknown default: // A mode added in a later OS release; its semantics are unknown, so approximating it // with the allowed mode could silently diverge from what the caller requested. @@ -171,8 +225,11 @@ ) } } else { - callingMode = allowedMode + requestedMode = allowedMode } + // The limit overrides every mode, including `required`, which would otherwise keep + // calling tools indefinitely. + let callingMode = isAtToolCallingTurnLimit ? FunctionCallingConfig.Mode.none : requestedMode functionCallingConfig = FunctionCallingConfig { $0.mode = callingMode } } else { functionCallingConfig = nil diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift index 15fd2599613..a9bf3a8cebe 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift @@ -48,7 +48,13 @@ typealias FoundationModelsTool = FoundationModels.Tool #endif - @Suite("GeminiLanguageModel Tests", .serialized, .requireFoundationModels) + @Suite( + "GeminiLanguageModel Tests", + .serialized, + .requireFoundationModels, + // Fails runaway tool-call loops instead of hanging. + .timeLimit(.minutes(1)) + ) struct GeminiLanguageModelTests { @Generable(description: "A city summary") @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) @@ -883,7 +889,8 @@ func sessionRespondDisablesFunctionCallingAtToolCallingTurnLimit() async throws { defer { MockHTTPURLProtocol.reset() } let maximumTurns = GeminiLanguageModel.Executor.Configuration.defaultMaximumToolCallingTurns - let recordedRequests = try Self.mockToolCallingLoop(functionCallResponseCount: maximumTurns) + try #require(maximumTurns == 10) + let recordedRequests = try Self.mockToolCallingLoop { $0 <= maximumTurns } let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) let response = try await session.respond( @@ -891,8 +898,9 @@ options: GenerationOptions(toolCallingMode: .required) ) - #expect(maximumTurns == 10) #expect(response.content == Self.weatherResponseText) + #expect(Self.toolCallsEntryCount(in: session.transcript) == maximumTurns) + #expect(Self.toolOutputEntryCount(in: session.transcript) == maximumTurns) let requests = recordedRequests.all #expect(requests.count == maximumTurns + 1) #expect(requests.dropLast().allSatisfy { $0.toolConfig?.functionCallingConfig?.mode == .any }) @@ -906,19 +914,22 @@ func functionCallAfterToolCallingTurnLimitThrowsInvalidResponse() async throws { defer { MockHTTPURLProtocol.reset() } let maximumTurns = GeminiLanguageModel.Executor.Configuration.defaultMaximumToolCallingTurns - let recordedRequests = try Self.mockToolCallingLoop(functionCallResponseCount: .max) + let recordedRequests = try Self.mockToolCallingLoop { _ in true } let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + session.transcriptErrorHandlingPolicy = .preserveTranscript do { _ = try await session.respond(to: "What is the weather in Paris?") Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { - #expect(invalidResponse.debugDescription.contains("maximum of 10 tool-calling turns")) + #expect( + invalidResponse.debugDescription.contains("limit of \(maximumTurns) tool-calling turns")) } catch { Issue.record("Unexpected error thrown: \(error)") } #expect(recordedRequests.all.count == maximumTurns + 1) + #expect(Self.toolOutputEntryCount(in: session.transcript) == maximumTurns) } @Test @@ -926,24 +937,55 @@ func newPromptResetsToolCallingTurnCount() async throws { defer { MockHTTPURLProtocol.reset() } let maximumTurns = GeminiLanguageModel.Executor.Configuration.defaultMaximumToolCallingTurns - let recordedRequests = try Self.mockToolCallingLoop(functionCallResponseCount: maximumTurns) + // Each prompt receives `maximumTurns` function calls, then text. + let recordedRequests = try Self.mockToolCallingLoop { requestNumber in + requestNumber % (maximumTurns + 1) != 0 + } let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) _ = try await session.respond(to: "What is the weather in Paris?") - let response = try await session.respond(to: "Thanks! Anything else?") + let response = try await session.respond(to: "And in London?") + + #expect(response.content == Self.weatherResponseText) + let modes = recordedRequests.all.map { $0.toolConfig?.functionCallingConfig?.mode } + let validatedModes = Array( + repeating: FunctionCallingConfig.Mode?.some(.validated), count: maximumTurns) + let promptModes = validatedModes + [FunctionCallingConfig.Mode.none] + #expect(modes == promptModes + promptModes) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func parallelFunctionCallsCountAsOneToolCallingTurn() async throws { + defer { MockHTTPURLProtocol.reset() } + let maximumTurns = GeminiLanguageModel.Executor.Configuration.defaultMaximumToolCallingTurns + let recordedRequests = try Self.mockToolCallingLoop(functionCallsPerResponse: 2) { + $0 <= maximumTurns + } + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + + let response = try await session.respond( + to: "What is the weather in Paris and London?", + options: GenerationOptions(toolCallingMode: .required) + ) #expect(response.content == Self.weatherResponseText) + #expect(Self.toolCallsEntryCount(in: session.transcript) == maximumTurns) + #expect(Self.toolOutputEntryCount(in: session.transcript) == 2 * maximumTurns) let requests = recordedRequests.all - #expect(requests.count == maximumTurns + 2) - #expect(requests.last?.toolConfig?.functionCallingConfig?.mode == .validated) + #expect(requests.count == maximumTurns + 1) + #expect(requests.dropLast().allSatisfy { $0.toolConfig?.functionCallingConfig?.mode == .any }) + #expect( + requests.last?.toolConfig?.functionCallingConfig?.mode == FunctionCallingConfig.Mode.none) } @Test @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) func functionCallWithToolCallingModeDisallowedThrowsInvalidResponse() async throws { defer { MockHTTPURLProtocol.reset() } - let recordedRequests = try Self.mockToolCallingLoop(functionCallResponseCount: .max) + let recordedRequests = try Self.mockToolCallingLoop { _ in true } let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + session.transcriptErrorHandlingPolicy = .preserveTranscript do { _ = try await session.respond( @@ -957,6 +999,54 @@ Issue.record("Unexpected error thrown: \(error)") } + let requests = recordedRequests.all + #expect(requests.count == 1) + #expect(requests.first?.tools == nil) + #expect(Self.toolOutputEntryCount(in: session.transcript) == 0) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func disallowedRequestAfterToolCallsDeclaresToolsWithFunctionCallingNone() async throws { + defer { MockHTTPURLProtocol.reset() } + let recordedRequests = try Self.mockToolCallingLoop { $0 == 1 } + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + _ = try await session.respond(to: "What is the weather in Paris?") + + let response = try await session.respond( + to: "Summarize that without calling tools.", + options: GenerationOptions(toolCallingMode: .disallowed) + ) + + #expect(response.content == Self.weatherResponseText) + let requests = recordedRequests.all + #expect(requests.count == 3) + let lastRequest = try #require(requests.last) + #expect(lastRequest.tools?.first?.functionDeclarations?.map(\.name) == ["get_weather"]) + #expect( + lastRequest.toolConfig?.functionCallingConfig?.mode == FunctionCallingConfig.Mode.none) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func functionCallToToolThatIsNotEnabledThrowsInvalidResponse() async throws { + defer { MockHTTPURLProtocol.reset() } + let recordedRequests = try Self.mockToolCallingLoop(functionName: "get_current_time") { _ in + true + } + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + + do { + _ = try await session.respond(to: "What time is it?") + Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + #expect( + invalidResponse.debugDescription.contains( + "`get_current_time`, which isn't an enabled tool")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + #expect(recordedRequests.all.count == 1) } @@ -964,14 +1054,15 @@ @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) func functionCallWithoutToolsThrowsInvalidResponse() async throws { defer { MockHTTPURLProtocol.reset() } - let recordedRequests = try Self.mockToolCallingLoop(functionCallResponseCount: .max) + let recordedRequests = try Self.mockToolCallingLoop { _ in true } let session = LanguageModelSession(model: Self.makeMockModel()) do { _ = try await session.respond(to: "What is the weather in Paris?") Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { - #expect(invalidResponse.debugDescription.contains("no tools were enabled")) + #expect( + invalidResponse.debugDescription.contains("`get_weather`, which isn't an enabled tool")) } catch { Issue.record("Unexpected error thrown: \(error)") } @@ -979,9 +1070,96 @@ #expect(recordedRequests.all.count == 1) } + /// A case for `functionCallRejectionReason(_:)`. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + @available(tvOS, unavailable) + struct FunctionCallRejectionCase: Sendable, CustomTestStringConvertible { + /// The tool calling mode of the request. + let toolCallingMode: GenerationOptions.ToolCallingMode? + + /// The name of the function that Gemini calls. + let functionName: String + + /// Whether the request enables `MockWeatherTool`. + let enablesWeatherTool: Bool + + /// Whether the session has reached the tool-calling turn limit. + let isAtToolCallingTurnLimit: Bool + + /// A substring of the expected rejection reason, or `nil` if the call is allowed. + let expectedReasonSubstring: String? + + var testDescription: String { + """ + mode: \(toolCallingMode.map { "\($0.kind)" } ?? "nil"), function: \(functionName), \ + tool enabled: \(enablesWeatherTool), at limit: \(isAtToolCallingTurnLimit) + """ + } + + /// The cases to test, in order of rejection precedence. + static let all: [FunctionCallRejectionCase] = [ + .init( + toolCallingMode: .disallowed, functionName: "get_weather", enablesWeatherTool: true, + isAtToolCallingTurnLimit: false, expectedReasonSubstring: "`disallowed`"), + .init( + toolCallingMode: .disallowed, functionName: "get_weather", enablesWeatherTool: false, + isAtToolCallingTurnLimit: true, expectedReasonSubstring: "`disallowed`"), + .init( + toolCallingMode: .required, functionName: "get_weather", enablesWeatherTool: true, + isAtToolCallingTurnLimit: true, expectedReasonSubstring: "limit of 2 tool-calling turns"), + .init( + toolCallingMode: nil, functionName: "get_time", enablesWeatherTool: false, + isAtToolCallingTurnLimit: true, expectedReasonSubstring: "limit of 2 tool-calling turns"), + .init( + toolCallingMode: nil, functionName: "get_weather", enablesWeatherTool: false, + isAtToolCallingTurnLimit: false, expectedReasonSubstring: "isn't an enabled tool"), + .init( + toolCallingMode: .allowed, functionName: "get_time", enablesWeatherTool: true, + isAtToolCallingTurnLimit: false, expectedReasonSubstring: "isn't an enabled tool"), + .init( + toolCallingMode: nil, functionName: "get_weather", enablesWeatherTool: true, + isAtToolCallingTurnLimit: false, expectedReasonSubstring: nil), + .init( + toolCallingMode: .allowed, functionName: "get_weather", enablesWeatherTool: true, + isAtToolCallingTurnLimit: false, expectedReasonSubstring: nil), + .init( + toolCallingMode: .required, functionName: "get_weather", enablesWeatherTool: true, + isAtToolCallingTurnLimit: false, expectedReasonSubstring: nil), + ] + } + + @Test(arguments: FunctionCallRejectionCase.all) + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func functionCallRejectionReason(_ testCase: FunctionCallRejectionCase) throws { + let request = LanguageModelExecutorGenerationRequest( + id: UUID(), + transcript: Transcript(entries: []), + enabledTools: testCase.enablesWeatherTool + ? [Transcript.ToolDefinition(tool: MockWeatherTool())] : [], + schema: nil, + generationOptions: GenerationOptions(toolCallingMode: testCase.toolCallingMode), + contextOptions: ContextOptions(), + metadata: [:] + ) + + let reason = GeminiLanguageModel.Executor.functionCallRejectionReason( + forFunctionNamed: testCase.functionName, + in: request, + isAtToolCallingTurnLimit: testCase.isAtToolCallingTurnLimit, + maximumToolCallingTurns: 2 + ) + + if let expectedReasonSubstring = testCase.expectedReasonSubstring { + let reason = try #require(reason) + #expect(reason.contains(expectedReasonSubstring)) + } else { + #expect(reason == nil) + } + } + // MARK: - Helper Methods - /// The text of the final response returned by `mockToolCallingLoop(functionCallResponseCount:)`. + /// The final text response sent by `mockToolCallingLoop`. private static let weatherResponseText = "The weather in Paris is sunny, 72°F." /// Records the Gemini requests intercepted by `MockHTTPURLProtocol`. @@ -1004,24 +1182,26 @@ } } - /// Mocks a model that calls `MockWeatherTool` and then responds with text. + /// Mocks a model that responds to each request with either function calls or text. /// - /// - Parameter functionCallResponseCount: The number of requests that receive a function call - /// in response; later requests receive `weatherResponseText`. + /// - Parameters: + /// - functionName: The name of the function that the mock calls. + /// - functionCallsPerResponse: The number of parallel function calls in each function-call + /// response. + /// - respondsWithFunctionCalls: Returns whether the request with the given 1-based number + /// receives function calls; other requests receive `weatherResponseText`. /// - Returns: The requests that the mock receives. @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) private static func mockToolCallingLoop( - functionCallResponseCount: Int + functionName: String = "get_weather", + functionCallsPerResponse: Int = 1, + respondsWithFunctionCalls: @escaping @Sendable (_ requestNumber: Int) -> Bool ) throws -> RecordedRequests { let expectedURL = try makeExpectedStreamURL() let httpResponse = try HTTPURLResponse.mock( url: expectedURL, headerFields: ["Content-Type": "text/event-stream"] ) - let functionCallSSE = """ - data: {"candidates": [{"content": {"parts": [{"functionCall": {"id": "call-1", "name": "get_weather", "args": {"location": "Paris"}}}], "role": "model"}, "finishReason": "STOP", "index": 0}]} - - """ let textSSE = """ data: {"candidates": [{"content": {"parts": [{"text": "\(weatherResponseText)"}], "role": "model"}, "finishReason": "STOP", "index": 0}]} @@ -1032,7 +1212,21 @@ let requestNumber = recordedRequests.append( try JSONDecoder().decode(GenerateContentRequest.self, from: body) ) - let payload = requestNumber <= functionCallResponseCount ? functionCallSSE : textSSE + let payload: String + if respondsWithFunctionCalls(requestNumber) { + let parts = (1...functionCallsPerResponse).map { index in + """ + {"functionCall": {"id": "call-\(requestNumber)-\(index)", "name": "\(functionName)", \ + "args": {"location": "Paris"}}} + """ + } + payload = """ + data: {"candidates": [{"content": {"parts": [\(parts.joined(separator: ", "))], "role": "model"}, "finishReason": "STOP", "index": 0}]} + + """ + } else { + payload = textSSE + } proto.client?.urlProtocol(proto, didReceive: httpResponse, cacheStoragePolicy: .notAllowed) proto.client?.urlProtocol(proto, didLoad: Data(payload.utf8)) proto.client?.urlProtocolDidFinishLoading(proto) @@ -1040,6 +1234,30 @@ return recordedRequests } + /// Returns the number of tool-calling turns (non-empty tool-calls entries) in a transcript. + /// + /// - Parameter transcript: The transcript to search. + /// - Returns: The number of non-empty `Transcript.Entry.toolCalls` entries. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + private static func toolCallsEntryCount(in transcript: Transcript) -> Int { + transcript.filter { entry in + guard case .toolCalls(let toolCalls) = entry else { return false } + return !toolCalls.isEmpty + }.count + } + + /// Returns the number of tool outputs in a transcript. + /// + /// - Parameter transcript: The transcript to search. + /// - Returns: The number of `Transcript.Entry.toolOutput` entries. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + private static func toolOutputEntryCount(in transcript: Transcript) -> Int { + transcript.filter { entry in + guard case .toolOutput = entry else { return false } + return true + }.count + } + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) private static func makeMockModel( diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiRequestTranslatorTests.swift b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiRequestTranslatorTests.swift index 49012c8410d..d198bff219d 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiRequestTranslatorTests.swift +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiRequestTranslatorTests.swift @@ -76,7 +76,7 @@ metadata: [:] ) - let result = try GeminiRequestTranslator.translate(request) + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) #expect(result.systemInstruction == nil) #expect(result.contents?.count == 1) @@ -105,7 +105,7 @@ metadata: [:] ) - let result = try GeminiRequestTranslator.translate(request) + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) #expect(result.contents?.count == 1) let generationConfig = try #require(result.generationConfig) @@ -217,19 +217,23 @@ func translatesToolCallingModes() throws { let allowedConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: .allowed, - hasFunctionDeclarations: true + hasFunctionDeclarations: true, + isAtToolCallingTurnLimit: false ) let requiredConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: .required, - hasFunctionDeclarations: true + hasFunctionDeclarations: true, + isAtToolCallingTurnLimit: false ) let disallowedConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: .disallowed, - hasFunctionDeclarations: true + hasFunctionDeclarations: true, + isAtToolCallingTurnLimit: false ) let unsetConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: nil, - hasFunctionDeclarations: true + hasFunctionDeclarations: true, + isAtToolCallingTurnLimit: false ) #expect(allowedConfig?.functionCallingConfig?.mode == .validated) @@ -243,19 +247,23 @@ func translatesToolCallingModesWithoutDeclarationsReturnsNil() throws { let allowedConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: .allowed, - hasFunctionDeclarations: false + hasFunctionDeclarations: false, + isAtToolCallingTurnLimit: false ) let requiredConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: .required, - hasFunctionDeclarations: false + hasFunctionDeclarations: false, + isAtToolCallingTurnLimit: false ) let disallowedConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: .disallowed, - hasFunctionDeclarations: false + hasFunctionDeclarations: false, + isAtToolCallingTurnLimit: false ) let nilConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: nil, - hasFunctionDeclarations: false + hasFunctionDeclarations: false, + isAtToolCallingTurnLimit: false ) #expect(allowedConfig == nil) @@ -298,8 +306,10 @@ metadata: [:] ) - let resultWithTool = try GeminiRequestTranslator.translate(requestWithTool) - let resultWithoutTools = try GeminiRequestTranslator.translate(requestWithoutTools) + let resultWithTool = try GeminiRequestTranslator.translate( + requestWithTool, isAtToolCallingTurnLimit: false) + let resultWithoutTools = try GeminiRequestTranslator.translate( + requestWithoutTools, isAtToolCallingTurnLimit: false) let unwrappedToolConfig = try #require(resultWithTool.toolConfig) let functionCallingConfig = try #require(unwrappedToolConfig.functionCallingConfig) @@ -332,7 +342,7 @@ metadata: [:] ) - let result = try GeminiRequestTranslator.translate(request) + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) let toolConfig = try #require(result.toolConfig) let functionCallingConfig = try #require(toolConfig.functionCallingConfig) @@ -341,37 +351,49 @@ @Test @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) - func toolCallingTurnsCountsToolCallsSinceMostRecentPrompt() throws { - let arguments = try GeneratedContent(json: #"{"city": "Paris"}"#) - let parisCall = Transcript.ToolCall( - id: "call-1", toolName: "getWeather", arguments: arguments) - let londonCall = Transcript.ToolCall( - id: "call-2", toolName: "getWeather", arguments: arguments) - let tokyoCall = Transcript.ToolCall( - id: "call-3", toolName: "getWeather", arguments: arguments) + func toolCallingTurnCountCountsToolCallsSinceMostRecentPrompt() throws { + let parisCall = try Self.makeToolCall(id: "call-1", city: "Paris") + let londonCall = try Self.makeToolCall(id: "call-2", city: "London") + let tokyoCall = try Self.makeToolCall(id: "call-3", city: "Tokyo") + let romeCall = try Self.makeToolCall(id: "call-4", city: "Rome") let transcript = Transcript( entries: [ + .instructions( + Transcript.Instructions( + segments: [.text(Transcript.TextSegment(content: "Be brief."))], + toolDefinitions: [] + ) + ), .prompt(Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Paris?"))])), .toolCalls(Transcript.ToolCalls(id: "calls-1", [parisCall])), .toolOutput(Self.makeToolOutput(id: "call-1")), .response( Transcript.Response(segments: [.text(Transcript.TextSegment(content: "Sunny."))])), .prompt(Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Others?"))])), + .reasoning( + Transcript.Reasoning( + segments: [.text(Transcript.TextSegment(content: "Check two cities."))]) + ), .toolCalls(Transcript.ToolCalls(id: "calls-2", [londonCall, tokyoCall])), .toolOutput(Self.makeToolOutput(id: "call-2")), .toolOutput(Self.makeToolOutput(id: "call-3")), .toolCalls(Transcript.ToolCalls(id: "calls-3", [])), - .toolCalls(Transcript.ToolCalls(id: "calls-4", [parisCall])), - .toolOutput(Self.makeToolOutput(id: "call-1")), + .toolCalls(Transcript.ToolCalls(id: "calls-4", [romeCall])), + .toolOutput(Self.makeToolOutput(id: "call-4")), ] ) - let turns = GeminiRequestTranslator.toolCallingTurns(in: transcript) - let emptyTranscriptTurns = GeminiRequestTranslator.toolCallingTurns( - in: Transcript(entries: [])) + let turnCount = GeminiRequestTranslator.toolCallingTurnCount(in: transcript) - #expect(turns == 2) - #expect(emptyTranscriptTurns == 0) + #expect(turnCount == 2) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func toolCallingTurnCountInEmptyTranscriptIsZero() { + let turnCount = GeminiRequestTranslator.toolCallingTurnCount(in: Transcript(entries: [])) + + #expect(turnCount == 0) } @Test(arguments: [ @@ -428,8 +450,135 @@ #expect(result.toolConfig?.functionCallingConfig?.mode == FunctionCallingConfig.Mode.none) } + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesToolConfigAtToolCallingTurnLimitWithoutDeclarationsReturnsNil() throws { + let toolConfig = try GeminiRequestTranslator.translateToolConfig( + toolCallingMode: .required, + hasFunctionDeclarations: false, + isAtToolCallingTurnLimit: true + ) + + #expect(toolConfig == nil) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesDisallowedRequestAfterToolCallsDeclaresInstructionsToolsWithNone() throws { + let toolDefinition = Self.makeWeatherToolDefinition() + let parisCall = try Self.makeToolCall(id: "call-1", city: "Paris") + let request = LanguageModelExecutorGenerationRequest( + id: UUID(), + transcript: Transcript( + entries: [ + .instructions(Transcript.Instructions(segments: [], toolDefinitions: [toolDefinition])), + .prompt( + Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Paris?"))])), + .toolCalls(Transcript.ToolCalls(id: "calls-1", [parisCall])), + .toolOutput(Self.makeToolOutput(id: "call-1")), + ] + ), + enabledTools: [], + schema: nil, + generationOptions: GenerationOptions(toolCallingMode: .disallowed), + contextOptions: ContextOptions(), + metadata: [:] + ) + + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) + + let functionDeclarations = try #require(result.tools?.first?.functionDeclarations) + #expect(functionDeclarations.map(\.name) == ["get_current_weather"]) + #expect(result.toolConfig?.functionCallingConfig?.mode == FunctionCallingConfig.Mode.none) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesDisallowedRequestWithoutToolCallsOmitsTools() throws { + let toolDefinition = Self.makeWeatherToolDefinition() + let request = LanguageModelExecutorGenerationRequest( + id: UUID(), + transcript: Transcript( + entries: [ + .instructions(Transcript.Instructions(segments: [], toolDefinitions: [toolDefinition])), + .prompt( + Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Paris?"))])), + ] + ), + enabledTools: [], + schema: nil, + generationOptions: GenerationOptions(toolCallingMode: .disallowed), + contextOptions: ContextOptions(), + metadata: [:] + ) + + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) + + #expect(result.tools == nil) + #expect(result.toolConfig == nil) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesRequestWithoutEnabledToolsAfterToolCallsOmitsTools() throws { + let toolDefinition = Self.makeWeatherToolDefinition() + let parisCall = try Self.makeToolCall(id: "call-1", city: "Paris") + let request = LanguageModelExecutorGenerationRequest( + id: UUID(), + transcript: Transcript( + entries: [ + .instructions(Transcript.Instructions(segments: [], toolDefinitions: [toolDefinition])), + .prompt( + Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Paris?"))])), + .toolCalls(Transcript.ToolCalls(id: "calls-1", [parisCall])), + .toolOutput(Self.makeToolOutput(id: "call-1")), + ] + ), + enabledTools: [], + schema: nil, + generationOptions: GenerationOptions(toolCallingMode: .allowed), + contextOptions: ContextOptions(), + metadata: [:] + ) + + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) + + #expect(result.tools == nil) + #expect(result.toolConfig == nil) + } + // MARK: - Helper Methods + /// Returns a `getWeather` tool call for a city. + /// + /// - Parameters: + /// - id: The identifier of the tool call. + /// - city: The city argument of the tool call. + /// - Returns: A tool call to `getWeather`. + /// - Throws: An error if the arguments can't be parsed. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + private static func makeToolCall(id: String, city: String) throws -> Transcript.ToolCall { + Transcript.ToolCall( + id: id, + toolName: "getWeather", + arguments: try GeneratedContent(json: #"{"city": "\#(city)"}"#) + ) + } + + /// Returns the definition of a tool that gets the current weather for a city. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + private static func makeWeatherToolDefinition() -> Transcript.ToolDefinition { + Transcript.ToolDefinition( + name: "get_current_weather", + description: "Get the current weather for a city.", + parameters: LocationArguments.generationSchema + ) + } + + /// Returns the output of a `getWeather` tool call. + /// + /// - Parameter id: The identifier of the tool call that produced the output. + /// - Returns: The tool output. @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) private static func makeToolOutput(id: String) -> Transcript.ToolOutput { Transcript.ToolOutput( diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ToolCallingIntegrationTests.swift b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ToolCallingIntegrationTests.swift index b8e7a6ccf26..8ccca39eb4d 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ToolCallingIntegrationTests.swift +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ToolCallingIntegrationTests.swift @@ -280,14 +280,20 @@ @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) @available(tvOS, unavailable) struct CountingWeatherTool: FoundationModelsTool { - /// The number of calls, shared by all copies of the tool. - final class CallCount: Sendable { - let value = Mutex(0) + /// A call counter that all copies of the tool share. + final class CallCounter: Sendable { + /// The number of calls so far. + let count = Mutex(0) } let name = "getWeather" let description = "Get the current weather for a city." - let callCount = CallCount() + + /// The counter of calls to the tool, shared by all copies of the tool. + private let counter = CallCounter() + + /// The number of times the model has called the tool. + var callCount: Int { counter.count.withLock { $0 } } @Generable @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) @@ -298,16 +304,12 @@ } func call(arguments: Arguments) async throws -> String { - callCount.value.withLock { $0 += 1 } + counter.count.withLock { $0 += 1 } return "The current weather in \(arguments.city) is 22°C and sunny." } } - @Test( - .requireIntegrationTestingBackend, - .timeLimit(.minutes(2)), - arguments: IntegrationTestingBackend.availableBackends - ) + @Test(.requireIntegrationTestingBackend, arguments: IntegrationTestingBackend.availableBackends) @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) func sessionRespondRequiredToolCallingStopsAtToolCallingTurnLimit( backend: IntegrationTestingBackend @@ -316,6 +318,8 @@ let model = try await backend.makeModel() let tool = CountingWeatherTool() let session = LanguageModelSession(model: model, tools: [tool]) + // Keeps the tool calls in the transcript if Gemini calls a tool after the limit. + session.transcriptErrorHandlingPolicy = .preserveTranscript // With `required`, the model calls a tool in every response until the limit is reached. do { @@ -324,12 +328,19 @@ options: GenerationOptions(toolCallingMode: .required) ) #expect(!response.content.isEmpty) - } catch GeminiLanguageModel.Error.invalidResponse { - // Gemini sometimes returns a function call after being told not to, which ends the turn - // with this error instead of a response. + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + // Gemini sometimes calls a tool after being told not to, which ends the turn with this + // error instead of a response. + #expect(invalidResponse.debugDescription.contains("tool-calling turns per prompt")) } - #expect(tool.callCount.value.withLock { $0 } == maximumTurns) + let toolCallingTurnCount = session.transcript.filter { entry in + guard case .toolCalls(let toolCalls) = entry else { return false } + return !toolCalls.isEmpty + }.count + #expect(toolCallingTurnCount == maximumTurns) + // A tool-calling turn can call the tool more than once. + #expect(tool.callCount >= maximumTurns) } } #endif // compiler(>=6.4) && (FoundationModelsFake || canImport(FoundationModels)) From 4eb0a6f6562d86d4b16fb2ed3a2846d5295fff7e Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Tue, 10 Nov 2026 19:50:03 -0500 Subject: [PATCH 5/8] docs(ai): warn that Gemini doesn't always follow GLM's disallowed mode Replace the advice to prefer `allowed` with a warning that changing the tool calling mode to `disallowed` partway through a prompt can end the prompt with `GeminiLanguageModel.Error.invalidResponse`, and reword the summary of `functionCallRejectionReason`. --- .../GeminiLanguageModel.swift | 8 +++++--- .../GeminiLanguageModelExecutor.swift | 16 +++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift index 5420ff75c03..534adca8401 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift @@ -128,9 +128,11 @@ /// /// If Gemini calls a tool when the tool calling mode is `disallowed`, or calls a tool that /// isn't enabled for the request, the session throws - /// ``GeminiLanguageModel/Error/invalidResponse(_:)`` and the tool doesn't run. Gemini doesn't - /// always follow `disallowed` after several tool calls in response to a prompt, so prefer - /// `allowed` when you change the mode after a tool call. + /// ``GeminiLanguageModel/Error/invalidResponse(_:)`` and the tool doesn't run. + /// + /// > Warning: Gemini doesn't always follow the `disallowed` tool calling mode, especially after + /// > several tool calls in response to a prompt. Changing the mode to `disallowed` partway + /// > through a prompt can end the prompt with this error instead of a response. /// /// By default, when the session throws an error, it reverts its transcript to how it was before /// the prompt, even if tools have already run. To keep the tool calls and their output, set the diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift index 1c2f9adfc2a..26817b40be2 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift @@ -300,15 +300,14 @@ } } - /// Returns why a function call in the response to a request must not be passed to the - /// session, or `nil` if it can be. + /// Returns the reason to reject a function call from Gemini, or `nil` if the session can run + /// it. /// /// `LanguageModelSession` runs any tool call that names one of the session's tools, even - /// when the tool calling mode is `disallowed` or the tool isn't enabled for the request - /// (tested on macOS 27.2). Gemini sometimes returns a function call when it's told not to - /// call functions, or for a function that the request doesn't declare, so the executor only - /// passes on function calls to the request's enabled tools while function calling is - /// enabled. + /// when the tool calling mode is `disallowed` or the tool isn't enabled for the request. + /// Gemini sometimes returns a function call when it's told not to call functions, or for a + /// function that the request doesn't declare, so the executor only passes on function calls + /// to the request's enabled tools while function calling is enabled. /// /// - Parameters: /// - name: The name of the function that Gemini called. @@ -316,8 +315,7 @@ /// - isAtToolCallingTurnLimit: Whether the session has reached the maximum number of /// tool-calling turns for the current prompt. /// - maximumToolCallingTurns: The maximum number of tool-calling turns per prompt. - /// - Returns: A description of why the function call must not be passed to the session, or - /// `nil` if it can be. + /// - Returns: The reason to reject the function call, or `nil` if the session can run it. static func functionCallRejectionReason( forFunctionNamed name: String, in request: LanguageModelExecutorGenerationRequest, From 8274cf0468509e86099ab3608dd55e929062a882 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Tue, 10 Nov 2026 20:13:07 -0500 Subject: [PATCH 6/8] fix(ai): make GLM's disabled-function-calling declarations best effort Declaring the transcript's instructions tools when function calling is disabled is best effort, so send the request without declarations if a tool's schema can't be translated, instead of failing a request that previously succeeded. Document that the fallback also applies at the turn limit and that it uses the latest instructions' tools. Also: - Reject function calls with an empty name like those without a name. - Use `isFunctionCallingDisabled` to choose the `NONE` mode. - Clarify which parts the pre-streaming check keeps out of the transcript, and assert that every function call was checked. - Reorder the tool calling docs, name the error in the `disallowed` warning, and mention the `transcriptErrorHandlingPolicy(_:)` modifier. - Test that a rejected function call's chunk isn't streamed, the fallback at the turn limit and with an untranslatable schema, and empty function names; rename the unknown-tool test. - Mention the turn limit in the test READMEs. --- .../GeminiLanguageModel.swift | 26 +++--- .../GeminiLanguageModelExecutor.swift | 18 ++-- .../GeminiRequestTranslator.swift | 56 ++++++++---- .../GeminiLanguageModelTests.swift | 89 +++++++++++++++++-- .../GeminiRequestTranslatorTests.swift | 83 ++++++++++++++++- .../IntegrationTests/README.md | 4 +- .../Tests/GeminiLanguageModelTests/README.md | 2 +- 7 files changed, 229 insertions(+), 49 deletions(-) diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift index 534adca8401..02fd58f5f93 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift @@ -126,23 +126,25 @@ /// modifier of a `LanguageModelSession.DynamicProfile`, for example from `required` to /// `allowed` after the first tool call, as shown in [`GenerationOptions.ToolCallingMode`](https://developer.apple.com/documentation/foundationmodels/generationoptions/toolcallingmode-swift.struct). /// - /// If Gemini calls a tool when the tool calling mode is `disallowed`, or calls a tool that - /// isn't enabled for the request, the session throws - /// ``GeminiLanguageModel/Error/invalidResponse(_:)`` and the tool doesn't run. + /// The limit is counted from the transcript that the session sends to the model, after any + /// `historyTransform(_:)` of a `LanguageModelSession.DynamicProfile`. A transform that removes + /// earlier tool calls can stop the limit from being reached, and a transform that removes + /// prompts can make the limit apply across prompts. + /// + /// If Gemini calls a tool when the tool calling mode is `disallowed`, or calls a tool that the + /// session doesn't have, the session throws ``GeminiLanguageModel/Error/invalidResponse(_:)`` + /// and the tool doesn't run. /// /// > Warning: Gemini doesn't always follow the `disallowed` tool calling mode, especially after /// > several tool calls in response to a prompt. Changing the mode to `disallowed` partway - /// > through a prompt can end the prompt with this error instead of a response. + /// > through a prompt can make the session throw + /// > ``GeminiLanguageModel/Error/invalidResponse(_:)`` instead of returning a response. /// /// By default, when the session throws an error, it reverts its transcript to how it was before - /// the prompt, even if tools have already run. To keep the tool calls and their output, set the - /// session's [`transcriptErrorHandlingPolicy`](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/transcripterrorhandlingpolicy) - /// to `preserveTranscript`. - /// - /// The limit is counted from the transcript that the session sends to the model, after any - /// `historyTransform(_:)` of a `LanguageModelSession.DynamicProfile`. A transform that removes - /// earlier tool calls can stop the limit from being reached, and a transform that removes - /// prompts can make the limit apply across prompts. + /// the prompt, even if tools have already run. To keep the tool calls and their outputs, set + /// the session's [`transcriptErrorHandlingPolicy`](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/transcripterrorhandlingpolicy) + /// to `preserveTranscript`, or use the [`transcriptErrorHandlingPolicy(_:)`](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/dynamicprofile/transcripterrorhandlingpolicy%28_%29) + /// modifier of a `LanguageModelSession.DynamicProfile`. public var capabilities: LanguageModelCapabilities { LanguageModelCapabilities([ .guidedGeneration, diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift index 26817b40be2..2de7d85fdef 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift @@ -155,9 +155,9 @@ if let candidates = chunk.candidates, let candidate = candidates.first, let parts = candidate.content?.parts { - // Check every function call in the chunk before streaming any of its parts, so that - // a rejected function call doesn't leave a partial response (such as its thought - // signature) in the transcript. + // Check every function call in the chunk before streaming any of the chunk's parts, + // so that a rejected function call's own parts (such as its thought signature) + // aren't streamed. Parts from earlier chunks have already been streamed. try Self.validateFunctionCalls( in: parts, for: request, @@ -200,9 +200,13 @@ } } - // `validateFunctionCalls` has already checked that every function call has a name - // and is allowed for this request. - if case .functionCall(let call) = part.data, let name = call.name { + if case .functionCall(let call) = part.data { + // `validateFunctionCalls` has already checked that every function call in this + // chunk has a name and is allowed for this request. + guard let name = call.name, !name.isEmpty else { + assertionFailure("A function call without a name wasn't rejected.") + continue + } let callID = call.id ?? UUID().uuidString let argsString: String if let args = call.args, !args.isEmpty { @@ -280,7 +284,7 @@ ) throws { for part in parts { guard case .functionCall(let call) = part.data else { continue } - guard let name = call.name else { + guard let name = call.name, !name.isEmpty else { throw GeminiLanguageModel.Error.invalidResponse( GeminiLanguageModel.Error.InvalidResponse( debugDescription: "Gemini returned a function call without a name." diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift index 12aa96cf16b..226f0716f40 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift @@ -42,12 +42,20 @@ let (contents, systemInstruction) = try GeminiTranscriptTranslator.translate( request.transcript ) - let tools = try translateTools( - functionDeclarationToolDefinitions( - for: request, - isAtToolCallingTurnLimit: isAtToolCallingTurnLimit + let tools: [GeminiAPIDataModels.Tool]? + if request.enabledToolDefinitions.isEmpty { + // Declaring the tools in this case is best effort: function calling is disabled, so if a + // tool's schema can't be translated, send the request without declarations instead of + // failing it. + tools = try? translateTools( + toolDefinitionsForDisabledFunctionCalling( + in: request, + isAtToolCallingTurnLimit: isAtToolCallingTurnLimit + ) ) - ) + } else { + tools = try translateTools(request.enabledToolDefinitions) + } let hasFunctionDeclarations = tools?.contains { $0.functionDeclarations?.isEmpty == false } ?? false @@ -81,24 +89,30 @@ isAtToolCallingTurnLimit || toolCallingMode?.kind == .disallowed } - /// Returns the tool definitions to declare as functions in the Gemini request. + /// Returns the tool definitions to declare as functions in a request that enables no tools. + /// + /// With the `disallowed` tool calling mode, the session enables no tools, so a transcript with + /// earlier tool calls would refer to functions that the request doesn't declare. When function + /// calling is disabled (the `disallowed` mode or the tool-calling turn limit) and the + /// transcript has tool calls, this returns the tool definitions from the transcript's most + /// recent instructions, so that the request declares them with function calling disabled, + /// matching the final request after the turn limit is reached. /// - /// This is usually the request's enabled tools. With the `disallowed` tool calling mode, the - /// session enables no tools, so a transcript with earlier tool calls would refer to functions - /// that the request doesn't declare. In that case, this returns the tool definitions from the - /// transcript's instructions instead, and the request disables function calling, matching the - /// final request after the tool-calling turn limit is reached. + /// The instructions only list the tools whose schema is included in instructions, from the + /// session's current tools. Function calling is disabled, so any function call is rejected + /// whether or not its function is declared. /// /// - Parameters: /// - request: The generation request from the Foundation Models session. /// - isAtToolCallingTurnLimit: Whether the session has reached the maximum number of /// tool-calling turns for the current prompt. - /// - Returns: The tool definitions to declare as functions. - static func functionDeclarationToolDefinitions( - for request: LanguageModelExecutorGenerationRequest, + /// - Returns: The tool definitions to declare as functions, or an empty array if the request + /// shouldn't declare any. + static func toolDefinitionsForDisabledFunctionCalling( + in request: LanguageModelExecutorGenerationRequest, isAtToolCallingTurnLimit: Bool ) -> [Transcript.ToolDefinition] { - guard request.enabledToolDefinitions.isEmpty, + guard isFunctionCallingDisabled( toolCallingMode: request.generationOptions.toolCallingMode, isAtToolCallingTurnLimit: isAtToolCallingTurnLimit @@ -108,7 +122,7 @@ return !toolCalls.isEmpty }) else { - return request.enabledToolDefinitions + return [] } for entry in request.transcript.reversed() { if case .instructions(let instructions) = entry, !instructions.toolDefinitions.isEmpty { @@ -227,9 +241,13 @@ } else { requestedMode = allowedMode } - // The limit overrides every mode, including `required`, which would otherwise keep - // calling tools indefinitely. - let callingMode = isAtToolCallingTurnLimit ? FunctionCallingConfig.Mode.none : requestedMode + // The turn limit disables function calling whatever the requested mode is, including + // `required`, which would otherwise keep calling tools indefinitely. + let callingMode = + isFunctionCallingDisabled( + toolCallingMode: toolCallingMode, + isAtToolCallingTurnLimit: isAtToolCallingTurnLimit + ) ? FunctionCallingConfig.Mode.none : requestedMode functionCallingConfig = FunctionCallingConfig { $0.mode = callingMode } } else { functionCallingConfig = nil diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift index a9bf3a8cebe..e38c10e0840 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift @@ -1029,7 +1029,7 @@ @Test @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) - func functionCallToToolThatIsNotEnabledThrowsInvalidResponse() async throws { + func functionCallToUnknownToolThrowsInvalidResponse() async throws { defer { MockHTTPURLProtocol.reset() } let recordedRequests = try Self.mockToolCallingLoop(functionName: "get_current_time") { _ in true @@ -1070,7 +1070,70 @@ #expect(recordedRequests.all.count == 1) } - /// A case for `functionCallRejectionReason(_:)`. + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func functionCallWithEmptyNameThrowsInvalidResponse() async throws { + defer { MockHTTPURLProtocol.reset() } + let recordedRequests = try Self.mockToolCallingLoop(functionName: "") { _ in true } + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + + do { + _ = try await session.respond(to: "What is the weather in Paris?") + Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + #expect(invalidResponse.debugDescription.contains("without a name")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + + #expect(recordedRequests.all.count == 1) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func streamResponseDoesNotStreamPartsOfChunkWithRejectedFunctionCall() async throws { + defer { MockHTTPURLProtocol.reset() } + let expectedURL = try Self.makeExpectedStreamURL() + let httpResponse = try HTTPURLResponse.mock( + url: expectedURL, + headerFields: ["Content-Type": "text/event-stream"] + ) + // One chunk with a thought, text, and a function call with a thought signature. + let ssePayload = """ + data: {"candidates": [{"content": {"parts": [{"text": "Checking the weather.", "thought": true}, {"text": "Let me check."}, {"functionCall": {"id": "call-1", "name": "get_weather", "args": {"location": "Paris"}}, "thoughtSignature": "c2lnbmF0dXJl"}], "role": "model"}, "finishReason": "STOP", "index": 0}]} + + """ + MockHTTPURLProtocol.setHandler(for: expectedURL) { _, proto in + proto.client?.urlProtocol(proto, didReceive: httpResponse, cacheStoragePolicy: .notAllowed) + proto.client?.urlProtocol(proto, didLoad: Data(ssePayload.utf8)) + proto.client?.urlProtocolDidFinishLoading(proto) + } + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + session.transcriptErrorHandlingPolicy = .preserveTranscript + + do { + let stream = session.streamResponse( + to: "What is the weather in Paris?", + options: GenerationOptions(toolCallingMode: .disallowed) + ) + for try await _ in stream {} + Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + #expect(invalidResponse.debugDescription.contains("tool calling mode was `disallowed`")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + + let promptIndex = try #require( + session.transcript.firstIndex { entry in + guard case .prompt = entry else { return false } + return true + } + ) + #expect(session.transcript[(promptIndex + 1)...].isEmpty) + } + + /// A test case for `GeminiLanguageModel.Executor.functionCallRejectionReason`. @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) @available(tvOS, unavailable) struct FunctionCallRejectionCase: Sendable, CustomTestStringConvertible { @@ -1089,11 +1152,25 @@ /// A substring of the expected rejection reason, or `nil` if the call is allowed. let expectedReasonSubstring: String? + /// A description of the test case for test reports. var testDescription: String { - """ - mode: \(toolCallingMode.map { "\($0.kind)" } ?? "nil"), function: \(functionName), \ - tool enabled: \(enablesWeatherTool), at limit: \(isAtToolCallingTurnLimit) - """ + let modeDescription: String + switch toolCallingMode?.kind { + case nil: + modeDescription = "nil" + case .allowed: + modeDescription = "allowed" + case .required: + modeDescription = "required" + case .disallowed: + modeDescription = "disallowed" + @unknown default: + modeDescription = "unknown" + } + return """ + mode: \(modeDescription), function: \(functionName), \ + tool enabled: \(enablesWeatherTool), at limit: \(isAtToolCallingTurnLimit) + """ } /// The cases to test, in order of rejection precedence. diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiRequestTranslatorTests.swift b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiRequestTranslatorTests.swift index d198bff219d..c22d9621ffd 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiRequestTranslatorTests.swift +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiRequestTranslatorTests.swift @@ -50,6 +50,15 @@ var city: String } + @Generable( + description: "Postal code arguments with a pattern guide that Gemini doesn't support") + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + @available(tvOS, unavailable) + struct PostalCodeArguments { + @Guide(description: "A postal code", .pattern(#/^\d{5}$/#)) + var postalCode: String + } + @Generable(description: "Empty arguments") @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) @available(tvOS, unavailable) @@ -374,6 +383,8 @@ Transcript.Reasoning( segments: [.text(Transcript.TextSegment(content: "Check two cities."))]) ), + .response( + Transcript.Response(segments: [.text(Transcript.TextSegment(content: "Checking."))])), .toolCalls(Transcript.ToolCalls(id: "calls-2", [londonCall, tokyoCall])), .toolOutput(Self.makeToolOutput(id: "call-2")), .toolOutput(Self.makeToolOutput(id: "call-3")), @@ -488,10 +499,78 @@ let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) let functionDeclarations = try #require(result.tools?.first?.functionDeclarations) - #expect(functionDeclarations.map(\.name) == ["get_current_weather"]) + #expect(functionDeclarations.map(\.name) == ["getWeather"]) #expect(result.toolConfig?.functionCallingConfig?.mode == FunctionCallingConfig.Mode.none) } + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesRequestWithoutEnabledToolsAtToolCallingTurnLimitDeclaresInstructionsTools() + throws + { + let toolDefinition = Self.makeWeatherToolDefinition() + let parisCall = try Self.makeToolCall(id: "call-1", city: "Paris") + let request = LanguageModelExecutorGenerationRequest( + id: UUID(), + transcript: Transcript( + entries: [ + .instructions(Transcript.Instructions(segments: [], toolDefinitions: [toolDefinition])), + .prompt( + Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Paris?"))])), + .toolCalls(Transcript.ToolCalls(id: "calls-1", [parisCall])), + .toolOutput(Self.makeToolOutput(id: "call-1")), + ] + ), + enabledTools: [], + schema: nil, + generationOptions: GenerationOptions(toolCallingMode: .required), + contextOptions: ContextOptions(), + metadata: [:] + ) + + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: true) + + let functionDeclarations = try #require(result.tools?.first?.functionDeclarations) + #expect(functionDeclarations.map(\.name) == ["getWeather"]) + #expect(result.toolConfig?.functionCallingConfig?.mode == FunctionCallingConfig.Mode.none) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesDisallowedRequestWithUntranslatableInstructionsToolOmitsTools() throws { + let toolDefinition = Transcript.ToolDefinition( + name: "lookUpPostalCode", + description: "Look up a postal code.", + parameters: PostalCodeArguments.generationSchema + ) + let lookUpCall = Transcript.ToolCall( + id: "call-1", + toolName: "lookUpPostalCode", + arguments: try GeneratedContent(json: #"{"postalCode": "94043"}"#) + ) + let request = LanguageModelExecutorGenerationRequest( + id: UUID(), + transcript: Transcript( + entries: [ + .instructions(Transcript.Instructions(segments: [], toolDefinitions: [toolDefinition])), + .prompt( + Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "94043?"))])), + .toolCalls(Transcript.ToolCalls(id: "calls-1", [lookUpCall])), + ] + ), + enabledTools: [], + schema: nil, + generationOptions: GenerationOptions(toolCallingMode: .disallowed), + contextOptions: ContextOptions(), + metadata: [:] + ) + + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) + + #expect(result.tools == nil) + #expect(result.toolConfig == nil) + } + @Test @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) func translatesDisallowedRequestWithoutToolCallsOmitsTools() throws { @@ -569,7 +648,7 @@ @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) private static func makeWeatherToolDefinition() -> Transcript.ToolDefinition { Transcript.ToolDefinition( - name: "get_current_weather", + name: "getWeather", description: "Get the current weather for a city.", parameters: LocationArguments.generationSchema ) diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/README.md b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/README.md index 0606a39e3a9..44555e8ae06 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/README.md +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/README.md @@ -61,8 +61,8 @@ All integration tests are parameterized across Parameterized integration tests for tool calling (function calling) using FoundationModels `Tool` definitions, covering single-turn tool calls, sequential tool calls, parallel tool calls, tool calling mode configuration, - parameterless tools with empty arguments, and thought signatures recorded - as reasoning entries. + parameterless tools with empty arguments, thought signatures recorded as + reasoning entries, and the limit on tool-calling turns per prompt. * [`ImageUnderstandingIntegrationTests.swift`](ImageUnderstandingIntegrationTests.swift): Parameterized integration tests for image input (`vision`) using images generated in code, covering single and multiple images, `@Generable` diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/README.md b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/README.md index 349c962373c..421a7c86c8a 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/README.md +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/README.md @@ -20,7 +20,7 @@ swift test --filter GeminiLanguageModelTests |---|---|---| | [`GeminiLanguageModelTests.swift`](GeminiLanguageModelTests.swift) | Protocol conformance, single/multi-turn responses, streaming, tool calling | Unit (`MockHTTPURLProtocol`) | | [`GenerationSchema+GeminiTests.swift`](GenerationSchema+GeminiTests.swift) | JSON Schema encoding and property ordering conversion | Unit (pure transformation) | -| [`GeminiRequestTranslatorTests.swift`](GeminiRequestTranslatorTests.swift) | Request, generation config, tool declarations, and tool mode mapping | Unit (pure transformation) | +| [`GeminiRequestTranslatorTests.swift`](GeminiRequestTranslatorTests.swift) | Request, generation config, tool declarations, tool mode mapping, and tool-calling turn counting | Unit (pure transformation) | | [`GeminiTranscriptTranslatorTests.swift`](GeminiTranscriptTranslatorTests.swift) | Apple `Transcript` <-> Gemini payload mapping (tools, reasoning, prompts, image parts) | Unit (pure transformation) | | [`GeminiImageTranslatorTests.swift`](GeminiImageTranslatorTests.swift) | Image attachment encoding, orientation, file pass-through, and URL handling | Unit (images generated in code) | | [`TestImages.swift`](TestImages.swift) | Helpers that generate, encode, and sample test images without bundled resources | Test utility | From 0d41b33b6c990dfe20e5e5df47bb3b010b932aa3 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Wed, 11 Nov 2026 15:34:48 -0500 Subject: [PATCH 7/8] fix(ai): disable GLM function calls without tools A request that enables no tools, such as one from a DynamicProfile that disables every tool, sent earlier function calls without declarations or a tool config unless the tool calling mode was `disallowed`. Treat any request without enabled tools as having function calling disabled, whatever the mode, and declare the latest instructions' tools with mode `NONE` when the transcript has tool calls. Translate these fallback declarations per tool, so that one tool whose schema can't be translated no longer drops the other declarations and the `NONE` mode with them. Also: - Remove the unused configurable tool-calling turn limit and its precondition; the limit is now `Executor.maximumToolCallingTurns`. - Replace an `assertionFailure` in the streaming loop with a throwing `functionName(of:)` helper, so that the check can't trap. - Fix Apple documentation links that were missing the `:` in `(_:)`. - Say "isn't enabled for the request" in the tool calling docs, note that each tool-calling turn adds several transcript entries, and say which instructions entry the fallback declarations come from. - Test a prompt after a preserved turn-limit error, rejected calls in the same and a later chunk, requests without enabled tools in each mode, multiple instructions entries, and partially translatable tools. --- .../GeminiLanguageModel.swift | 21 +- .../GeminiLanguageModelExecutor.swift | 96 ++++----- .../GeminiRequestTranslator.swift | 125 +++++++----- .../GeminiLanguageModelTests.swift | 178 +++++++++++++++- .../GeminiRequestTranslatorTests.swift | 190 ++++++++++++++++-- .../ToolCallingIntegrationTests.swift | 2 +- 6 files changed, 472 insertions(+), 140 deletions(-) diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift index 02fd58f5f93..ac54649124f 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift @@ -42,7 +42,7 @@ /// [`LanguageModel`](https://developer.apple.com/documentation/foundationmodels/languagemodel) /// protocol and can be used with the Foundation Models framework when [initializing](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/init%28model:tools:instructions:%29) /// a [`LanguageModelSession`](https://developer.apple.com/documentation/foundationmodels/languagemodelsession), - /// or by [setting the model](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/dynamicprofile/model%28_%29) + /// or by [setting the model](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/dynamicprofile/model%28_:%29) /// on a [`DynamicProfile`](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/dynamicprofile). /// /// For more details on using Gemini to generate content with the Foundation Models framework, @@ -122,18 +122,21 @@ /// /// The limit also applies when the tool calling mode is `required`. With `required`, the model /// otherwise keeps calling tools until a tool throws an error or the mode changes. To stop - /// sooner, change the mode with the [`toolCallingMode(_:)`](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/dynamicprofile/toolcallingmode%28_%29) + /// sooner, change the mode with the [`toolCallingMode(_:)`](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/dynamicprofile/toolcallingmode%28_:%29) /// modifier of a `LanguageModelSession.DynamicProfile`, for example from `required` to /// `allowed` after the first tool call, as shown in [`GenerationOptions.ToolCallingMode`](https://developer.apple.com/documentation/foundationmodels/generationoptions/toolcallingmode-swift.struct). /// /// The limit is counted from the transcript that the session sends to the model, after any - /// `historyTransform(_:)` of a `LanguageModelSession.DynamicProfile`. A transform that removes - /// earlier tool calls can stop the limit from being reached, and a transform that removes - /// prompts can make the limit apply across prompts. + /// [`historyTransform(_:)`](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/dynamicprofile/historytransform%28_:%29) + /// of a `LanguageModelSession.DynamicProfile`. A transform that removes earlier tool calls can + /// stop the limit from being reached, and a transform that removes prompts can make the limit + /// apply across prompts. Each tool-calling turn adds several transcript entries, such as the + /// tool calls, their outputs, and the model's reasoning, so a transform that keeps a fixed + /// number of recent entries keeps far fewer tool-calling turns than entries. /// - /// If Gemini calls a tool when the tool calling mode is `disallowed`, or calls a tool that the - /// session doesn't have, the session throws ``GeminiLanguageModel/Error/invalidResponse(_:)`` - /// and the tool doesn't run. + /// If Gemini calls a tool when the tool calling mode is `disallowed`, or calls a tool that + /// isn't enabled for the request, the session throws + /// ``GeminiLanguageModel/Error/invalidResponse(_:)`` and the tool doesn't run. /// /// > Warning: Gemini doesn't always follow the `disallowed` tool calling mode, especially after /// > several tool calls in response to a prompt. Changing the mode to `disallowed` partway @@ -143,7 +146,7 @@ /// By default, when the session throws an error, it reverts its transcript to how it was before /// the prompt, even if tools have already run. To keep the tool calls and their outputs, set /// the session's [`transcriptErrorHandlingPolicy`](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/transcripterrorhandlingpolicy) - /// to `preserveTranscript`, or use the [`transcriptErrorHandlingPolicy(_:)`](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/dynamicprofile/transcripterrorhandlingpolicy%28_%29) + /// to `preserveTranscript`, or use the [`transcriptErrorHandlingPolicy(_:)`](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/dynamicprofile/transcripterrorhandlingpolicy%28_:%29) /// modifier of a `LanguageModelSession.DynamicProfile`. public var capabilities: LanguageModelCapabilities { LanguageModelCapabilities([ diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift index 2de7d85fdef..4551326630f 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift @@ -46,17 +46,6 @@ /// The `URLSessionConfiguration` to use. let sessionConfiguration: URLSessionConfiguration - /// The maximum number of tool-calling turns allowed in response to a single prompt. - /// - /// A tool-calling turn is a model response that calls one or more tools. When this limit - /// is reached, the next request tells Gemini not to call functions; if it returns a - /// function call anyway, the executor throws `GeminiLanguageModel.Error.invalidResponse`. - let maximumToolCallingTurns: Int - - /// The default maximum number of tool-calling turns allowed in response to a single - /// prompt. - static let defaultMaximumToolCallingTurns = 10 - /// Initializes an executor configuration. /// /// - Parameters: @@ -64,24 +53,27 @@ /// - endpointConfiguration: The network endpoint configuration. /// - headerProvider: An optional async provider for dynamic headers. /// - sessionConfiguration: The `URLSessionConfiguration` to use. - /// - maximumToolCallingTurns: The maximum number of tool-calling turns allowed in - /// response to a single prompt. Must be greater than zero. init( modelResource: ModelResource, endpointConfiguration: EndpointConfiguration, headerProvider: HeaderProvider?, - sessionConfiguration: URLSessionConfiguration, - maximumToolCallingTurns: Int = Configuration.defaultMaximumToolCallingTurns + sessionConfiguration: URLSessionConfiguration ) { - precondition(maximumToolCallingTurns > 0, "maximumToolCallingTurns must be positive.") self.modelResource = modelResource self.endpointConfiguration = endpointConfiguration self.headerProvider = headerProvider self.sessionConfiguration = sessionConfiguration - self.maximumToolCallingTurns = maximumToolCallingTurns } } + /// The maximum number of tool-calling turns allowed in response to a single prompt. + /// + /// A tool-calling turn is a model response that calls one or more tools. When this limit is + /// reached, the next request tells Gemini not to call functions; if it returns a function + /// call anyway, the executor throws `GeminiLanguageModel.Error.invalidResponse`. Keep the + /// `GeminiLanguageModel.capabilities` docs and the changelog in sync with this value. + static let maximumToolCallingTurns = 10 + private let configuration: Configuration /// Initializes a new executor with the specified configuration. @@ -108,7 +100,7 @@ // limit the number of tool-calling turns to bound the number of requests. let isAtToolCallingTurnLimit = GeminiRequestTranslator.toolCallingTurnCount(in: request.transcript) - >= configuration.maximumToolCallingTurns + >= Self.maximumToolCallingTurns let generateRequest = try GeminiRequestTranslator.translate( request, isAtToolCallingTurnLimit: isAtToolCallingTurnLimit @@ -157,12 +149,13 @@ { // Check every function call in the chunk before streaming any of the chunk's parts, // so that a rejected function call's own parts (such as its thought signature) - // aren't streamed. Parts from earlier chunks have already been streamed. + // aren't streamed. Parts from earlier chunks have already been streamed: when this + // throws, the session discards their tool calls, but with the `preserveTranscript` + // policy it keeps their text and reasoning. try Self.validateFunctionCalls( in: parts, for: request, - isAtToolCallingTurnLimit: isAtToolCallingTurnLimit, - maximumToolCallingTurns: configuration.maximumToolCallingTurns + isAtToolCallingTurnLimit: isAtToolCallingTurnLimit ) for part in parts { @@ -202,11 +195,8 @@ if case .functionCall(let call) = part.data { // `validateFunctionCalls` has already checked that every function call in this - // chunk has a name and is allowed for this request. - guard let name = call.name, !name.isEmpty else { - assertionFailure("A function call without a name wasn't rejected.") - continue - } + // chunk has a name and is allowed for this request, so this doesn't throw. + let name = try Self.functionName(of: call) let callID = call.id ?? UUID().uuidString let argsString: String if let args = call.args, !args.isEmpty { @@ -273,29 +263,20 @@ /// - request: The generation request from the Foundation Models session. /// - isAtToolCallingTurnLimit: Whether the session has reached the maximum number of /// tool-calling turns for the current prompt. - /// - maximumToolCallingTurns: The maximum number of tool-calling turns per prompt. /// - Throws: `GeminiLanguageModel.Error.invalidResponse` if a function call has no name or /// if the request doesn't allow it (see `functionCallRejectionReason`). static func validateFunctionCalls( in parts: [Part], for request: LanguageModelExecutorGenerationRequest, - isAtToolCallingTurnLimit: Bool, - maximumToolCallingTurns: Int + isAtToolCallingTurnLimit: Bool ) throws { for part in parts { guard case .functionCall(let call) = part.data else { continue } - guard let name = call.name, !name.isEmpty else { - throw GeminiLanguageModel.Error.invalidResponse( - GeminiLanguageModel.Error.InvalidResponse( - debugDescription: "Gemini returned a function call without a name." - ) - ) - } + let name = try functionName(of: call) if let reason = functionCallRejectionReason( forFunctionNamed: name, in: request, - isAtToolCallingTurnLimit: isAtToolCallingTurnLimit, - maximumToolCallingTurns: maximumToolCallingTurns + isAtToolCallingTurnLimit: isAtToolCallingTurnLimit ) { throw GeminiLanguageModel.Error.invalidResponse( GeminiLanguageModel.Error.InvalidResponse(debugDescription: reason) @@ -304,6 +285,23 @@ } } + /// Returns the name of the function that a function call from Gemini calls. + /// + /// - Parameter call: A function call from Gemini. + /// - Returns: The non-empty name of the function. + /// - Throws: `GeminiLanguageModel.Error.invalidResponse` if the function call has no name or + /// an empty name. + static func functionName(of call: FunctionCall) throws -> String { + guard let name = call.name, !name.isEmpty else { + throw GeminiLanguageModel.Error.invalidResponse( + GeminiLanguageModel.Error.InvalidResponse( + debugDescription: "Gemini returned a function call without a name." + ) + ) + } + return name + } + /// Returns the reason to reject a function call from Gemini, or `nil` if the session can run /// it. /// @@ -318,25 +316,19 @@ /// - request: The generation request from the Foundation Models session. /// - isAtToolCallingTurnLimit: Whether the session has reached the maximum number of /// tool-calling turns for the current prompt. - /// - maximumToolCallingTurns: The maximum number of tool-calling turns per prompt. /// - Returns: The reason to reject the function call, or `nil` if the session can run it. static func functionCallRejectionReason( forFunctionNamed name: String, in request: LanguageModelExecutorGenerationRequest, - isAtToolCallingTurnLimit: Bool, - maximumToolCallingTurns: Int + isAtToolCallingTurnLimit: Bool ) -> String? { - let toolCallingMode = request.generationOptions.toolCallingMode - if GeminiRequestTranslator.isFunctionCallingDisabled( - toolCallingMode: toolCallingMode, - isAtToolCallingTurnLimit: isAtToolCallingTurnLimit - ) { - if toolCallingMode?.kind == .disallowed { - return """ - Gemini returned a function call to `\(name)` when the tool calling mode was \ - `disallowed`. - """ - } + if request.generationOptions.toolCallingMode?.kind == .disallowed { + return """ + Gemini returned a function call to `\(name)` when the tool calling mode was \ + `disallowed`. + """ + } + if isAtToolCallingTurnLimit { return """ Gemini returned a function call to `\(name)` after function calling was disabled \ because the limit of \(maximumToolCallingTurns) tool-calling turns per prompt was \ diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift index 226f0716f40..9d38ed1e79c 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift @@ -44,15 +44,10 @@ ) let tools: [GeminiAPIDataModels.Tool]? if request.enabledToolDefinitions.isEmpty { - // Declaring the tools in this case is best effort: function calling is disabled, so if a - // tool's schema can't be translated, send the request without declarations instead of - // failing it. - tools = try? translateTools( - toolDefinitionsForDisabledFunctionCalling( - in: request, - isAtToolCallingTurnLimit: isAtToolCallingTurnLimit - ) - ) + // Declaring the tools in this case is best effort: function calling is disabled, so skip + // any tool whose schema can't be translated instead of failing the request. If no tool + // can be translated, the request is sent without declarations. + tools = translateToolsBestEffort(toolDefinitionsForRequestWithoutEnabledTools(in: request)) } else { tools = try translateTools(request.enabledToolDefinitions) } @@ -66,7 +61,10 @@ $0.toolConfig = try translateToolConfig( toolCallingMode: request.generationOptions.toolCallingMode, hasFunctionDeclarations: hasFunctionDeclarations, - isAtToolCallingTurnLimit: isAtToolCallingTurnLimit + isFunctionCallingDisabled: isFunctionCallingDisabled( + for: request, + isAtToolCallingTurnLimit: isAtToolCallingTurnLimit + ) ) $0.generationConfig = try translateGenerationConfig(schema: request.schema) } @@ -74,49 +72,47 @@ /// Returns whether a request must not call functions. /// - /// Function calling is disabled when the tool calling mode is `disallowed` or when the session - /// has reached the maximum number of tool-calling turns for the current prompt. + /// Function calling is disabled when the tool calling mode is `disallowed`, when the request + /// enables no tools (whatever the tool calling mode is), or when the session has reached the + /// maximum number of tool-calling turns for the current prompt. /// /// - Parameters: - /// - toolCallingMode: The tool calling mode from the request. + /// - request: The generation request from the Foundation Models session. /// - isAtToolCallingTurnLimit: Whether the session has reached the maximum number of /// tool-calling turns for the current prompt. /// - Returns: `true` if the request must not call functions. static func isFunctionCallingDisabled( - toolCallingMode: GenerationOptions.ToolCallingMode?, + for request: LanguageModelExecutorGenerationRequest, isAtToolCallingTurnLimit: Bool ) -> Bool { - isAtToolCallingTurnLimit || toolCallingMode?.kind == .disallowed + isAtToolCallingTurnLimit + || request.enabledToolDefinitions.isEmpty + || request.generationOptions.toolCallingMode?.kind == .disallowed } /// Returns the tool definitions to declare as functions in a request that enables no tools. /// - /// With the `disallowed` tool calling mode, the session enables no tools, so a transcript with - /// earlier tool calls would refer to functions that the request doesn't declare. When function - /// calling is disabled (the `disallowed` mode or the tool-calling turn limit) and the - /// transcript has tool calls, this returns the tool definitions from the transcript's most - /// recent instructions, so that the request declares them with function calling disabled, - /// matching the final request after the turn limit is reached. + /// A request that enables no tools disables function calling, whatever the tool calling mode + /// is. The session enables no tools with the `disallowed` tool calling mode, and can also + /// disable every tool for a request, for example with a `LanguageModelSession.DynamicProfile`. + /// A transcript with earlier tool calls would then refer to functions that the request + /// doesn't declare. If the transcript has tool calls, this returns the tool definitions from + /// the most recent instructions entry that lists tools, so that the request declares them + /// with function calling disabled, matching the final request after the turn limit is + /// reached. /// - /// The instructions only list the tools whose schema is included in instructions, from the - /// session's current tools. Function calling is disabled, so any function call is rejected - /// whether or not its function is declared. + /// Instructions entries only list the tools whose schema is included in instructions, from + /// the session's tools when the entry was added. Function calling is disabled, so any + /// function call is rejected whether or not its function is declared. /// - /// - Parameters: - /// - request: The generation request from the Foundation Models session. - /// - isAtToolCallingTurnLimit: Whether the session has reached the maximum number of - /// tool-calling turns for the current prompt. + /// - Parameter request: The generation request from the Foundation Models session. /// - Returns: The tool definitions to declare as functions, or an empty array if the request /// shouldn't declare any. - static func toolDefinitionsForDisabledFunctionCalling( - in request: LanguageModelExecutorGenerationRequest, - isAtToolCallingTurnLimit: Bool + static func toolDefinitionsForRequestWithoutEnabledTools( + in request: LanguageModelExecutorGenerationRequest ) -> [Transcript.ToolDefinition] { guard - isFunctionCallingDisabled( - toolCallingMode: request.generationOptions.toolCallingMode, - isAtToolCallingTurnLimit: isAtToolCallingTurnLimit - ), + request.enabledToolDefinitions.isEmpty, request.transcript.contains(where: { entry in guard case .toolCalls(let toolCalls) = entry else { return false } return !toolCalls.isEmpty @@ -181,16 +177,44 @@ ) throws -> [GeminiAPIDataModels.Tool]? { guard !enabledToolDefinitions.isEmpty else { return nil } - let declarations = try enabledToolDefinitions.map { toolDefinition in - try FunctionDeclaration { - $0.name = toolDefinition.name - $0.description = toolDefinition.description - $0.parametersJSONSchema = try toolDefinition.parameters.toGeminiJSONSchema() - } - } + let declarations = try enabledToolDefinitions.map(translateFunctionDeclaration) return [GeminiAPIDataModels.Tool { $0.functionDeclarations = declarations }] } + /// Translates tool definitions into a list of Gemini `Tool` objects, skipping any tool whose + /// definition can't be translated. + /// + /// Use this only when declaring the tools is best effort, such as when function calling is + /// disabled, so that one untranslatable tool doesn't drop the declarations of the others. + /// + /// - Parameter toolDefinitions: The tool definitions to declare as functions. + /// - Returns: A list containing a single `Tool` with the translatable function declarations, + /// or `nil` if none of the tool definitions can be translated. + static func translateToolsBestEffort( + _ toolDefinitions: [Transcript.ToolDefinition] + ) -> [GeminiAPIDataModels.Tool]? { + // A tool's schema can fail to translate, for example with an unsupported generation guide. + let declarations = toolDefinitions.compactMap { try? translateFunctionDeclaration($0) } + guard !declarations.isEmpty else { return nil } + + return [GeminiAPIDataModels.Tool { $0.functionDeclarations = declarations }] + } + + /// Translates a tool definition into a Gemini `FunctionDeclaration`. + /// + /// - Parameter toolDefinition: The tool definition to translate. + /// - Returns: A function declaration with the tool's name, description, and parameters. + /// - Throws: An error if schema translation fails. + static func translateFunctionDeclaration( + _ toolDefinition: Transcript.ToolDefinition + ) throws -> FunctionDeclaration { + try FunctionDeclaration { + $0.name = toolDefinition.name + $0.description = toolDefinition.description + $0.parametersJSONSchema = try toolDefinition.parameters.toGeminiJSONSchema() + } + } + /// Translates tool calling options into a Gemini `ToolConfig`. /// /// - Parameters: @@ -199,16 +223,16 @@ /// - hasFunctionDeclarations: Whether the request declares any functions. /// `functionCallingConfig` only governs function calling, so it is omitted entirely when /// there are none. - /// - isAtToolCallingTurnLimit: Whether the session has reached the maximum number of - /// tool-calling turns for the current prompt. If `true`, function calling is disabled - /// whatever `toolCallingMode` is, so that the model gives a final response. + /// - isFunctionCallingDisabled: Whether the request must not call functions (see + /// `isFunctionCallingDisabled(for:isAtToolCallingTurnLimit:)`). If `true`, function calling + /// is disabled whatever `toolCallingMode` is, so that the model gives a final response. /// - Returns: A `ToolConfig`, or `nil` if no configuration is needed. /// - Throws: `LanguageModelError.unsupportedCapability` if `toolCallingMode` is a mode that /// this version of the SDK does not recognize. static func translateToolConfig( toolCallingMode: GenerationOptions.ToolCallingMode?, hasFunctionDeclarations: Bool, - isAtToolCallingTurnLimit: Bool + isFunctionCallingDisabled: Bool ) throws -> ToolConfig? { // Computed independently of the `ToolConfig` check below to allow future options (such // as `includeServerSideToolInvocations`) when built-in tools are present without function @@ -241,13 +265,10 @@ } else { requestedMode = allowedMode } - // The turn limit disables function calling whatever the requested mode is, including - // `required`, which would otherwise keep calling tools indefinitely. + // Disabled function calling (such as at the turn limit) overrides the requested mode, + // including `required`, which would otherwise keep calling tools indefinitely. let callingMode = - isFunctionCallingDisabled( - toolCallingMode: toolCallingMode, - isAtToolCallingTurnLimit: isAtToolCallingTurnLimit - ) ? FunctionCallingConfig.Mode.none : requestedMode + isFunctionCallingDisabled ? FunctionCallingConfig.Mode.none : requestedMode functionCallingConfig = FunctionCallingConfig { $0.mode = callingMode } } else { functionCallingConfig = nil diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift index e38c10e0840..c98a6b1d8f7 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift @@ -888,7 +888,7 @@ @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) func sessionRespondDisablesFunctionCallingAtToolCallingTurnLimit() async throws { defer { MockHTTPURLProtocol.reset() } - let maximumTurns = GeminiLanguageModel.Executor.Configuration.defaultMaximumToolCallingTurns + let maximumTurns = GeminiLanguageModel.Executor.maximumToolCallingTurns try #require(maximumTurns == 10) let recordedRequests = try Self.mockToolCallingLoop { $0 <= maximumTurns } let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) @@ -913,7 +913,7 @@ @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) func functionCallAfterToolCallingTurnLimitThrowsInvalidResponse() async throws { defer { MockHTTPURLProtocol.reset() } - let maximumTurns = GeminiLanguageModel.Executor.Configuration.defaultMaximumToolCallingTurns + let maximumTurns = GeminiLanguageModel.Executor.maximumToolCallingTurns let recordedRequests = try Self.mockToolCallingLoop { _ in true } let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) session.transcriptErrorHandlingPolicy = .preserveTranscript @@ -936,7 +936,7 @@ @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) func newPromptResetsToolCallingTurnCount() async throws { defer { MockHTTPURLProtocol.reset() } - let maximumTurns = GeminiLanguageModel.Executor.Configuration.defaultMaximumToolCallingTurns + let maximumTurns = GeminiLanguageModel.Executor.maximumToolCallingTurns // Each prompt receives `maximumTurns` function calls, then text. let recordedRequests = try Self.mockToolCallingLoop { requestNumber in requestNumber % (maximumTurns + 1) != 0 @@ -958,7 +958,7 @@ @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) func parallelFunctionCallsCountAsOneToolCallingTurn() async throws { defer { MockHTTPURLProtocol.reset() } - let maximumTurns = GeminiLanguageModel.Executor.Configuration.defaultMaximumToolCallingTurns + let maximumTurns = GeminiLanguageModel.Executor.maximumToolCallingTurns let recordedRequests = try Self.mockToolCallingLoop(functionCallsPerResponse: 2) { $0 <= maximumTurns } @@ -1133,6 +1133,115 @@ #expect(session.transcript[(promptIndex + 1)...].isEmpty) } + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func promptAfterToolCallingTurnLimitErrorWithPreservedTranscriptSucceeds() async throws { + defer { MockHTTPURLProtocol.reset() } + let maximumTurns = GeminiLanguageModel.Executor.maximumToolCallingTurns + // The first prompt receives function calls until it fails at the turn limit. + let recordedRequests = try Self.mockToolCallingLoop { $0 <= maximumTurns + 1 } + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + session.transcriptErrorHandlingPolicy = .preserveTranscript + await #expect(throws: GeminiLanguageModel.Error.self) { + try await session.respond(to: "What is the weather in Paris?") + } + + let response = try await session.respond(to: "Summarize what you found.") + + #expect(response.content == Self.weatherResponseText) + let requests = recordedRequests.all + #expect(requests.count == maximumTurns + 2) + let lastRequest = try #require(requests.last) + #expect(lastRequest.toolConfig?.functionCallingConfig?.mode == .validated) + // The transcript ends with the last tool output, so the new prompt joins its user turn. + let lastContent = try #require(lastRequest.contents?.last) + #expect(lastContent.role == "user") + let lastParts = try #require(lastContent.parts) + #expect(lastParts.count == 2) + guard case .functionResponse(let functionResponse) = lastParts.first?.data else { + Issue.record("Expected a functionResponse part before the new prompt.") + return + } + #expect(functionResponse.name == "get_weather") + guard case .text(let promptText) = lastParts.last?.data else { + Issue.record("Expected the new prompt's text part.") + return + } + #expect(promptText == "Summarize what you found.") + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func rejectedFunctionCallDoesNotStreamAllowedCallOrTextInSameChunk() async throws { + defer { MockHTTPURLProtocol.reset() } + let recordedRequests = try Self.mockStreamResponses([ + Self.functionCallsChunk( + [ + (id: "call-1", name: "get_weather"), + (id: "call-2", name: "get_current_time"), + ], + leadingText: "Let me check." + ) + ]) + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + session.transcriptErrorHandlingPolicy = .preserveTranscript + + do { + _ = try await session.respond(to: "What are the weather and time in Paris?") + Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + #expect( + invalidResponse.debugDescription.contains( + "`get_current_time`, which isn't an enabled tool")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + + #expect(recordedRequests.all.count == 1) + let promptIndex = try #require( + session.transcript.firstIndex { entry in + guard case .prompt = entry else { return false } + return true + } + ) + #expect(session.transcript[(promptIndex + 1)...].isEmpty) + } + + /// Checks that a function call from an earlier chunk of a response, which has already been + /// streamed to the session, isn't kept or run when a later chunk is rejected, even when the + /// transcript is preserved. + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func rejectedFunctionCallInLaterChunkKeepsNoEarlierToolCalls() async throws { + defer { MockHTTPURLProtocol.reset() } + let recordedRequests = try Self.mockStreamResponses([ + Self.functionCallsChunk([(id: "call-1", name: "get_weather")], isFinal: false) + + Self.functionCallsChunk([(id: "call-2", name: "get_current_time")]) + ]) + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + session.transcriptErrorHandlingPolicy = .preserveTranscript + + do { + _ = try await session.respond(to: "What are the weather and time in Paris?") + Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + #expect( + invalidResponse.debugDescription.contains( + "`get_current_time`, which isn't an enabled tool")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + + #expect(recordedRequests.all.count == 1) + let promptIndex = try #require( + session.transcript.firstIndex { entry in + guard case .prompt = entry else { return false } + return true + } + ) + #expect(session.transcript[(promptIndex + 1)...].isEmpty) + } + /// A test case for `GeminiLanguageModel.Executor.functionCallRejectionReason`. @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) @available(tvOS, unavailable) @@ -1183,10 +1292,12 @@ isAtToolCallingTurnLimit: true, expectedReasonSubstring: "`disallowed`"), .init( toolCallingMode: .required, functionName: "get_weather", enablesWeatherTool: true, - isAtToolCallingTurnLimit: true, expectedReasonSubstring: "limit of 2 tool-calling turns"), + isAtToolCallingTurnLimit: true, + expectedReasonSubstring: "limit of 10 tool-calling turns"), .init( toolCallingMode: nil, functionName: "get_time", enablesWeatherTool: false, - isAtToolCallingTurnLimit: true, expectedReasonSubstring: "limit of 2 tool-calling turns"), + isAtToolCallingTurnLimit: true, + expectedReasonSubstring: "limit of 10 tool-calling turns"), .init( toolCallingMode: nil, functionName: "get_weather", enablesWeatherTool: false, isAtToolCallingTurnLimit: false, expectedReasonSubstring: "isn't an enabled tool"), @@ -1222,8 +1333,7 @@ let reason = GeminiLanguageModel.Executor.functionCallRejectionReason( forFunctionNamed: testCase.functionName, in: request, - isAtToolCallingTurnLimit: testCase.isAtToolCallingTurnLimit, - maximumToolCallingTurns: 2 + isAtToolCallingTurnLimit: testCase.isAtToolCallingTurnLimit ) if let expectedReasonSubstring = testCase.expectedReasonSubstring { @@ -1311,6 +1421,57 @@ return recordedRequests } + /// Mocks a model that responds to each request with a server-sent events payload. + /// + /// - Parameter ssePayloads: The payloads to respond with, in request order. Requests after + /// the last payload receive the last payload. + /// - Returns: The requests that the mock receives. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + private static func mockStreamResponses(_ ssePayloads: [String]) throws -> RecordedRequests { + try #require(!ssePayloads.isEmpty) + let expectedURL = try makeExpectedStreamURL() + let httpResponse = try HTTPURLResponse.mock( + url: expectedURL, + headerFields: ["Content-Type": "text/event-stream"] + ) + let recordedRequests = RecordedRequests() + MockHTTPURLProtocol.setHandler(for: expectedURL) { request, proto in + let body = try #require(request.httpBodyData) + let requestNumber = recordedRequests.append( + try JSONDecoder().decode(GenerateContentRequest.self, from: body) + ) + let payload = ssePayloads[min(requestNumber, ssePayloads.count) - 1] + proto.client?.urlProtocol(proto, didReceive: httpResponse, cacheStoragePolicy: .notAllowed) + proto.client?.urlProtocol(proto, didLoad: Data(payload.utf8)) + proto.client?.urlProtocolDidFinishLoading(proto) + } + return recordedRequests + } + + /// Returns a server-sent event for a streamed chunk with parallel function calls. + /// + /// - Parameters: + /// - calls: The identifier and function name of each function call, in order. + /// - leadingText: The text of a part before the function calls, or `nil` for none. + /// - isFinal: Whether the chunk ends the response with a `STOP` finish reason. + /// - Returns: The `data:` line and its terminating blank line. + private static func functionCallsChunk( + _ calls: [(id: String, name: String)], + leadingText: String? = nil, + isFinal: Bool = true + ) -> String { + let textParts = leadingText.map { [#"{"text": "\#($0)"}"#] } ?? [] + let functionCallParts = calls.map { call in + #"{"functionCall": {"id": "\#(call.id)", "name": "\#(call.name)", "# + + #""args": {"location": "Paris"}}}"# + } + let parts = textParts + functionCallParts + let finishReason = isFinal ? #", "finishReason": "STOP""# : "" + return #"data: {"candidates": [{"content": {"parts": [\#(parts.joined(separator: ", "))], "# + + #""role": "model"}\#(finishReason), "index": 0}]}"# + + "\n\n" + } + /// Returns the number of tool-calling turns (non-empty tool-calls entries) in a transcript. /// /// - Parameter transcript: The transcript to search. @@ -1336,7 +1497,6 @@ } @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) - private static func makeMockModel( modelResource: ModelResource = .gemini38Flash, endpointConfiguration: EndpointConfiguration = .geminiDeveloperAPI, diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiRequestTranslatorTests.swift b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiRequestTranslatorTests.swift index c22d9621ffd..6a9e9d1eae3 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiRequestTranslatorTests.swift +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiRequestTranslatorTests.swift @@ -227,22 +227,22 @@ let allowedConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: .allowed, hasFunctionDeclarations: true, - isAtToolCallingTurnLimit: false + isFunctionCallingDisabled: false ) let requiredConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: .required, hasFunctionDeclarations: true, - isAtToolCallingTurnLimit: false + isFunctionCallingDisabled: false ) let disallowedConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: .disallowed, hasFunctionDeclarations: true, - isAtToolCallingTurnLimit: false + isFunctionCallingDisabled: false ) let unsetConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: nil, hasFunctionDeclarations: true, - isAtToolCallingTurnLimit: false + isFunctionCallingDisabled: false ) #expect(allowedConfig?.functionCallingConfig?.mode == .validated) @@ -257,22 +257,22 @@ let allowedConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: .allowed, hasFunctionDeclarations: false, - isAtToolCallingTurnLimit: false + isFunctionCallingDisabled: false ) let requiredConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: .required, hasFunctionDeclarations: false, - isAtToolCallingTurnLimit: false + isFunctionCallingDisabled: false ) let disallowedConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: .disallowed, hasFunctionDeclarations: false, - isAtToolCallingTurnLimit: false + isFunctionCallingDisabled: false ) let nilConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: nil, hasFunctionDeclarations: false, - isAtToolCallingTurnLimit: false + isFunctionCallingDisabled: false ) #expect(allowedConfig == nil) @@ -414,7 +414,7 @@ .disallowed, ]) @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) - func translatesToolConfigAtToolCallingTurnLimitToNone( + func translatesToolConfigWithFunctionCallingDisabledToNone( toolCallingMode: GenerationOptions.ToolCallingMode? ) throws { let encoder = JSONEncoder() @@ -423,7 +423,7 @@ let toolConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: toolCallingMode, hasFunctionDeclarations: true, - isAtToolCallingTurnLimit: true + isFunctionCallingDisabled: true ) let mode = try #require(toolConfig?.functionCallingConfig?.mode) @@ -463,11 +463,11 @@ @Test @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) - func translatesToolConfigAtToolCallingTurnLimitWithoutDeclarationsReturnsNil() throws { + func translatesToolConfigWithFunctionCallingDisabledWithoutDeclarationsReturnsNil() throws { let toolConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: .required, hasFunctionDeclarations: false, - isAtToolCallingTurnLimit: true + isFunctionCallingDisabled: true ) #expect(toolConfig == nil) @@ -571,6 +571,80 @@ #expect(result.toolConfig == nil) } + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesDisallowedRequestDeclaresOnlyTranslatableInstructionsTools() throws { + let postalCodeToolDefinition = Transcript.ToolDefinition( + name: "lookUpPostalCode", + description: "Look up a postal code.", + parameters: PostalCodeArguments.generationSchema + ) + let parisCall = try Self.makeToolCall(id: "call-1", city: "Paris") + let request = LanguageModelExecutorGenerationRequest( + id: UUID(), + transcript: Transcript( + entries: [ + .instructions( + Transcript.Instructions( + segments: [], + toolDefinitions: [Self.makeWeatherToolDefinition(), postalCodeToolDefinition] + ) + ), + .prompt( + Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Paris?"))])), + .toolCalls(Transcript.ToolCalls(id: "calls-1", [parisCall])), + .toolOutput(Self.makeToolOutput(id: "call-1")), + ] + ), + enabledTools: [], + schema: nil, + generationOptions: GenerationOptions(toolCallingMode: .disallowed), + contextOptions: ContextOptions(), + metadata: [:] + ) + + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) + + let functionDeclarations = try #require(result.tools?.first?.functionDeclarations) + #expect(functionDeclarations.map(\.name) == ["getWeather"]) + #expect(result.toolConfig?.functionCallingConfig?.mode == FunctionCallingConfig.Mode.none) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesRequestWithUntranslatableEnabledToolThrows() { + let request = LanguageModelExecutorGenerationRequest( + id: UUID(), + transcript: Transcript( + entries: [ + .prompt( + Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "94043?"))])) + ] + ), + enabledTools: [ + Self.makeWeatherToolDefinition(), + Transcript.ToolDefinition( + name: "lookUpPostalCode", + description: "Look up a postal code.", + parameters: PostalCodeArguments.generationSchema + ), + ], + schema: nil, + generationOptions: GenerationOptions(toolCallingMode: .allowed), + contextOptions: ContextOptions(), + metadata: [:] + ) + + do { + _ = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) + Issue.record("Expected unsupportedGenerationGuide error.") + } catch LanguageModelError.unsupportedGenerationGuide(let error) { + #expect(error.debugDescription.contains("pattern")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + } + @Test @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) func translatesDisallowedRequestWithoutToolCallsOmitsTools() throws { @@ -597,9 +671,15 @@ #expect(result.toolConfig == nil) } - @Test + @Test(arguments: [ + GenerationOptions.ToolCallingMode?.none, + .allowed, + .required, + ]) @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) - func translatesRequestWithoutEnabledToolsAfterToolCallsOmitsTools() throws { + func translatesRequestWithoutEnabledToolsAfterToolCallsDeclaresInstructionsToolsWithNone( + toolCallingMode: GenerationOptions.ToolCallingMode? + ) throws { let toolDefinition = Self.makeWeatherToolDefinition() let parisCall = try Self.makeToolCall(id: "call-1", city: "Paris") let request = LanguageModelExecutorGenerationRequest( @@ -615,19 +695,95 @@ ), enabledTools: [], schema: nil, - generationOptions: GenerationOptions(toolCallingMode: .allowed), + generationOptions: GenerationOptions(toolCallingMode: toolCallingMode), contextOptions: ContextOptions(), metadata: [:] ) let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) - #expect(result.tools == nil) - #expect(result.toolConfig == nil) + let functionDeclarations = try #require(result.tools?.first?.functionDeclarations) + #expect(functionDeclarations.map(\.name) == ["getWeather"]) + #expect(result.toolConfig?.functionCallingConfig?.mode == FunctionCallingConfig.Mode.none) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesRequestWithoutEnabledToolsDeclaresMostRecentInstructionsTools() throws { + let request = try Self.makeRequestWithoutEnabledTools( + olderInstructionsTools: [Self.makeWeatherToolDefinition()], + newerInstructionsTools: [Self.makeTimeToolDefinition()] + ) + + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) + + let functionDeclarations = try #require(result.tools?.first?.functionDeclarations) + #expect(functionDeclarations.map(\.name) == ["getTime"]) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesRequestWithoutEnabledToolsSkipsInstructionsWithoutTools() throws { + let request = try Self.makeRequestWithoutEnabledTools( + olderInstructionsTools: [Self.makeWeatherToolDefinition()], + newerInstructionsTools: [] + ) + + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) + + let functionDeclarations = try #require(result.tools?.first?.functionDeclarations) + #expect(functionDeclarations.map(\.name) == ["getWeather"]) } // MARK: - Helper Methods + /// Returns a request that enables no tools, after a tool call, with two instructions entries. + /// + /// - Parameters: + /// - olderInstructionsTools: The tools listed by the instructions before the tool call. + /// - newerInstructionsTools: The tools listed by the instructions after the tool call. + /// - Returns: A request with the `allowed` tool calling mode and no enabled tools. + /// - Throws: An error if the tool call's arguments can't be parsed. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + private static func makeRequestWithoutEnabledTools( + olderInstructionsTools: [Transcript.ToolDefinition], + newerInstructionsTools: [Transcript.ToolDefinition] + ) throws -> LanguageModelExecutorGenerationRequest { + LanguageModelExecutorGenerationRequest( + id: UUID(), + transcript: Transcript( + entries: [ + .instructions( + Transcript.Instructions(segments: [], toolDefinitions: olderInstructionsTools)), + .prompt( + Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Paris?"))])), + .toolCalls( + Transcript.ToolCalls(id: "calls-1", [try makeToolCall(id: "call-1", city: "Paris")])), + .toolOutput(makeToolOutput(id: "call-1")), + .instructions( + Transcript.Instructions(segments: [], toolDefinitions: newerInstructionsTools)), + .prompt( + Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Time?"))])), + ] + ), + enabledTools: [], + schema: nil, + generationOptions: GenerationOptions(toolCallingMode: .allowed), + contextOptions: ContextOptions(), + metadata: [:] + ) + } + + /// Returns the definition of a tool that gets the current time. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + private static func makeTimeToolDefinition() -> Transcript.ToolDefinition { + Transcript.ToolDefinition( + name: "getTime", + description: "Get the current time.", + parameters: LocationArguments.generationSchema + ) + } + /// Returns a `getWeather` tool call for a city. /// /// - Parameters: diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ToolCallingIntegrationTests.swift b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ToolCallingIntegrationTests.swift index 8ccca39eb4d..590e0ef2621 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ToolCallingIntegrationTests.swift +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ToolCallingIntegrationTests.swift @@ -314,7 +314,7 @@ func sessionRespondRequiredToolCallingStopsAtToolCallingTurnLimit( backend: IntegrationTestingBackend ) async throws { - let maximumTurns = GeminiLanguageModel.Executor.Configuration.defaultMaximumToolCallingTurns + let maximumTurns = GeminiLanguageModel.Executor.maximumToolCallingTurns let model = try await backend.makeModel() let tool = CountingWeatherTool() let session = LanguageModelSession(model: model, tools: [tool]) From a1cad15e2871fe15bf3a9f1ebfe2332d3bbf4151 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Wed, 11 Nov 2026 18:02:08 -0500 Subject: [PATCH 8/8] fix(ai): reject GLM tool-call finish reasons With function calling disabled, Gemini can block its own function call and stop with the `UNEXPECTED_TOOL_CALL` finish reason (or stop with `MALFORMED_FUNCTION_CALL`), typically with no content, instead of returning a function call part. The executor only checked function call parts, so the session ended with an empty response. Throw `GeminiLanguageModel.Error.invalidResponse` for these finish reasons, as for a function call when function calling is disabled. Also use the `NONE` mode without checking the requested tool calling mode when function calling is disabled, so that a tool calling mode from a later OS release no longer throws `unsupportedCapability` for a request without enabled tools after earlier tool calls. --- .../GeminiLanguageModelExecutor.swift | 27 +++++++++ .../GeminiRequestTranslator.swift | 31 +++++------ .../GeminiLanguageModelTests.swift | 55 +++++++++++++++++++ 3 files changed, 97 insertions(+), 16 deletions(-) diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift index 4551326630f..3c271569272 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift @@ -143,6 +143,7 @@ try Task.checkCancellation() try GeminiErrorMapper.checkGuardrails(in: chunk) + try Self.validateFinishReason(of: chunk) if let candidates = chunk.candidates, let candidate = candidates.first, let parts = candidate.content?.parts @@ -256,6 +257,32 @@ } } + /// Checks that a response chunk didn't stop because of a function call that Gemini blocked. + /// + /// Instead of returning a function call part, Gemini can stop a response with the + /// `UNEXPECTED_TOOL_CALL` finish reason (for example, when it tries to call a function while + /// function calling is disabled) or the `MALFORMED_FUNCTION_CALL` finish reason, typically + /// with no content. + /// + /// - Parameter chunk: A response chunk from Gemini. + /// - Throws: `GeminiLanguageModel.Error.invalidResponse` if the chunk's finish reason is + /// `UNEXPECTED_TOOL_CALL` or `MALFORMED_FUNCTION_CALL`. + static func validateFinishReason(of chunk: GenerateContentResponse) throws { + guard let finishReason = chunk.candidates?.first?.finishReason, + finishReason == .unexpectedToolCall || finishReason == .malformedFunctionCall + else { + return + } + throw GeminiLanguageModel.Error.invalidResponse( + GeminiLanguageModel.Error.InvalidResponse( + debugDescription: """ + Gemini stopped with the `\(finishReason.rawValue)` finish reason instead of \ + returning a valid function call. + """ + ) + ) + } + /// Checks that the function calls in a response can be passed to the session. /// /// - Parameters: diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift index 9d38ed1e79c..ac237dfb1d4 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift @@ -227,8 +227,8 @@ /// `isFunctionCallingDisabled(for:isAtToolCallingTurnLimit:)`). If `true`, function calling /// is disabled whatever `toolCallingMode` is, so that the model gives a final response. /// - Returns: A `ToolConfig`, or `nil` if no configuration is needed. - /// - Throws: `LanguageModelError.unsupportedCapability` if `toolCallingMode` is a mode that - /// this version of the SDK does not recognize. + /// - Throws: `LanguageModelError.unsupportedCapability` if function calling isn't disabled and + /// `toolCallingMode` is a mode that this version of the SDK does not recognize. static func translateToolConfig( toolCallingMode: GenerationOptions.ToolCallingMode?, hasFunctionDeclarations: Bool, @@ -239,19 +239,19 @@ // declarations. let functionCallingConfig: FunctionCallingConfig? if hasFunctionDeclarations { - // `GenerationOptions.toolCallingMode` is optional and defaults to `nil` unless set by the - // caller. Omitting `functionCallingConfig` in that case causes the backend to default to - // `AUTO`, so `nil` maps to the allowed mode. - let allowedMode = FunctionCallingConfig.Mode.validated - let requestedMode: FunctionCallingConfig.Mode - if let mode = toolCallingMode { + let callingMode: FunctionCallingConfig.Mode + if isFunctionCallingDisabled { + // Disabled function calling (such as at the turn limit) overrides the requested mode, + // including `required`, which would otherwise keep calling tools indefinitely. + callingMode = FunctionCallingConfig.Mode.none + } else if let mode = toolCallingMode { switch mode.kind { case .allowed: - requestedMode = allowedMode + callingMode = .validated case .required: - requestedMode = .any + callingMode = .any case .disallowed: - requestedMode = FunctionCallingConfig.Mode.none + callingMode = FunctionCallingConfig.Mode.none @unknown default: // A mode added in a later OS release; its semantics are unknown, so approximating it // with the allowed mode could silently diverge from what the caller requested. @@ -263,12 +263,11 @@ ) } } else { - requestedMode = allowedMode + // `GenerationOptions.toolCallingMode` is optional and defaults to `nil` unless set by + // the caller. Omitting `functionCallingConfig` in that case causes the backend to + // default to `AUTO`, so `nil` maps to the allowed mode. + callingMode = .validated } - // Disabled function calling (such as at the turn limit) overrides the requested mode, - // including `required`, which would otherwise keep calling tools indefinitely. - let callingMode = - isFunctionCallingDisabled ? FunctionCallingConfig.Mode.none : requestedMode functionCallingConfig = FunctionCallingConfig { $0.mode = callingMode } } else { functionCallingConfig = nil diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift index c98a6b1d8f7..7db3a9a574c 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift @@ -1242,6 +1242,53 @@ #expect(session.transcript[(promptIndex + 1)...].isEmpty) } + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func unexpectedToolCallAfterToolCallingTurnLimitThrowsInvalidResponse() async throws { + defer { MockHTTPURLProtocol.reset() } + let maximumTurns = GeminiLanguageModel.Executor.maximumToolCallingTurns + let functionCallPayloads = (1...maximumTurns).map { requestNumber in + Self.functionCallsChunk([(id: "call-\(requestNumber)", name: "get_weather")]) + } + let recordedRequests = try Self.mockStreamResponses( + functionCallPayloads + [Self.finishReasonChunk("UNEXPECTED_TOOL_CALL")] + ) + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + + do { + _ = try await session.respond(to: "What is the weather in Paris?") + Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + #expect(invalidResponse.debugDescription.contains("`UNEXPECTED_TOOL_CALL`")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + + let requests = recordedRequests.all + #expect(requests.count == maximumTurns + 1) + #expect( + requests.last?.toolConfig?.functionCallingConfig?.mode == FunctionCallingConfig.Mode.none) + } + + @Test(arguments: ["UNEXPECTED_TOOL_CALL", "MALFORMED_FUNCTION_CALL"]) + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func functionCallFinishReasonThrowsInvalidResponse(finishReason: String) async throws { + defer { MockHTTPURLProtocol.reset() } + let recordedRequests = try Self.mockStreamResponses([Self.finishReasonChunk(finishReason)]) + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + + do { + _ = try await session.respond(to: "What is the weather in Paris?") + Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + #expect(invalidResponse.debugDescription.contains("`\(finishReason)`")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + + #expect(recordedRequests.all.count == 1) + } + /// A test case for `GeminiLanguageModel.Executor.functionCallRejectionReason`. @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) @available(tvOS, unavailable) @@ -1448,6 +1495,14 @@ return recordedRequests } + /// Returns a server-sent event for a streamed chunk with a finish reason and no content. + /// + /// - Parameter finishReason: The raw value of the chunk's finish reason. + /// - Returns: The `data:` line and its terminating blank line. + private static func finishReasonChunk(_ finishReason: String) -> String { + #"data: {"candidates": [{"finishReason": "\#(finishReason)", "index": 0}]}"# + "\n\n" + } + /// Returns a server-sent event for a streamed chunk with parallel function calls. /// /// - Parameters: