diff --git a/FirebaseAI/CHANGELOG.md b/FirebaseAI/CHANGELOG.md index 87e1d5869cf..07ccab593ee 100644 --- a/FirebaseAI/CHANGELOG.md +++ b/FirebaseAI/CHANGELOG.md @@ -8,6 +8,15 @@ `start` and `end`, removed `VoiceActivityType.unknown` from the public API, and made `LiveServerVoiceActivity.audioOffset` a non-optional `TimeInterval` (defaulting to `0`). +- [changed] `GeminiLanguageModel` now limits each prompt to 10 tool-calling + turns, including when `toolCallingMode` is `.required`. After 10 turns, the + next request asks Gemini for a final response without calling tools; if + Gemini calls a tool anyway, the session throws + `GeminiLanguageModel.Error.invalidResponse`. (#17043) +- [fixed] Fixed `GeminiLanguageModel` running a tool, or throwing a + `GeneratedContent.ParsingError`, when Gemini called a tool that wasn't enabled + for the request, including when `toolCallingMode` was `.disallowed`. It now + throws `GeminiLanguageModel.Error.invalidResponse` instead. (#17043) - [fixed] Fixed a crash at launch on iOS 17 and earlier, and the corresponding macOS, tvOS, watchOS, and visionOS versions, in apps built with Xcode 27 when the app or a library (such as Realm) enumerated Objective-C classes with diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift index 2ec41890b80..ac54649124f 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModel.swift @@ -42,7 +42,7 @@ /// [`LanguageModel`](https://developer.apple.com/documentation/foundationmodels/languagemodel) /// protocol and can be used with the Foundation Models framework when [initializing](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/init%28model:tools:instructions:%29) /// a [`LanguageModelSession`](https://developer.apple.com/documentation/foundationmodels/languagemodelsession), - /// or by [setting the model](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/dynamicprofile/model%28_%29) + /// or by [setting the model](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/dynamicprofile/model%28_:%29) /// on a [`DynamicProfile`](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/dynamicprofile). /// /// For more details on using Gemini to generate content with the Foundation Models framework, @@ -107,6 +107,47 @@ /// images in the session are re-sent with every request. /// - Image files are read again for every request in the session, so a file must remain /// available for as long as the session that references it is used. + /// + /// ### Tool calling + /// + /// With the `toolCalling` capability, the model can call the tools that you provide to the + /// session. To limit cost when the model keeps calling tools, the model can make at most 10 + /// tool-calling turns in response to each prompt. A tool-calling turn is one response that + /// calls one or more tools. + /// + /// - After 10 tool-calling turns, the next request tells Gemini not to call tools, so that + /// the model gives a final response. + /// - If Gemini calls a tool anyway, the session throws + /// ``GeminiLanguageModel/Error/invalidResponse(_:)``. + /// + /// The limit also applies when the tool calling mode is `required`. With `required`, the model + /// otherwise keeps calling tools until a tool throws an error or the mode changes. To stop + /// sooner, change the mode with the [`toolCallingMode(_:)`](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/dynamicprofile/toolcallingmode%28_:%29) + /// modifier of a `LanguageModelSession.DynamicProfile`, for example from `required` to + /// `allowed` after the first tool call, as shown in [`GenerationOptions.ToolCallingMode`](https://developer.apple.com/documentation/foundationmodels/generationoptions/toolcallingmode-swift.struct). + /// + /// The limit is counted from the transcript that the session sends to the model, after any + /// [`historyTransform(_:)`](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/dynamicprofile/historytransform%28_:%29) + /// of a `LanguageModelSession.DynamicProfile`. A transform that removes earlier tool calls can + /// stop the limit from being reached, and a transform that removes prompts can make the limit + /// apply across prompts. Each tool-calling turn adds several transcript entries, such as the + /// tool calls, their outputs, and the model's reasoning, so a transform that keeps a fixed + /// number of recent entries keeps far fewer tool-calling turns than entries. + /// + /// If Gemini calls a tool when the tool calling mode is `disallowed`, or calls a tool that + /// isn't enabled for the request, the session throws + /// ``GeminiLanguageModel/Error/invalidResponse(_:)`` and the tool doesn't run. + /// + /// > Warning: Gemini doesn't always follow the `disallowed` tool calling mode, especially after + /// > several tool calls in response to a prompt. Changing the mode to `disallowed` partway + /// > through a prompt can make the session throw + /// > ``GeminiLanguageModel/Error/invalidResponse(_:)`` instead of returning a response. + /// + /// By default, when the session throws an error, it reverts its transcript to how it was before + /// the prompt, even if tools have already run. To keep the tool calls and their outputs, set + /// the session's [`transcriptErrorHandlingPolicy`](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/transcripterrorhandlingpolicy) + /// to `preserveTranscript`, or use the [`transcriptErrorHandlingPolicy(_:)`](https://developer.apple.com/documentation/foundationmodels/languagemodelsession/dynamicprofile/transcripterrorhandlingpolicy%28_:%29) + /// modifier of a `LanguageModelSession.DynamicProfile`. public var capabilities: LanguageModelCapabilities { LanguageModelCapabilities([ .guidedGeneration, diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift index f5446a27bfa..3c271569272 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiLanguageModelExecutor.swift @@ -66,6 +66,14 @@ } } + /// The maximum number of tool-calling turns allowed in response to a single prompt. + /// + /// A tool-calling turn is a model response that calls one or more tools. When this limit is + /// reached, the next request tells Gemini not to call functions; if it returns a function + /// call anyway, the executor throws `GeminiLanguageModel.Error.invalidResponse`. Keep the + /// `GeminiLanguageModel.capabilities` docs and the changelog in sync with this value. + static let maximumToolCallingTurns = 10 + private let configuration: Configuration /// Initializes a new executor with the specified configuration. @@ -88,7 +96,15 @@ model: GeminiLanguageModel, streamingInto channel: LanguageModelExecutorGenerationChannel ) async throws { - let generateRequest = try GeminiRequestTranslator.translate(request) + // The session keeps calling tools for as long as the model returns function calls, so + // limit the number of tool-calling turns to bound the number of requests. + let isAtToolCallingTurnLimit = + GeminiRequestTranslator.toolCallingTurnCount(in: request.transcript) + >= Self.maximumToolCallingTurns + let generateRequest = try GeminiRequestTranslator.translate( + request, + isAtToolCallingTurnLimit: isAtToolCallingTurnLimit + ) let client = GeminiAPIClient( modelResource: configuration.modelResource, @@ -127,10 +143,22 @@ try Task.checkCancellation() try GeminiErrorMapper.checkGuardrails(in: chunk) + try Self.validateFinishReason(of: chunk) if let candidates = chunk.candidates, let candidate = candidates.first, let parts = candidate.content?.parts { + // Check every function call in the chunk before streaming any of the chunk's parts, + // so that a rejected function call's own parts (such as its thought signature) + // aren't streamed. Parts from earlier chunks have already been streamed: when this + // throws, the session discards their tool calls, but with the `preserveTranscript` + // policy it keeps their text and reasoning. + try Self.validateFunctionCalls( + in: parts, + for: request, + isAtToolCallingTurnLimit: isAtToolCallingTurnLimit + ) + for part in parts { if let thoughtSignature = part.thoughtSignature, !thoughtSignature.isEmpty { let signatureData = Data(thoughtSignature.utf8) @@ -167,13 +195,9 @@ } if case .functionCall(let call) = part.data { - guard let name = call.name else { - throw GeminiLanguageModel.Error.invalidResponse( - GeminiLanguageModel.Error.InvalidResponse( - debugDescription: "Gemini returned a function call without a name." - ) - ) - } + // `validateFunctionCalls` has already checked that every function call in this + // chunk has a name and is allowed for this request, so this doesn't throw. + let name = try Self.functionName(of: call) let callID = call.id ?? UUID().uuidString let argsString: String if let args = call.args, !args.isEmpty { @@ -232,6 +256,117 @@ throw GeminiErrorMapper.map(error) } } + + /// Checks that a response chunk didn't stop because of a function call that Gemini blocked. + /// + /// Instead of returning a function call part, Gemini can stop a response with the + /// `UNEXPECTED_TOOL_CALL` finish reason (for example, when it tries to call a function while + /// function calling is disabled) or the `MALFORMED_FUNCTION_CALL` finish reason, typically + /// with no content. + /// + /// - Parameter chunk: A response chunk from Gemini. + /// - Throws: `GeminiLanguageModel.Error.invalidResponse` if the chunk's finish reason is + /// `UNEXPECTED_TOOL_CALL` or `MALFORMED_FUNCTION_CALL`. + static func validateFinishReason(of chunk: GenerateContentResponse) throws { + guard let finishReason = chunk.candidates?.first?.finishReason, + finishReason == .unexpectedToolCall || finishReason == .malformedFunctionCall + else { + return + } + throw GeminiLanguageModel.Error.invalidResponse( + GeminiLanguageModel.Error.InvalidResponse( + debugDescription: """ + Gemini stopped with the `\(finishReason.rawValue)` finish reason instead of \ + returning a valid function call. + """ + ) + ) + } + + /// Checks that the function calls in a response can be passed to the session. + /// + /// - Parameters: + /// - parts: The parts of a response chunk from Gemini. + /// - request: The generation request from the Foundation Models session. + /// - isAtToolCallingTurnLimit: Whether the session has reached the maximum number of + /// tool-calling turns for the current prompt. + /// - Throws: `GeminiLanguageModel.Error.invalidResponse` if a function call has no name or + /// if the request doesn't allow it (see `functionCallRejectionReason`). + static func validateFunctionCalls( + in parts: [Part], + for request: LanguageModelExecutorGenerationRequest, + isAtToolCallingTurnLimit: Bool + ) throws { + for part in parts { + guard case .functionCall(let call) = part.data else { continue } + let name = try functionName(of: call) + if let reason = functionCallRejectionReason( + forFunctionNamed: name, + in: request, + isAtToolCallingTurnLimit: isAtToolCallingTurnLimit + ) { + throw GeminiLanguageModel.Error.invalidResponse( + GeminiLanguageModel.Error.InvalidResponse(debugDescription: reason) + ) + } + } + } + + /// Returns the name of the function that a function call from Gemini calls. + /// + /// - Parameter call: A function call from Gemini. + /// - Returns: The non-empty name of the function. + /// - Throws: `GeminiLanguageModel.Error.invalidResponse` if the function call has no name or + /// an empty name. + static func functionName(of call: FunctionCall) throws -> String { + guard let name = call.name, !name.isEmpty else { + throw GeminiLanguageModel.Error.invalidResponse( + GeminiLanguageModel.Error.InvalidResponse( + debugDescription: "Gemini returned a function call without a name." + ) + ) + } + return name + } + + /// Returns the reason to reject a function call from Gemini, or `nil` if the session can run + /// it. + /// + /// `LanguageModelSession` runs any tool call that names one of the session's tools, even + /// when the tool calling mode is `disallowed` or the tool isn't enabled for the request. + /// Gemini sometimes returns a function call when it's told not to call functions, or for a + /// function that the request doesn't declare, so the executor only passes on function calls + /// to the request's enabled tools while function calling is enabled. + /// + /// - Parameters: + /// - name: The name of the function that Gemini called. + /// - request: The generation request from the Foundation Models session. + /// - isAtToolCallingTurnLimit: Whether the session has reached the maximum number of + /// tool-calling turns for the current prompt. + /// - Returns: The reason to reject the function call, or `nil` if the session can run it. + static func functionCallRejectionReason( + forFunctionNamed name: String, + in request: LanguageModelExecutorGenerationRequest, + isAtToolCallingTurnLimit: Bool + ) -> String? { + if request.generationOptions.toolCallingMode?.kind == .disallowed { + return """ + Gemini returned a function call to `\(name)` when the tool calling mode was \ + `disallowed`. + """ + } + if isAtToolCallingTurnLimit { + return """ + Gemini returned a function call to `\(name)` after function calling was disabled \ + because the limit of \(maximumToolCallingTurns) tool-calling turns per prompt was \ + reached. + """ + } + if !request.enabledToolDefinitions.contains(where: { $0.name == name }) { + return "Gemini returned a function call to `\(name)`, which isn't an enabled tool." + } + return nil + } } } #endif // compiler(>=6.4) && (FoundationModelsFake || canImport(FoundationModels)) diff --git a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift index ac6be97891a..ac237dfb1d4 100644 --- a/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift +++ b/GeminiLanguageModel/Sources/GeminiLanguageModel/GeminiRequestTranslator.swift @@ -27,17 +27,30 @@ enum GeminiRequestTranslator { /// Translates a `LanguageModelExecutorGenerationRequest` into a `GenerateContentRequest`. /// - /// - Parameter request: The generation request from the Foundation Models session. + /// - Parameters: + /// - request: The generation request from the Foundation Models session. + /// - isAtToolCallingTurnLimit: Whether the session has reached the maximum number of + /// tool-calling turns for the current prompt. If `true`, the request tells Gemini not to + /// call functions. /// - Returns: A `GenerateContentRequest` configured for the Gemini API. /// - Throws: An error if transcript or schema translation fails, or if the request specifies /// an unrecognized tool calling mode. static func translate( - _ request: LanguageModelExecutorGenerationRequest + _ request: LanguageModelExecutorGenerationRequest, + isAtToolCallingTurnLimit: Bool ) throws -> GenerateContentRequest { let (contents, systemInstruction) = try GeminiTranscriptTranslator.translate( request.transcript ) - let tools = try translateTools(request.enabledToolDefinitions) + let tools: [GeminiAPIDataModels.Tool]? + if request.enabledToolDefinitions.isEmpty { + // Declaring the tools in this case is best effort: function calling is disabled, so skip + // any tool whose schema can't be translated instead of failing the request. If no tool + // can be translated, the request is sent without declarations. + tools = translateToolsBestEffort(toolDefinitionsForRequestWithoutEnabledTools(in: request)) + } else { + tools = try translateTools(request.enabledToolDefinitions) + } let hasFunctionDeclarations = tools?.contains { $0.functionDeclarations?.isEmpty == false } ?? false @@ -47,12 +60,97 @@ $0.tools = tools $0.toolConfig = try translateToolConfig( toolCallingMode: request.generationOptions.toolCallingMode, - hasFunctionDeclarations: hasFunctionDeclarations + hasFunctionDeclarations: hasFunctionDeclarations, + isFunctionCallingDisabled: isFunctionCallingDisabled( + for: request, + isAtToolCallingTurnLimit: isAtToolCallingTurnLimit + ) ) $0.generationConfig = try translateGenerationConfig(schema: request.schema) } } + /// Returns whether a request must not call functions. + /// + /// Function calling is disabled when the tool calling mode is `disallowed`, when the request + /// enables no tools (whatever the tool calling mode is), or when the session has reached the + /// maximum number of tool-calling turns for the current prompt. + /// + /// - Parameters: + /// - request: The generation request from the Foundation Models session. + /// - isAtToolCallingTurnLimit: Whether the session has reached the maximum number of + /// tool-calling turns for the current prompt. + /// - Returns: `true` if the request must not call functions. + static func isFunctionCallingDisabled( + for request: LanguageModelExecutorGenerationRequest, + isAtToolCallingTurnLimit: Bool + ) -> Bool { + isAtToolCallingTurnLimit + || request.enabledToolDefinitions.isEmpty + || request.generationOptions.toolCallingMode?.kind == .disallowed + } + + /// Returns the tool definitions to declare as functions in a request that enables no tools. + /// + /// A request that enables no tools disables function calling, whatever the tool calling mode + /// is. The session enables no tools with the `disallowed` tool calling mode, and can also + /// disable every tool for a request, for example with a `LanguageModelSession.DynamicProfile`. + /// A transcript with earlier tool calls would then refer to functions that the request + /// doesn't declare. If the transcript has tool calls, this returns the tool definitions from + /// the most recent instructions entry that lists tools, so that the request declares them + /// with function calling disabled, matching the final request after the turn limit is + /// reached. + /// + /// Instructions entries only list the tools whose schema is included in instructions, from + /// the session's tools when the entry was added. Function calling is disabled, so any + /// function call is rejected whether or not its function is declared. + /// + /// - Parameter request: The generation request from the Foundation Models session. + /// - Returns: The tool definitions to declare as functions, or an empty array if the request + /// shouldn't declare any. + static func toolDefinitionsForRequestWithoutEnabledTools( + in request: LanguageModelExecutorGenerationRequest + ) -> [Transcript.ToolDefinition] { + guard + request.enabledToolDefinitions.isEmpty, + request.transcript.contains(where: { entry in + guard case .toolCalls(let toolCalls) = entry else { return false } + return !toolCalls.isEmpty + }) + else { + return [] + } + for entry in request.transcript.reversed() { + if case .instructions(let instructions) = entry, !instructions.toolDefinitions.isEmpty { + return instructions.toolDefinitions + } + } + return [] + } + + /// Returns the number of tool-calling turns since the most recent prompt in a transcript. + /// + /// A tool-calling turn is a model response that calls one or more tools. The session records + /// each one as a single `Transcript.Entry.toolCalls` entry, so parallel tool calls count as + /// one turn. Each prompt starts a new count. + /// + /// - Parameter transcript: The transcript of the session. + /// - Returns: The number of tool-calling turns since the most recent prompt. + static func toolCallingTurnCount(in transcript: Transcript) -> Int { + var turnCount = 0 + for entry in transcript.reversed() { + switch entry { + case .prompt: + return turnCount + case .toolCalls(let toolCalls) where !toolCalls.isEmpty: + turnCount += 1 + default: + break + } + } + return turnCount + } + /// Translates an optional `GenerationSchema` into a Gemini `GenerationConfig`. /// /// - Parameter schema: An optional generation schema specifying structured output constraints. @@ -79,16 +177,44 @@ ) throws -> [GeminiAPIDataModels.Tool]? { guard !enabledToolDefinitions.isEmpty else { return nil } - let declarations = try enabledToolDefinitions.map { toolDefinition in - try FunctionDeclaration { - $0.name = toolDefinition.name - $0.description = toolDefinition.description - $0.parametersJSONSchema = try toolDefinition.parameters.toGeminiJSONSchema() - } - } + let declarations = try enabledToolDefinitions.map(translateFunctionDeclaration) + return [GeminiAPIDataModels.Tool { $0.functionDeclarations = declarations }] + } + + /// Translates tool definitions into a list of Gemini `Tool` objects, skipping any tool whose + /// definition can't be translated. + /// + /// Use this only when declaring the tools is best effort, such as when function calling is + /// disabled, so that one untranslatable tool doesn't drop the declarations of the others. + /// + /// - Parameter toolDefinitions: The tool definitions to declare as functions. + /// - Returns: A list containing a single `Tool` with the translatable function declarations, + /// or `nil` if none of the tool definitions can be translated. + static func translateToolsBestEffort( + _ toolDefinitions: [Transcript.ToolDefinition] + ) -> [GeminiAPIDataModels.Tool]? { + // A tool's schema can fail to translate, for example with an unsupported generation guide. + let declarations = toolDefinitions.compactMap { try? translateFunctionDeclaration($0) } + guard !declarations.isEmpty else { return nil } + return [GeminiAPIDataModels.Tool { $0.functionDeclarations = declarations }] } + /// Translates a tool definition into a Gemini `FunctionDeclaration`. + /// + /// - Parameter toolDefinition: The tool definition to translate. + /// - Returns: A function declaration with the tool's name, description, and parameters. + /// - Throws: An error if schema translation fails. + static func translateFunctionDeclaration( + _ toolDefinition: Transcript.ToolDefinition + ) throws -> FunctionDeclaration { + try FunctionDeclaration { + $0.name = toolDefinition.name + $0.description = toolDefinition.description + $0.parametersJSONSchema = try toolDefinition.parameters.toGeminiJSONSchema() + } + } + /// Translates tool calling options into a Gemini `ToolConfig`. /// /// - Parameters: @@ -97,27 +223,31 @@ /// - hasFunctionDeclarations: Whether the request declares any functions. /// `functionCallingConfig` only governs function calling, so it is omitted entirely when /// there are none. + /// - isFunctionCallingDisabled: Whether the request must not call functions (see + /// `isFunctionCallingDisabled(for:isAtToolCallingTurnLimit:)`). If `true`, function calling + /// is disabled whatever `toolCallingMode` is, so that the model gives a final response. /// - Returns: A `ToolConfig`, or `nil` if no configuration is needed. - /// - Throws: `LanguageModelError.unsupportedCapability` if `toolCallingMode` is a mode that - /// this version of the SDK does not recognize. + /// - Throws: `LanguageModelError.unsupportedCapability` if function calling isn't disabled and + /// `toolCallingMode` is a mode that this version of the SDK does not recognize. static func translateToolConfig( toolCallingMode: GenerationOptions.ToolCallingMode?, - hasFunctionDeclarations: Bool + hasFunctionDeclarations: Bool, + isFunctionCallingDisabled: Bool ) throws -> ToolConfig? { // Computed independently of the `ToolConfig` check below to allow future options (such // as `includeServerSideToolInvocations`) when built-in tools are present without function // declarations. let functionCallingConfig: FunctionCallingConfig? if hasFunctionDeclarations { - // `GenerationOptions.toolCallingMode` is optional and defaults to `nil` unless set by the - // caller. Omitting `functionCallingConfig` in that case causes the backend to default to - // `AUTO`, so `nil` maps to the allowed mode. - let allowedMode = FunctionCallingConfig.Mode.validated let callingMode: FunctionCallingConfig.Mode - if let mode = toolCallingMode { + if isFunctionCallingDisabled { + // Disabled function calling (such as at the turn limit) overrides the requested mode, + // including `required`, which would otherwise keep calling tools indefinitely. + callingMode = FunctionCallingConfig.Mode.none + } else if let mode = toolCallingMode { switch mode.kind { case .allowed: - callingMode = allowedMode + callingMode = .validated case .required: callingMode = .any case .disallowed: @@ -133,7 +263,10 @@ ) } } else { - callingMode = allowedMode + // `GenerationOptions.toolCallingMode` is optional and defaults to `nil` unless set by + // the caller. Omitting `functionCallingConfig` in that case causes the backend to + // default to `AUTO`, so `nil` maps to the allowed mode. + callingMode = .validated } functionCallingConfig = FunctionCallingConfig { $0.mode = callingMode } } else { diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift index 2e77766fd4e..7db3a9a574c 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiLanguageModelTests.swift @@ -48,7 +48,13 @@ typealias FoundationModelsTool = FoundationModels.Tool #endif - @Suite("GeminiLanguageModel Tests", .serialized, .requireFoundationModels) + @Suite( + "GeminiLanguageModel Tests", + .serialized, + .requireFoundationModels, + // Fails runaway tool-call loops instead of hanging. + .timeLimit(.minutes(1)) + ) struct GeminiLanguageModelTests { @Generable(description: "A city summary") @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) @@ -691,7 +697,7 @@ proto.client?.urlProtocol(proto, didLoad: Data(ssePayload.utf8)) proto.client?.urlProtocolDidFinishLoading(proto) } - let session = LanguageModelSession(model: model) + let session = LanguageModelSession(model: model, tools: [MockWeatherTool()]) do { _ = try await session.respond(to: "What's the weather?") @@ -878,10 +884,674 @@ #expect(fr.response?["result"] == .string("12:00 PM")) } + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func sessionRespondDisablesFunctionCallingAtToolCallingTurnLimit() async throws { + defer { MockHTTPURLProtocol.reset() } + let maximumTurns = GeminiLanguageModel.Executor.maximumToolCallingTurns + try #require(maximumTurns == 10) + let recordedRequests = try Self.mockToolCallingLoop { $0 <= maximumTurns } + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + + let response = try await session.respond( + to: "What is the weather in Paris?", + options: GenerationOptions(toolCallingMode: .required) + ) + + #expect(response.content == Self.weatherResponseText) + #expect(Self.toolCallsEntryCount(in: session.transcript) == maximumTurns) + #expect(Self.toolOutputEntryCount(in: session.transcript) == maximumTurns) + let requests = recordedRequests.all + #expect(requests.count == maximumTurns + 1) + #expect(requests.dropLast().allSatisfy { $0.toolConfig?.functionCallingConfig?.mode == .any }) + #expect( + requests.last?.toolConfig?.functionCallingConfig?.mode == FunctionCallingConfig.Mode.none) + #expect(requests.last?.tools?.first?.functionDeclarations?.first?.name == "get_weather") + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func functionCallAfterToolCallingTurnLimitThrowsInvalidResponse() async throws { + defer { MockHTTPURLProtocol.reset() } + let maximumTurns = GeminiLanguageModel.Executor.maximumToolCallingTurns + let recordedRequests = try Self.mockToolCallingLoop { _ in true } + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + session.transcriptErrorHandlingPolicy = .preserveTranscript + + do { + _ = try await session.respond(to: "What is the weather in Paris?") + Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + #expect( + invalidResponse.debugDescription.contains("limit of \(maximumTurns) tool-calling turns")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + + #expect(recordedRequests.all.count == maximumTurns + 1) + #expect(Self.toolOutputEntryCount(in: session.transcript) == maximumTurns) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func newPromptResetsToolCallingTurnCount() async throws { + defer { MockHTTPURLProtocol.reset() } + let maximumTurns = GeminiLanguageModel.Executor.maximumToolCallingTurns + // Each prompt receives `maximumTurns` function calls, then text. + let recordedRequests = try Self.mockToolCallingLoop { requestNumber in + requestNumber % (maximumTurns + 1) != 0 + } + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + _ = try await session.respond(to: "What is the weather in Paris?") + + let response = try await session.respond(to: "And in London?") + + #expect(response.content == Self.weatherResponseText) + let modes = recordedRequests.all.map { $0.toolConfig?.functionCallingConfig?.mode } + let validatedModes = Array( + repeating: FunctionCallingConfig.Mode?.some(.validated), count: maximumTurns) + let promptModes = validatedModes + [FunctionCallingConfig.Mode.none] + #expect(modes == promptModes + promptModes) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func parallelFunctionCallsCountAsOneToolCallingTurn() async throws { + defer { MockHTTPURLProtocol.reset() } + let maximumTurns = GeminiLanguageModel.Executor.maximumToolCallingTurns + let recordedRequests = try Self.mockToolCallingLoop(functionCallsPerResponse: 2) { + $0 <= maximumTurns + } + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + + let response = try await session.respond( + to: "What is the weather in Paris and London?", + options: GenerationOptions(toolCallingMode: .required) + ) + + #expect(response.content == Self.weatherResponseText) + #expect(Self.toolCallsEntryCount(in: session.transcript) == maximumTurns) + #expect(Self.toolOutputEntryCount(in: session.transcript) == 2 * maximumTurns) + let requests = recordedRequests.all + #expect(requests.count == maximumTurns + 1) + #expect(requests.dropLast().allSatisfy { $0.toolConfig?.functionCallingConfig?.mode == .any }) + #expect( + requests.last?.toolConfig?.functionCallingConfig?.mode == FunctionCallingConfig.Mode.none) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func functionCallWithToolCallingModeDisallowedThrowsInvalidResponse() async throws { + defer { MockHTTPURLProtocol.reset() } + let recordedRequests = try Self.mockToolCallingLoop { _ in true } + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + session.transcriptErrorHandlingPolicy = .preserveTranscript + + do { + _ = try await session.respond( + to: "What is the weather in Paris?", + options: GenerationOptions(toolCallingMode: .disallowed) + ) + Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + #expect(invalidResponse.debugDescription.contains("tool calling mode was `disallowed`")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + + let requests = recordedRequests.all + #expect(requests.count == 1) + #expect(requests.first?.tools == nil) + #expect(Self.toolOutputEntryCount(in: session.transcript) == 0) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func disallowedRequestAfterToolCallsDeclaresToolsWithFunctionCallingNone() async throws { + defer { MockHTTPURLProtocol.reset() } + let recordedRequests = try Self.mockToolCallingLoop { $0 == 1 } + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + _ = try await session.respond(to: "What is the weather in Paris?") + + let response = try await session.respond( + to: "Summarize that without calling tools.", + options: GenerationOptions(toolCallingMode: .disallowed) + ) + + #expect(response.content == Self.weatherResponseText) + let requests = recordedRequests.all + #expect(requests.count == 3) + let lastRequest = try #require(requests.last) + #expect(lastRequest.tools?.first?.functionDeclarations?.map(\.name) == ["get_weather"]) + #expect( + lastRequest.toolConfig?.functionCallingConfig?.mode == FunctionCallingConfig.Mode.none) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func functionCallToUnknownToolThrowsInvalidResponse() async throws { + defer { MockHTTPURLProtocol.reset() } + let recordedRequests = try Self.mockToolCallingLoop(functionName: "get_current_time") { _ in + true + } + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + + do { + _ = try await session.respond(to: "What time is it?") + Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + #expect( + invalidResponse.debugDescription.contains( + "`get_current_time`, which isn't an enabled tool")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + + #expect(recordedRequests.all.count == 1) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func functionCallWithoutToolsThrowsInvalidResponse() async throws { + defer { MockHTTPURLProtocol.reset() } + let recordedRequests = try Self.mockToolCallingLoop { _ in true } + let session = LanguageModelSession(model: Self.makeMockModel()) + + do { + _ = try await session.respond(to: "What is the weather in Paris?") + Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + #expect( + invalidResponse.debugDescription.contains("`get_weather`, which isn't an enabled tool")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + + #expect(recordedRequests.all.count == 1) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func functionCallWithEmptyNameThrowsInvalidResponse() async throws { + defer { MockHTTPURLProtocol.reset() } + let recordedRequests = try Self.mockToolCallingLoop(functionName: "") { _ in true } + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + + do { + _ = try await session.respond(to: "What is the weather in Paris?") + Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + #expect(invalidResponse.debugDescription.contains("without a name")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + + #expect(recordedRequests.all.count == 1) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func streamResponseDoesNotStreamPartsOfChunkWithRejectedFunctionCall() async throws { + defer { MockHTTPURLProtocol.reset() } + let expectedURL = try Self.makeExpectedStreamURL() + let httpResponse = try HTTPURLResponse.mock( + url: expectedURL, + headerFields: ["Content-Type": "text/event-stream"] + ) + // One chunk with a thought, text, and a function call with a thought signature. + let ssePayload = """ + data: {"candidates": [{"content": {"parts": [{"text": "Checking the weather.", "thought": true}, {"text": "Let me check."}, {"functionCall": {"id": "call-1", "name": "get_weather", "args": {"location": "Paris"}}, "thoughtSignature": "c2lnbmF0dXJl"}], "role": "model"}, "finishReason": "STOP", "index": 0}]} + + """ + MockHTTPURLProtocol.setHandler(for: expectedURL) { _, proto in + proto.client?.urlProtocol(proto, didReceive: httpResponse, cacheStoragePolicy: .notAllowed) + proto.client?.urlProtocol(proto, didLoad: Data(ssePayload.utf8)) + proto.client?.urlProtocolDidFinishLoading(proto) + } + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + session.transcriptErrorHandlingPolicy = .preserveTranscript + + do { + let stream = session.streamResponse( + to: "What is the weather in Paris?", + options: GenerationOptions(toolCallingMode: .disallowed) + ) + for try await _ in stream {} + Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + #expect(invalidResponse.debugDescription.contains("tool calling mode was `disallowed`")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + + let promptIndex = try #require( + session.transcript.firstIndex { entry in + guard case .prompt = entry else { return false } + return true + } + ) + #expect(session.transcript[(promptIndex + 1)...].isEmpty) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func promptAfterToolCallingTurnLimitErrorWithPreservedTranscriptSucceeds() async throws { + defer { MockHTTPURLProtocol.reset() } + let maximumTurns = GeminiLanguageModel.Executor.maximumToolCallingTurns + // The first prompt receives function calls until it fails at the turn limit. + let recordedRequests = try Self.mockToolCallingLoop { $0 <= maximumTurns + 1 } + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + session.transcriptErrorHandlingPolicy = .preserveTranscript + await #expect(throws: GeminiLanguageModel.Error.self) { + try await session.respond(to: "What is the weather in Paris?") + } + + let response = try await session.respond(to: "Summarize what you found.") + + #expect(response.content == Self.weatherResponseText) + let requests = recordedRequests.all + #expect(requests.count == maximumTurns + 2) + let lastRequest = try #require(requests.last) + #expect(lastRequest.toolConfig?.functionCallingConfig?.mode == .validated) + // The transcript ends with the last tool output, so the new prompt joins its user turn. + let lastContent = try #require(lastRequest.contents?.last) + #expect(lastContent.role == "user") + let lastParts = try #require(lastContent.parts) + #expect(lastParts.count == 2) + guard case .functionResponse(let functionResponse) = lastParts.first?.data else { + Issue.record("Expected a functionResponse part before the new prompt.") + return + } + #expect(functionResponse.name == "get_weather") + guard case .text(let promptText) = lastParts.last?.data else { + Issue.record("Expected the new prompt's text part.") + return + } + #expect(promptText == "Summarize what you found.") + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func rejectedFunctionCallDoesNotStreamAllowedCallOrTextInSameChunk() async throws { + defer { MockHTTPURLProtocol.reset() } + let recordedRequests = try Self.mockStreamResponses([ + Self.functionCallsChunk( + [ + (id: "call-1", name: "get_weather"), + (id: "call-2", name: "get_current_time"), + ], + leadingText: "Let me check." + ) + ]) + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + session.transcriptErrorHandlingPolicy = .preserveTranscript + + do { + _ = try await session.respond(to: "What are the weather and time in Paris?") + Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + #expect( + invalidResponse.debugDescription.contains( + "`get_current_time`, which isn't an enabled tool")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + + #expect(recordedRequests.all.count == 1) + let promptIndex = try #require( + session.transcript.firstIndex { entry in + guard case .prompt = entry else { return false } + return true + } + ) + #expect(session.transcript[(promptIndex + 1)...].isEmpty) + } + + /// Checks that a function call from an earlier chunk of a response, which has already been + /// streamed to the session, isn't kept or run when a later chunk is rejected, even when the + /// transcript is preserved. + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func rejectedFunctionCallInLaterChunkKeepsNoEarlierToolCalls() async throws { + defer { MockHTTPURLProtocol.reset() } + let recordedRequests = try Self.mockStreamResponses([ + Self.functionCallsChunk([(id: "call-1", name: "get_weather")], isFinal: false) + + Self.functionCallsChunk([(id: "call-2", name: "get_current_time")]) + ]) + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + session.transcriptErrorHandlingPolicy = .preserveTranscript + + do { + _ = try await session.respond(to: "What are the weather and time in Paris?") + Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + #expect( + invalidResponse.debugDescription.contains( + "`get_current_time`, which isn't an enabled tool")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + + #expect(recordedRequests.all.count == 1) + let promptIndex = try #require( + session.transcript.firstIndex { entry in + guard case .prompt = entry else { return false } + return true + } + ) + #expect(session.transcript[(promptIndex + 1)...].isEmpty) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func unexpectedToolCallAfterToolCallingTurnLimitThrowsInvalidResponse() async throws { + defer { MockHTTPURLProtocol.reset() } + let maximumTurns = GeminiLanguageModel.Executor.maximumToolCallingTurns + let functionCallPayloads = (1...maximumTurns).map { requestNumber in + Self.functionCallsChunk([(id: "call-\(requestNumber)", name: "get_weather")]) + } + let recordedRequests = try Self.mockStreamResponses( + functionCallPayloads + [Self.finishReasonChunk("UNEXPECTED_TOOL_CALL")] + ) + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + + do { + _ = try await session.respond(to: "What is the weather in Paris?") + Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + #expect(invalidResponse.debugDescription.contains("`UNEXPECTED_TOOL_CALL`")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + + let requests = recordedRequests.all + #expect(requests.count == maximumTurns + 1) + #expect( + requests.last?.toolConfig?.functionCallingConfig?.mode == FunctionCallingConfig.Mode.none) + } + + @Test(arguments: ["UNEXPECTED_TOOL_CALL", "MALFORMED_FUNCTION_CALL"]) + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func functionCallFinishReasonThrowsInvalidResponse(finishReason: String) async throws { + defer { MockHTTPURLProtocol.reset() } + let recordedRequests = try Self.mockStreamResponses([Self.finishReasonChunk(finishReason)]) + let session = LanguageModelSession(model: Self.makeMockModel(), tools: [MockWeatherTool()]) + + do { + _ = try await session.respond(to: "What is the weather in Paris?") + Issue.record("Expected GeminiLanguageModel.Error.invalidResponse") + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + #expect(invalidResponse.debugDescription.contains("`\(finishReason)`")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + + #expect(recordedRequests.all.count == 1) + } + + /// A test case for `GeminiLanguageModel.Executor.functionCallRejectionReason`. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + @available(tvOS, unavailable) + struct FunctionCallRejectionCase: Sendable, CustomTestStringConvertible { + /// The tool calling mode of the request. + let toolCallingMode: GenerationOptions.ToolCallingMode? + + /// The name of the function that Gemini calls. + let functionName: String + + /// Whether the request enables `MockWeatherTool`. + let enablesWeatherTool: Bool + + /// Whether the session has reached the tool-calling turn limit. + let isAtToolCallingTurnLimit: Bool + + /// A substring of the expected rejection reason, or `nil` if the call is allowed. + let expectedReasonSubstring: String? + + /// A description of the test case for test reports. + var testDescription: String { + let modeDescription: String + switch toolCallingMode?.kind { + case nil: + modeDescription = "nil" + case .allowed: + modeDescription = "allowed" + case .required: + modeDescription = "required" + case .disallowed: + modeDescription = "disallowed" + @unknown default: + modeDescription = "unknown" + } + return """ + mode: \(modeDescription), function: \(functionName), \ + tool enabled: \(enablesWeatherTool), at limit: \(isAtToolCallingTurnLimit) + """ + } + + /// The cases to test, in order of rejection precedence. + static let all: [FunctionCallRejectionCase] = [ + .init( + toolCallingMode: .disallowed, functionName: "get_weather", enablesWeatherTool: true, + isAtToolCallingTurnLimit: false, expectedReasonSubstring: "`disallowed`"), + .init( + toolCallingMode: .disallowed, functionName: "get_weather", enablesWeatherTool: false, + isAtToolCallingTurnLimit: true, expectedReasonSubstring: "`disallowed`"), + .init( + toolCallingMode: .required, functionName: "get_weather", enablesWeatherTool: true, + isAtToolCallingTurnLimit: true, + expectedReasonSubstring: "limit of 10 tool-calling turns"), + .init( + toolCallingMode: nil, functionName: "get_time", enablesWeatherTool: false, + isAtToolCallingTurnLimit: true, + expectedReasonSubstring: "limit of 10 tool-calling turns"), + .init( + toolCallingMode: nil, functionName: "get_weather", enablesWeatherTool: false, + isAtToolCallingTurnLimit: false, expectedReasonSubstring: "isn't an enabled tool"), + .init( + toolCallingMode: .allowed, functionName: "get_time", enablesWeatherTool: true, + isAtToolCallingTurnLimit: false, expectedReasonSubstring: "isn't an enabled tool"), + .init( + toolCallingMode: nil, functionName: "get_weather", enablesWeatherTool: true, + isAtToolCallingTurnLimit: false, expectedReasonSubstring: nil), + .init( + toolCallingMode: .allowed, functionName: "get_weather", enablesWeatherTool: true, + isAtToolCallingTurnLimit: false, expectedReasonSubstring: nil), + .init( + toolCallingMode: .required, functionName: "get_weather", enablesWeatherTool: true, + isAtToolCallingTurnLimit: false, expectedReasonSubstring: nil), + ] + } + + @Test(arguments: FunctionCallRejectionCase.all) + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func functionCallRejectionReason(_ testCase: FunctionCallRejectionCase) throws { + let request = LanguageModelExecutorGenerationRequest( + id: UUID(), + transcript: Transcript(entries: []), + enabledTools: testCase.enablesWeatherTool + ? [Transcript.ToolDefinition(tool: MockWeatherTool())] : [], + schema: nil, + generationOptions: GenerationOptions(toolCallingMode: testCase.toolCallingMode), + contextOptions: ContextOptions(), + metadata: [:] + ) + + let reason = GeminiLanguageModel.Executor.functionCallRejectionReason( + forFunctionNamed: testCase.functionName, + in: request, + isAtToolCallingTurnLimit: testCase.isAtToolCallingTurnLimit + ) + + if let expectedReasonSubstring = testCase.expectedReasonSubstring { + let reason = try #require(reason) + #expect(reason.contains(expectedReasonSubstring)) + } else { + #expect(reason == nil) + } + } + // MARK: - Helper Methods + /// The final text response sent by `mockToolCallingLoop`. + private static let weatherResponseText = "The weather in Paris is sunny, 72°F." + + /// Records the Gemini requests intercepted by `MockHTTPURLProtocol`. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + private final class RecordedRequests: Sendable { + private let requests = Mutex<[GenerateContentRequest]>([]) + + /// The recorded requests, in the order they were sent. + var all: [GenerateContentRequest] { requests.withLock { $0 } } + + /// Records a request and returns the number of requests recorded so far. + /// + /// - Parameter request: The request to record. + /// - Returns: The number of requests recorded, including `request`. + func append(_ request: GenerateContentRequest) -> Int { + requests.withLock { + $0.append(request) + return $0.count + } + } + } + + /// Mocks a model that responds to each request with either function calls or text. + /// + /// - Parameters: + /// - functionName: The name of the function that the mock calls. + /// - functionCallsPerResponse: The number of parallel function calls in each function-call + /// response. + /// - respondsWithFunctionCalls: Returns whether the request with the given 1-based number + /// receives function calls; other requests receive `weatherResponseText`. + /// - Returns: The requests that the mock receives. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + private static func mockToolCallingLoop( + functionName: String = "get_weather", + functionCallsPerResponse: Int = 1, + respondsWithFunctionCalls: @escaping @Sendable (_ requestNumber: Int) -> Bool + ) throws -> RecordedRequests { + let expectedURL = try makeExpectedStreamURL() + let httpResponse = try HTTPURLResponse.mock( + url: expectedURL, + headerFields: ["Content-Type": "text/event-stream"] + ) + let textSSE = """ + data: {"candidates": [{"content": {"parts": [{"text": "\(weatherResponseText)"}], "role": "model"}, "finishReason": "STOP", "index": 0}]} + + """ + let recordedRequests = RecordedRequests() + MockHTTPURLProtocol.setHandler(for: expectedURL) { request, proto in + let body = try #require(request.httpBodyData) + let requestNumber = recordedRequests.append( + try JSONDecoder().decode(GenerateContentRequest.self, from: body) + ) + let payload: String + if respondsWithFunctionCalls(requestNumber) { + let parts = (1...functionCallsPerResponse).map { index in + """ + {"functionCall": {"id": "call-\(requestNumber)-\(index)", "name": "\(functionName)", \ + "args": {"location": "Paris"}}} + """ + } + payload = """ + data: {"candidates": [{"content": {"parts": [\(parts.joined(separator: ", "))], "role": "model"}, "finishReason": "STOP", "index": 0}]} + + """ + } else { + payload = textSSE + } + proto.client?.urlProtocol(proto, didReceive: httpResponse, cacheStoragePolicy: .notAllowed) + proto.client?.urlProtocol(proto, didLoad: Data(payload.utf8)) + proto.client?.urlProtocolDidFinishLoading(proto) + } + return recordedRequests + } + + /// Mocks a model that responds to each request with a server-sent events payload. + /// + /// - Parameter ssePayloads: The payloads to respond with, in request order. Requests after + /// the last payload receive the last payload. + /// - Returns: The requests that the mock receives. @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + private static func mockStreamResponses(_ ssePayloads: [String]) throws -> RecordedRequests { + try #require(!ssePayloads.isEmpty) + let expectedURL = try makeExpectedStreamURL() + let httpResponse = try HTTPURLResponse.mock( + url: expectedURL, + headerFields: ["Content-Type": "text/event-stream"] + ) + let recordedRequests = RecordedRequests() + MockHTTPURLProtocol.setHandler(for: expectedURL) { request, proto in + let body = try #require(request.httpBodyData) + let requestNumber = recordedRequests.append( + try JSONDecoder().decode(GenerateContentRequest.self, from: body) + ) + let payload = ssePayloads[min(requestNumber, ssePayloads.count) - 1] + proto.client?.urlProtocol(proto, didReceive: httpResponse, cacheStoragePolicy: .notAllowed) + proto.client?.urlProtocol(proto, didLoad: Data(payload.utf8)) + proto.client?.urlProtocolDidFinishLoading(proto) + } + return recordedRequests + } + /// Returns a server-sent event for a streamed chunk with a finish reason and no content. + /// + /// - Parameter finishReason: The raw value of the chunk's finish reason. + /// - Returns: The `data:` line and its terminating blank line. + private static func finishReasonChunk(_ finishReason: String) -> String { + #"data: {"candidates": [{"finishReason": "\#(finishReason)", "index": 0}]}"# + "\n\n" + } + + /// Returns a server-sent event for a streamed chunk with parallel function calls. + /// + /// - Parameters: + /// - calls: The identifier and function name of each function call, in order. + /// - leadingText: The text of a part before the function calls, or `nil` for none. + /// - isFinal: Whether the chunk ends the response with a `STOP` finish reason. + /// - Returns: The `data:` line and its terminating blank line. + private static func functionCallsChunk( + _ calls: [(id: String, name: String)], + leadingText: String? = nil, + isFinal: Bool = true + ) -> String { + let textParts = leadingText.map { [#"{"text": "\#($0)"}"#] } ?? [] + let functionCallParts = calls.map { call in + #"{"functionCall": {"id": "\#(call.id)", "name": "\#(call.name)", "# + + #""args": {"location": "Paris"}}}"# + } + let parts = textParts + functionCallParts + let finishReason = isFinal ? #", "finishReason": "STOP""# : "" + return #"data: {"candidates": [{"content": {"parts": [\#(parts.joined(separator: ", "))], "# + + #""role": "model"}\#(finishReason), "index": 0}]}"# + + "\n\n" + } + + /// Returns the number of tool-calling turns (non-empty tool-calls entries) in a transcript. + /// + /// - Parameter transcript: The transcript to search. + /// - Returns: The number of non-empty `Transcript.Entry.toolCalls` entries. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + private static func toolCallsEntryCount(in transcript: Transcript) -> Int { + transcript.filter { entry in + guard case .toolCalls(let toolCalls) = entry else { return false } + return !toolCalls.isEmpty + }.count + } + + /// Returns the number of tool outputs in a transcript. + /// + /// - Parameter transcript: The transcript to search. + /// - Returns: The number of `Transcript.Entry.toolOutput` entries. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + private static func toolOutputEntryCount(in transcript: Transcript) -> Int { + transcript.filter { entry in + guard case .toolOutput = entry else { return false } + return true + }.count + } + + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) private static func makeMockModel( modelResource: ModelResource = .gemini38Flash, endpointConfiguration: EndpointConfiguration = .geminiDeveloperAPI, diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiRequestTranslatorTests.swift b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiRequestTranslatorTests.swift index 5e861ed975c..6a9e9d1eae3 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiRequestTranslatorTests.swift +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/GeminiRequestTranslatorTests.swift @@ -50,6 +50,15 @@ var city: String } + @Generable( + description: "Postal code arguments with a pattern guide that Gemini doesn't support") + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + @available(tvOS, unavailable) + struct PostalCodeArguments { + @Guide(description: "A postal code", .pattern(#/^\d{5}$/#)) + var postalCode: String + } + @Generable(description: "Empty arguments") @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) @available(tvOS, unavailable) @@ -76,7 +85,7 @@ metadata: [:] ) - let result = try GeminiRequestTranslator.translate(request) + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) #expect(result.systemInstruction == nil) #expect(result.contents?.count == 1) @@ -105,7 +114,7 @@ metadata: [:] ) - let result = try GeminiRequestTranslator.translate(request) + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) #expect(result.contents?.count == 1) let generationConfig = try #require(result.generationConfig) @@ -217,19 +226,23 @@ func translatesToolCallingModes() throws { let allowedConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: .allowed, - hasFunctionDeclarations: true + hasFunctionDeclarations: true, + isFunctionCallingDisabled: false ) let requiredConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: .required, - hasFunctionDeclarations: true + hasFunctionDeclarations: true, + isFunctionCallingDisabled: false ) let disallowedConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: .disallowed, - hasFunctionDeclarations: true + hasFunctionDeclarations: true, + isFunctionCallingDisabled: false ) let unsetConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: nil, - hasFunctionDeclarations: true + hasFunctionDeclarations: true, + isFunctionCallingDisabled: false ) #expect(allowedConfig?.functionCallingConfig?.mode == .validated) @@ -243,19 +256,23 @@ func translatesToolCallingModesWithoutDeclarationsReturnsNil() throws { let allowedConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: .allowed, - hasFunctionDeclarations: false + hasFunctionDeclarations: false, + isFunctionCallingDisabled: false ) let requiredConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: .required, - hasFunctionDeclarations: false + hasFunctionDeclarations: false, + isFunctionCallingDisabled: false ) let disallowedConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: .disallowed, - hasFunctionDeclarations: false + hasFunctionDeclarations: false, + isFunctionCallingDisabled: false ) let nilConfig = try GeminiRequestTranslator.translateToolConfig( toolCallingMode: nil, - hasFunctionDeclarations: false + hasFunctionDeclarations: false, + isFunctionCallingDisabled: false ) #expect(allowedConfig == nil) @@ -298,8 +315,10 @@ metadata: [:] ) - let resultWithTool = try GeminiRequestTranslator.translate(requestWithTool) - let resultWithoutTools = try GeminiRequestTranslator.translate(requestWithoutTools) + let resultWithTool = try GeminiRequestTranslator.translate( + requestWithTool, isAtToolCallingTurnLimit: false) + let resultWithoutTools = try GeminiRequestTranslator.translate( + requestWithoutTools, isAtToolCallingTurnLimit: false) let unwrappedToolConfig = try #require(resultWithTool.toolConfig) let functionCallingConfig = try #require(unwrappedToolConfig.functionCallingConfig) @@ -332,11 +351,476 @@ metadata: [:] ) - let result = try GeminiRequestTranslator.translate(request) + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) let toolConfig = try #require(result.toolConfig) let functionCallingConfig = try #require(toolConfig.functionCallingConfig) #expect(functionCallingConfig.mode == FunctionCallingConfig.Mode.validated) } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func toolCallingTurnCountCountsToolCallsSinceMostRecentPrompt() throws { + let parisCall = try Self.makeToolCall(id: "call-1", city: "Paris") + let londonCall = try Self.makeToolCall(id: "call-2", city: "London") + let tokyoCall = try Self.makeToolCall(id: "call-3", city: "Tokyo") + let romeCall = try Self.makeToolCall(id: "call-4", city: "Rome") + let transcript = Transcript( + entries: [ + .instructions( + Transcript.Instructions( + segments: [.text(Transcript.TextSegment(content: "Be brief."))], + toolDefinitions: [] + ) + ), + .prompt(Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Paris?"))])), + .toolCalls(Transcript.ToolCalls(id: "calls-1", [parisCall])), + .toolOutput(Self.makeToolOutput(id: "call-1")), + .response( + Transcript.Response(segments: [.text(Transcript.TextSegment(content: "Sunny."))])), + .prompt(Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Others?"))])), + .reasoning( + Transcript.Reasoning( + segments: [.text(Transcript.TextSegment(content: "Check two cities."))]) + ), + .response( + Transcript.Response(segments: [.text(Transcript.TextSegment(content: "Checking."))])), + .toolCalls(Transcript.ToolCalls(id: "calls-2", [londonCall, tokyoCall])), + .toolOutput(Self.makeToolOutput(id: "call-2")), + .toolOutput(Self.makeToolOutput(id: "call-3")), + .toolCalls(Transcript.ToolCalls(id: "calls-3", [])), + .toolCalls(Transcript.ToolCalls(id: "calls-4", [romeCall])), + .toolOutput(Self.makeToolOutput(id: "call-4")), + ] + ) + + let turnCount = GeminiRequestTranslator.toolCallingTurnCount(in: transcript) + + #expect(turnCount == 2) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func toolCallingTurnCountInEmptyTranscriptIsZero() { + let turnCount = GeminiRequestTranslator.toolCallingTurnCount(in: Transcript(entries: [])) + + #expect(turnCount == 0) + } + + @Test(arguments: [ + GenerationOptions.ToolCallingMode?.none, + .allowed, + .required, + .disallowed, + ]) + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesToolConfigWithFunctionCallingDisabledToNone( + toolCallingMode: GenerationOptions.ToolCallingMode? + ) throws { + let encoder = JSONEncoder() + encoder.outputFormatting = .sortedKeys + + let toolConfig = try GeminiRequestTranslator.translateToolConfig( + toolCallingMode: toolCallingMode, + hasFunctionDeclarations: true, + isFunctionCallingDisabled: true + ) + + let mode = try #require(toolConfig?.functionCallingConfig?.mode) + #expect(mode == FunctionCallingConfig.Mode.none) + let json = String(decoding: try encoder.encode(toolConfig), as: UTF8.self) + #expect(json == #"{"functionCallingConfig":{"mode":"NONE"}}"#) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesRequestAtToolCallingTurnLimitKeepsToolDeclarations() throws { + let toolDefinition = Transcript.ToolDefinition( + name: "get_current_weather", + description: "Get the current weather for a city.", + parameters: LocationArguments.generationSchema + ) + let request = LanguageModelExecutorGenerationRequest( + id: UUID(), + transcript: Transcript( + entries: [ + .prompt(Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Paris?"))])) + ] + ), + enabledTools: [toolDefinition], + schema: nil, + generationOptions: GenerationOptions(toolCallingMode: .required), + contextOptions: ContextOptions(), + metadata: [:] + ) + + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: true) + + let functionDeclarations = try #require(result.tools?.first?.functionDeclarations) + #expect(functionDeclarations.map(\.name) == ["get_current_weather"]) + #expect(result.toolConfig?.functionCallingConfig?.mode == FunctionCallingConfig.Mode.none) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesToolConfigWithFunctionCallingDisabledWithoutDeclarationsReturnsNil() throws { + let toolConfig = try GeminiRequestTranslator.translateToolConfig( + toolCallingMode: .required, + hasFunctionDeclarations: false, + isFunctionCallingDisabled: true + ) + + #expect(toolConfig == nil) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesDisallowedRequestAfterToolCallsDeclaresInstructionsToolsWithNone() throws { + let toolDefinition = Self.makeWeatherToolDefinition() + let parisCall = try Self.makeToolCall(id: "call-1", city: "Paris") + let request = LanguageModelExecutorGenerationRequest( + id: UUID(), + transcript: Transcript( + entries: [ + .instructions(Transcript.Instructions(segments: [], toolDefinitions: [toolDefinition])), + .prompt( + Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Paris?"))])), + .toolCalls(Transcript.ToolCalls(id: "calls-1", [parisCall])), + .toolOutput(Self.makeToolOutput(id: "call-1")), + ] + ), + enabledTools: [], + schema: nil, + generationOptions: GenerationOptions(toolCallingMode: .disallowed), + contextOptions: ContextOptions(), + metadata: [:] + ) + + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) + + let functionDeclarations = try #require(result.tools?.first?.functionDeclarations) + #expect(functionDeclarations.map(\.name) == ["getWeather"]) + #expect(result.toolConfig?.functionCallingConfig?.mode == FunctionCallingConfig.Mode.none) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesRequestWithoutEnabledToolsAtToolCallingTurnLimitDeclaresInstructionsTools() + throws + { + let toolDefinition = Self.makeWeatherToolDefinition() + let parisCall = try Self.makeToolCall(id: "call-1", city: "Paris") + let request = LanguageModelExecutorGenerationRequest( + id: UUID(), + transcript: Transcript( + entries: [ + .instructions(Transcript.Instructions(segments: [], toolDefinitions: [toolDefinition])), + .prompt( + Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Paris?"))])), + .toolCalls(Transcript.ToolCalls(id: "calls-1", [parisCall])), + .toolOutput(Self.makeToolOutput(id: "call-1")), + ] + ), + enabledTools: [], + schema: nil, + generationOptions: GenerationOptions(toolCallingMode: .required), + contextOptions: ContextOptions(), + metadata: [:] + ) + + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: true) + + let functionDeclarations = try #require(result.tools?.first?.functionDeclarations) + #expect(functionDeclarations.map(\.name) == ["getWeather"]) + #expect(result.toolConfig?.functionCallingConfig?.mode == FunctionCallingConfig.Mode.none) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesDisallowedRequestWithUntranslatableInstructionsToolOmitsTools() throws { + let toolDefinition = Transcript.ToolDefinition( + name: "lookUpPostalCode", + description: "Look up a postal code.", + parameters: PostalCodeArguments.generationSchema + ) + let lookUpCall = Transcript.ToolCall( + id: "call-1", + toolName: "lookUpPostalCode", + arguments: try GeneratedContent(json: #"{"postalCode": "94043"}"#) + ) + let request = LanguageModelExecutorGenerationRequest( + id: UUID(), + transcript: Transcript( + entries: [ + .instructions(Transcript.Instructions(segments: [], toolDefinitions: [toolDefinition])), + .prompt( + Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "94043?"))])), + .toolCalls(Transcript.ToolCalls(id: "calls-1", [lookUpCall])), + ] + ), + enabledTools: [], + schema: nil, + generationOptions: GenerationOptions(toolCallingMode: .disallowed), + contextOptions: ContextOptions(), + metadata: [:] + ) + + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) + + #expect(result.tools == nil) + #expect(result.toolConfig == nil) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesDisallowedRequestDeclaresOnlyTranslatableInstructionsTools() throws { + let postalCodeToolDefinition = Transcript.ToolDefinition( + name: "lookUpPostalCode", + description: "Look up a postal code.", + parameters: PostalCodeArguments.generationSchema + ) + let parisCall = try Self.makeToolCall(id: "call-1", city: "Paris") + let request = LanguageModelExecutorGenerationRequest( + id: UUID(), + transcript: Transcript( + entries: [ + .instructions( + Transcript.Instructions( + segments: [], + toolDefinitions: [Self.makeWeatherToolDefinition(), postalCodeToolDefinition] + ) + ), + .prompt( + Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Paris?"))])), + .toolCalls(Transcript.ToolCalls(id: "calls-1", [parisCall])), + .toolOutput(Self.makeToolOutput(id: "call-1")), + ] + ), + enabledTools: [], + schema: nil, + generationOptions: GenerationOptions(toolCallingMode: .disallowed), + contextOptions: ContextOptions(), + metadata: [:] + ) + + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) + + let functionDeclarations = try #require(result.tools?.first?.functionDeclarations) + #expect(functionDeclarations.map(\.name) == ["getWeather"]) + #expect(result.toolConfig?.functionCallingConfig?.mode == FunctionCallingConfig.Mode.none) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesRequestWithUntranslatableEnabledToolThrows() { + let request = LanguageModelExecutorGenerationRequest( + id: UUID(), + transcript: Transcript( + entries: [ + .prompt( + Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "94043?"))])) + ] + ), + enabledTools: [ + Self.makeWeatherToolDefinition(), + Transcript.ToolDefinition( + name: "lookUpPostalCode", + description: "Look up a postal code.", + parameters: PostalCodeArguments.generationSchema + ), + ], + schema: nil, + generationOptions: GenerationOptions(toolCallingMode: .allowed), + contextOptions: ContextOptions(), + metadata: [:] + ) + + do { + _ = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) + Issue.record("Expected unsupportedGenerationGuide error.") + } catch LanguageModelError.unsupportedGenerationGuide(let error) { + #expect(error.debugDescription.contains("pattern")) + } catch { + Issue.record("Unexpected error thrown: \(error)") + } + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesDisallowedRequestWithoutToolCallsOmitsTools() throws { + let toolDefinition = Self.makeWeatherToolDefinition() + let request = LanguageModelExecutorGenerationRequest( + id: UUID(), + transcript: Transcript( + entries: [ + .instructions(Transcript.Instructions(segments: [], toolDefinitions: [toolDefinition])), + .prompt( + Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Paris?"))])), + ] + ), + enabledTools: [], + schema: nil, + generationOptions: GenerationOptions(toolCallingMode: .disallowed), + contextOptions: ContextOptions(), + metadata: [:] + ) + + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) + + #expect(result.tools == nil) + #expect(result.toolConfig == nil) + } + + @Test(arguments: [ + GenerationOptions.ToolCallingMode?.none, + .allowed, + .required, + ]) + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesRequestWithoutEnabledToolsAfterToolCallsDeclaresInstructionsToolsWithNone( + toolCallingMode: GenerationOptions.ToolCallingMode? + ) throws { + let toolDefinition = Self.makeWeatherToolDefinition() + let parisCall = try Self.makeToolCall(id: "call-1", city: "Paris") + let request = LanguageModelExecutorGenerationRequest( + id: UUID(), + transcript: Transcript( + entries: [ + .instructions(Transcript.Instructions(segments: [], toolDefinitions: [toolDefinition])), + .prompt( + Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Paris?"))])), + .toolCalls(Transcript.ToolCalls(id: "calls-1", [parisCall])), + .toolOutput(Self.makeToolOutput(id: "call-1")), + ] + ), + enabledTools: [], + schema: nil, + generationOptions: GenerationOptions(toolCallingMode: toolCallingMode), + contextOptions: ContextOptions(), + metadata: [:] + ) + + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) + + let functionDeclarations = try #require(result.tools?.first?.functionDeclarations) + #expect(functionDeclarations.map(\.name) == ["getWeather"]) + #expect(result.toolConfig?.functionCallingConfig?.mode == FunctionCallingConfig.Mode.none) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesRequestWithoutEnabledToolsDeclaresMostRecentInstructionsTools() throws { + let request = try Self.makeRequestWithoutEnabledTools( + olderInstructionsTools: [Self.makeWeatherToolDefinition()], + newerInstructionsTools: [Self.makeTimeToolDefinition()] + ) + + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) + + let functionDeclarations = try #require(result.tools?.first?.functionDeclarations) + #expect(functionDeclarations.map(\.name) == ["getTime"]) + } + + @Test + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func translatesRequestWithoutEnabledToolsSkipsInstructionsWithoutTools() throws { + let request = try Self.makeRequestWithoutEnabledTools( + olderInstructionsTools: [Self.makeWeatherToolDefinition()], + newerInstructionsTools: [] + ) + + let result = try GeminiRequestTranslator.translate(request, isAtToolCallingTurnLimit: false) + + let functionDeclarations = try #require(result.tools?.first?.functionDeclarations) + #expect(functionDeclarations.map(\.name) == ["getWeather"]) + } + + // MARK: - Helper Methods + + /// Returns a request that enables no tools, after a tool call, with two instructions entries. + /// + /// - Parameters: + /// - olderInstructionsTools: The tools listed by the instructions before the tool call. + /// - newerInstructionsTools: The tools listed by the instructions after the tool call. + /// - Returns: A request with the `allowed` tool calling mode and no enabled tools. + /// - Throws: An error if the tool call's arguments can't be parsed. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + private static func makeRequestWithoutEnabledTools( + olderInstructionsTools: [Transcript.ToolDefinition], + newerInstructionsTools: [Transcript.ToolDefinition] + ) throws -> LanguageModelExecutorGenerationRequest { + LanguageModelExecutorGenerationRequest( + id: UUID(), + transcript: Transcript( + entries: [ + .instructions( + Transcript.Instructions(segments: [], toolDefinitions: olderInstructionsTools)), + .prompt( + Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Paris?"))])), + .toolCalls( + Transcript.ToolCalls(id: "calls-1", [try makeToolCall(id: "call-1", city: "Paris")])), + .toolOutput(makeToolOutput(id: "call-1")), + .instructions( + Transcript.Instructions(segments: [], toolDefinitions: newerInstructionsTools)), + .prompt( + Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: "Time?"))])), + ] + ), + enabledTools: [], + schema: nil, + generationOptions: GenerationOptions(toolCallingMode: .allowed), + contextOptions: ContextOptions(), + metadata: [:] + ) + } + + /// Returns the definition of a tool that gets the current time. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + private static func makeTimeToolDefinition() -> Transcript.ToolDefinition { + Transcript.ToolDefinition( + name: "getTime", + description: "Get the current time.", + parameters: LocationArguments.generationSchema + ) + } + + /// Returns a `getWeather` tool call for a city. + /// + /// - Parameters: + /// - id: The identifier of the tool call. + /// - city: The city argument of the tool call. + /// - Returns: A tool call to `getWeather`. + /// - Throws: An error if the arguments can't be parsed. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + private static func makeToolCall(id: String, city: String) throws -> Transcript.ToolCall { + Transcript.ToolCall( + id: id, + toolName: "getWeather", + arguments: try GeneratedContent(json: #"{"city": "\#(city)"}"#) + ) + } + + /// Returns the definition of a tool that gets the current weather for a city. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + private static func makeWeatherToolDefinition() -> Transcript.ToolDefinition { + Transcript.ToolDefinition( + name: "getWeather", + description: "Get the current weather for a city.", + parameters: LocationArguments.generationSchema + ) + } + + /// Returns the output of a `getWeather` tool call. + /// + /// - Parameter id: The identifier of the tool call that produced the output. + /// - Returns: The tool output. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + private static func makeToolOutput(id: String) -> Transcript.ToolOutput { + Transcript.ToolOutput( + id: id, + toolName: "getWeather", + segments: [.text(Transcript.TextSegment(content: "22°C and sunny"))] + ) + } } #endif // compiler(>=6.4) && (FoundationModelsFake || canImport(FoundationModels)) diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/README.md b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/README.md index 0606a39e3a9..44555e8ae06 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/README.md +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/README.md @@ -61,8 +61,8 @@ All integration tests are parameterized across Parameterized integration tests for tool calling (function calling) using FoundationModels `Tool` definitions, covering single-turn tool calls, sequential tool calls, parallel tool calls, tool calling mode configuration, - parameterless tools with empty arguments, and thought signatures recorded - as reasoning entries. + parameterless tools with empty arguments, thought signatures recorded as + reasoning entries, and the limit on tool-calling turns per prompt. * [`ImageUnderstandingIntegrationTests.swift`](ImageUnderstandingIntegrationTests.swift): Parameterized integration tests for image input (`vision`) using images generated in code, covering single and multiple images, `@Generable` diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ToolCallingIntegrationTests.swift b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ToolCallingIntegrationTests.swift index 01e0de73f82..590e0ef2621 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ToolCallingIntegrationTests.swift +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/IntegrationTests/ToolCallingIntegrationTests.swift @@ -21,6 +21,7 @@ #endif import GeminiAPIClient import GeminiTestUtilities + import Synchronization import Testing @testable import GeminiLanguageModel @@ -274,5 +275,72 @@ #expect(hasToolOutput) #expect(response.usage.totalTokenCount > 0) } + + /// A weather tool that counts how many times the model calls it. + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + @available(tvOS, unavailable) + struct CountingWeatherTool: FoundationModelsTool { + /// A call counter that all copies of the tool share. + final class CallCounter: Sendable { + /// The number of calls so far. + let count = Mutex(0) + } + + let name = "getWeather" + let description = "Get the current weather for a city." + + /// The counter of calls to the tool, shared by all copies of the tool. + private let counter = CallCounter() + + /// The number of times the model has called the tool. + var callCount: Int { counter.count.withLock { $0 } } + + @Generable + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + @available(tvOS, unavailable) + struct Arguments { + @Guide(description: "The city name to look up weather for") + var city: String + } + + func call(arguments: Arguments) async throws -> String { + counter.count.withLock { $0 += 1 } + return "The current weather in \(arguments.city) is 22°C and sunny." + } + } + + @Test(.requireIntegrationTestingBackend, arguments: IntegrationTestingBackend.availableBackends) + @available(macOS 27.0, iOS 27.0, watchOS 27.0, visionOS 27.0, *) + func sessionRespondRequiredToolCallingStopsAtToolCallingTurnLimit( + backend: IntegrationTestingBackend + ) async throws { + let maximumTurns = GeminiLanguageModel.Executor.maximumToolCallingTurns + let model = try await backend.makeModel() + let tool = CountingWeatherTool() + let session = LanguageModelSession(model: model, tools: [tool]) + // Keeps the tool calls in the transcript if Gemini calls a tool after the limit. + session.transcriptErrorHandlingPolicy = .preserveTranscript + + // With `required`, the model calls a tool in every response until the limit is reached. + do { + let response = try await session.respond( + to: "What is the weather in Paris right now?", + options: GenerationOptions(toolCallingMode: .required) + ) + #expect(!response.content.isEmpty) + } catch GeminiLanguageModel.Error.invalidResponse(let invalidResponse) { + // Gemini sometimes calls a tool after being told not to, which ends the turn with this + // error instead of a response. + #expect(invalidResponse.debugDescription.contains("tool-calling turns per prompt")) + } + + let toolCallingTurnCount = session.transcript.filter { entry in + guard case .toolCalls(let toolCalls) = entry else { return false } + return !toolCalls.isEmpty + }.count + #expect(toolCallingTurnCount == maximumTurns) + // A tool-calling turn can call the tool more than once. + #expect(tool.callCount >= maximumTurns) + } } #endif // compiler(>=6.4) && (FoundationModelsFake || canImport(FoundationModels)) diff --git a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/README.md b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/README.md index 349c962373c..421a7c86c8a 100644 --- a/GeminiLanguageModel/Tests/GeminiLanguageModelTests/README.md +++ b/GeminiLanguageModel/Tests/GeminiLanguageModelTests/README.md @@ -20,7 +20,7 @@ swift test --filter GeminiLanguageModelTests |---|---|---| | [`GeminiLanguageModelTests.swift`](GeminiLanguageModelTests.swift) | Protocol conformance, single/multi-turn responses, streaming, tool calling | Unit (`MockHTTPURLProtocol`) | | [`GenerationSchema+GeminiTests.swift`](GenerationSchema+GeminiTests.swift) | JSON Schema encoding and property ordering conversion | Unit (pure transformation) | -| [`GeminiRequestTranslatorTests.swift`](GeminiRequestTranslatorTests.swift) | Request, generation config, tool declarations, and tool mode mapping | Unit (pure transformation) | +| [`GeminiRequestTranslatorTests.swift`](GeminiRequestTranslatorTests.swift) | Request, generation config, tool declarations, tool mode mapping, and tool-calling turn counting | Unit (pure transformation) | | [`GeminiTranscriptTranslatorTests.swift`](GeminiTranscriptTranslatorTests.swift) | Apple `Transcript` <-> Gemini payload mapping (tools, reasoning, prompts, image parts) | Unit (pure transformation) | | [`GeminiImageTranslatorTests.swift`](GeminiImageTranslatorTests.swift) | Image attachment encoding, orientation, file pass-through, and URL handling | Unit (images generated in code) | | [`TestImages.swift`](TestImages.swift) | Helpers that generate, encode, and sample test images without bundled resources | Test utility |