Skip to content

Fix Gemma 4 nullable arguments in tool schemas #11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
atdrendel wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Fix Gemma 4 nullable arguments in tool schemas #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions IntegrationTesting/IntegrationTestingTests/ToolCallIntegrationTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@ struct ToolCallIntegrationTests {
try await ToolCallTests.lfm2EndToEndGeneration(container: container)
}

@Test func lfm2SupportsJSONArrayNullableToolArguments() async throws {
let container = try await models.llmContainer(for: .init(id: IntegrationTestModelIDs.lfm2))
try await ToolCallTests.lfm2SupportsJSONArrayNullableToolArguments(container: container)
}

// MARK: - GLM4

@Test func glm4FormatAutoDetection() async throws {
Expand All @@ -40,6 +45,11 @@ struct ToolCallIntegrationTests {
try await ToolCallTests.glm4EndToEndGeneration(container: container)
}

@Test func glm4SupportsJSONArrayNullableToolArguments() async throws {
let container = try await models.llmContainer(for: .init(id: IntegrationTestModelIDs.glm4))
try await ToolCallTests.glm4SupportsJSONArrayNullableToolArguments(container: container)
}

// MARK: - Mistral3

@Test func mistral3FormatAutoDetection() async throws {
Expand All @@ -60,6 +70,12 @@ struct ToolCallIntegrationTests {
try await ToolCallTests.mistral3MultiToolGeneration(container: container)
}

@Test func mistral3SupportsJSONArrayNullableToolArguments() async throws {
let container = try await models.llmContainer(
for: .init(id: IntegrationTestModelIDs.mistral3))
try await ToolCallTests.mistral3SupportsJSONArrayNullableToolArguments(container: container)
}

// MARK: - Nemotron

@Test func nemotronFormatAutoDetection() async throws {
Expand All @@ -80,6 +96,12 @@ struct ToolCallIntegrationTests {
try await ToolCallTests.nemotronMultiToolGeneration(container: container)
}

@Test func nemotronSupportsJSONArrayNullableToolArguments() async throws {
let container = try await models.llmContainer(
for: .init(id: IntegrationTestModelIDs.nemotron))
try await ToolCallTests.nemotronSupportsJSONArrayNullableToolArguments(container: container)
}

// MARK: - Qwen3.5

@Test func qwen35FormatAutoDetection() async throws {
Expand All @@ -99,4 +121,36 @@ struct ToolCallIntegrationTests {
for: .init(id: IntegrationTestModelIDs.qwen35))
try await ToolCallTests.qwen35MultiToolGeneration(container: container)
}

@Test func qwen35SupportsJSONArrayNullableToolArguments() async throws {
let container = try await models.llmContainer(
for: .init(id: IntegrationTestModelIDs.qwen35))
try await ToolCallTests.qwen35SupportsJSONArrayNullableToolArguments(container: container)
}

// MARK: - Gemma 4 Unified

@atdrendel atdrendel Jul 7, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Gemma 4 12B, Gemma4Unified, is the only Gemma 4 model I could get to work reliably. The smaller, ExB models all have bugs that are tracked elsewhere.


@Test func gemma4UnifiedFormatAutoDetection() async throws {
let container = try await models.vlmContainer(
for: .init(id: IntegrationTestModelIDs.gemma4Unified))
try await ToolCallTests.gemma4FormatAutoDetection(container: container)
}

@Test func gemma4UnifiedEndToEnd() async throws {
let container = try await models.vlmContainer(
for: .init(id: IntegrationTestModelIDs.gemma4Unified))
try await ToolCallTests.gemma4EndToEndGeneration(container: container)
}

@Test func gemma4UnifiedMultiTool() async throws {
let container = try await models.vlmContainer(
for: .init(id: IntegrationTestModelIDs.gemma4Unified))
try await ToolCallTests.gemma4MultiToolGeneration(container: container)
}

@Test func gemma4UnifiedSupportsJSONArrayNullableToolArguments() async throws {
let container = try await models.vlmContainer(
for: .init(id: IntegrationTestModelIDs.gemma4Unified))
try await ToolCallTests.gemma4SupportsJSONArrayNullableToolArguments(container: container)
}
}
Loading