diff --git a/README.md b/README.md index 7d95ae2..f719c94 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,9 @@ The providers are imported from [providers.py](/os_computer_use/providers.py) an - ShowUI (grounding) - Moonshot - Mistral AI (Pixtral for vision, Mistral Large for actions) +- MiniMax: + - MiniMax-M3 (vision + action) + - MiniMax-M2.7 (action only) If you add a new model or provider, please [make a PR](../../pulls) to this repository with the updated providers.py! @@ -103,11 +106,21 @@ GROQ_API_KEY=... GEMINI_API_KEY=... OPENAI_API_KEY=... ANTHROPIC_API_KEY=... +MINIMAX_API_KEY=... MOONSHOT_API_KEY=... # Required: Provide your Hugging Face token to bypass Gradio rate limits. HF_TOKEN=... ``` +Use `MiniMaxProvider` for the OpenAI-compatible API or +`MiniMaxAnthropicProvider` for the Anthropic-compatible API in `config.py`. +MiniMax uses global API endpoints by default. To use the China endpoints, add: + +```sh +MINIMAX_OPENAI_BASE_URL=https://api.minimaxi.com/v1 +MINIMAX_ANTHROPIC_BASE_URL=https://api.minimaxi.com/anthropic +``` + ### 4. Start the web interface Run the following command to start the agent: diff --git a/os_computer_use/config.py b/os_computer_use/config.py index 1b3d04b..a3fb608 100644 --- a/os_computer_use/config.py +++ b/os_computer_use/config.py @@ -10,6 +10,8 @@ # vision_model = providers.AnthropicProvider("claude-3.5-sonnet") # vision_model = providers.MoonshotProvider("moonshot-v1-vision") # vision_model = providers.MistralProvider("pixtral") +# vision_model = providers.MiniMaxProvider("minimax-m3") +# vision_model = providers.MiniMaxAnthropicProvider("minimax-m3") #vision_model = providers.GroqProvider("llama-3.2") vision_model = providers.OpenRouterProvider("qwen-2.5-vl") @@ -18,4 +20,6 @@ # action_model = providers.AnthropicProvider("claude-3.5-sonnet") # vision_model = providers.MoonshotProvider("moonshot-v1-vision") # action_model = providers.MistralProvider("mistral") -action_model = providers.GroqProvider("llama-3.3") \ No newline at end of file +# action_model = providers.MiniMaxProvider("minimax-m2.7") +# action_model = providers.MiniMaxAnthropicProvider("minimax-m2.7") +action_model = providers.GroqProvider("llama-3.3") diff --git a/os_computer_use/llm_provider.py b/os_computer_use/llm_provider.py index 86ce797..6a13a7a 100644 --- a/os_computer_use/llm_provider.py +++ b/os_computer_use/llm_provider.py @@ -175,7 +175,10 @@ def call(self, messages, functions=None): class AnthropicBaseProvider(LLMProvider): def create_client(self): - return Anthropic(api_key=self.api_key).messages + client_kwargs = {"api_key": self.api_key} + if self.base_url: + client_kwargs["base_url"] = self.base_url + return Anthropic(**client_kwargs).messages def create_function_def(self, name, details, properties, required): return { @@ -188,13 +191,21 @@ def create_function_def(self, name, details, properties, required): }, } - def create_image_block(self, base64_image): + def create_image_block(self, image_data: bytes): + image_type = "png" + try: + with Image.open(io.BytesIO(image_data)) as img: + image_type = img.format.lower() + except Exception as e: + print(f"Error detecting image type: {e}") + + encoded = base64.b64encode(image_data).decode("utf-8") return { "type": "image", "source": { "type": "base64", - "media_type": "image/png", - "data": base64_image, + "media_type": f"image/{image_type}", + "data": encoded, }, } diff --git a/os_computer_use/providers.py b/os_computer_use/providers.py index 9f25f48..6ff1ab4 100644 --- a/os_computer_use/providers.py +++ b/os_computer_use/providers.py @@ -80,6 +80,26 @@ class MistralProvider(MistralBaseProvider): } +_MINIMAX_ALIASES = { + "minimax-m3": "MiniMax-M3", + "minimax-m2.7": "MiniMax-M2.7", +} + + +class MiniMaxProvider(OpenAIBaseProvider): + base_url = os.getenv("MINIMAX_OPENAI_BASE_URL", "https://api.minimax.io/v1") + api_key = os.getenv("MINIMAX_API_KEY") + aliases = _MINIMAX_ALIASES + + +class MiniMaxAnthropicProvider(AnthropicBaseProvider): + base_url = os.getenv( + "MINIMAX_ANTHROPIC_BASE_URL", "https://api.minimax.io/anthropic" + ) + api_key = os.getenv("MINIMAX_API_KEY") + aliases = _MINIMAX_ALIASES + + class MoonshotProvider(OpenAIBaseProvider): base_url = "https://api.moonshot.cn/v1" api_key = os.getenv("MOONSHOT_API_KEY") diff --git a/tests/test_minimax_provider.py b/tests/test_minimax_provider.py new file mode 100644 index 0000000..79bf007 --- /dev/null +++ b/tests/test_minimax_provider.py @@ -0,0 +1,142 @@ +import base64 +import json +import unittest +from pathlib import Path +from unittest.mock import patch + +from anthropic import Anthropic +from httpx import Client, MockTransport, Response +from openai import OpenAI + +from os_computer_use.llm_provider import Message +from os_computer_use.providers import MiniMaxAnthropicProvider, MiniMaxProvider + + +class MiniMaxProviderTests(unittest.TestCase): + def test_aliases_cover_supported_models(self): + expected = { + "minimax-m3": "MiniMax-M3", + "minimax-m2.7": "MiniMax-M2.7", + } + self.assertEqual(MiniMaxProvider.aliases, expected) + self.assertEqual(MiniMaxAnthropicProvider.aliases, expected) + + def test_openai_endpoints_append_chat_completions_once(self): + endpoints = ( + ("https://api.minimax.io/v1", "/v1/chat/completions"), + ("https://api.minimaxi.com/v1", "/v1/chat/completions"), + ) + + for base_url, expected_path in endpoints: + with self.subTest(base_url=base_url): + seen_paths = [] + + def handler(request): + seen_paths.append(request.url.path) + return Response( + 200, + json={ + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 0, + "model": "MiniMax-M3", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "ok", + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + }, + }, + ) + + sdk_client = OpenAI( + api_key="test", + base_url=base_url, + http_client=Client(transport=MockTransport(handler)), + ) + try: + with ( + patch.object(MiniMaxProvider, "base_url", base_url), + patch( + "os_computer_use.llm_provider.OpenAI", + return_value=sdk_client, + ), + ): + provider = MiniMaxProvider("minimax-m3") + self.assertEqual( + provider.call([Message("hello", role="user")]), "ok" + ) + finally: + sdk_client.close() + + self.assertEqual(seen_paths, [expected_path]) + + def test_anthropic_endpoints_append_v1_messages_once(self): + endpoints = ( + ("https://api.minimax.io/anthropic", "/anthropic/v1/messages"), + ("https://api.minimaxi.com/anthropic", "/anthropic/v1/messages"), + ) + image_data = Path(__file__).with_name("test_screenshot.png").read_bytes() + + for base_url, expected_path in endpoints: + with self.subTest(base_url=base_url): + seen_requests = [] + + def handler(request): + seen_requests.append( + (request.url.path, json.loads(request.content)) + ) + return Response( + 200, + json={ + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": "MiniMax-M3", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + sdk_client = Anthropic( + api_key="test", + base_url=base_url, + http_client=Client(transport=MockTransport(handler)), + ) + try: + with ( + patch.object(MiniMaxAnthropicProvider, "base_url", base_url), + patch( + "os_computer_use.llm_provider.Anthropic", + return_value=sdk_client, + ), + ): + provider = MiniMaxAnthropicProvider("minimax-m3") + self.assertEqual( + provider.call([Message([image_data], role="user")]), "ok" + ) + finally: + sdk_client.close() + + self.assertEqual(len(seen_requests), 1) + path, body = seen_requests[0] + self.assertEqual(path, expected_path) + self.assertEqual(path.count("/v1/messages"), 1) + encoded_image = body["messages"][0]["content"][0]["source"]["data"] + self.assertIsInstance(encoded_image, str) + self.assertEqual(base64.b64decode(encoded_image), image_data) + + +if __name__ == "__main__": + unittest.main()