diff --git a/atomic-agents/atomic_agents/agents/atomic_agent.py b/atomic-agents/atomic_agents/agents/atomic_agent.py index d2ccfe6e..23f8fea5 100644 --- a/atomic-agents/atomic_agents/agents/atomic_agent.py +++ b/atomic-agents/atomic_agents/agents/atomic_agent.py @@ -429,9 +429,9 @@ def _serialize_history_for_token_count(self) -> List[Dict[str, Any]]: """ Serialize conversation history for token counting, handling multimodal content. - This method converts instructor multimodal objects (Image, Audio, PDF) to the - OpenAI format that LiteLLM's token counter expects. Text content is also - converted to the proper multimodal text format when mixed with media. + This method converts Instructor multimodal objects (Image, Audio, PDF) to the + format that LiteLLM's token counter expects. Native content-part dictionaries + are preserved, and text content is wrapped when mixed with media. Returns: List[Dict[str, Any]]: History messages in LiteLLM-compatible format. @@ -449,6 +449,8 @@ def _serialize_history_for_token_count(self) -> List[Dict[str, Any]]: if isinstance(item, str): # Text content - wrap in OpenAI text format serialized_content.append({"type": "text", "text": item}) + elif isinstance(item, dict): + serialized_content.append(item) elif isinstance(item, (Image, Audio, PDF)): # Multimodal object - use instructor's to_openai method try: diff --git a/atomic-agents/atomic_agents/context/chat_history.py b/atomic-agents/atomic_agents/context/chat_history.py index e4fbf91e..560e1de8 100644 --- a/atomic-agents/atomic_agents/context/chat_history.py +++ b/atomic-agents/atomic_agents/context/chat_history.py @@ -14,6 +14,14 @@ INSTRUCTOR_MULTIMODAL_TYPES = (Image, Audio, PDF) +def _is_video_content_part(obj) -> bool: + if not isinstance(obj, dict) or obj.get("type") != "video_url": + return False + + video_url = obj.get("video_url") + return isinstance(video_url, dict) and isinstance(video_url.get("url"), str) + + class Message(BaseModel): """ Represents a message in the chat history. @@ -126,8 +134,9 @@ def _extract_multimodal_info(obj): Recursively extract multimodal objects and build a Pydantic-compatible exclude spec. Walks the object tree to find all Instructor multimodal types (Image, Audio, PDF) - at any nesting depth, collecting them into a flat list and building an exclude - specification that can be passed to model_dump_json(exclude=...). + and native video content parts at any nesting depth, collecting them into a flat + list and building an exclude specification that can be passed to + model_dump_json(exclude=...). Args: obj: The object to inspect (BaseIOSchema, list, dict, or primitive). @@ -137,7 +146,7 @@ def _extract_multimodal_info(obj): - multimodal_objects: flat list of all multimodal objects found - exclude_spec: Pydantic exclude dict, True (exclude entirely), or None """ - if isinstance(obj, INSTRUCTOR_MULTIMODAL_TYPES): + if isinstance(obj, INSTRUCTOR_MULTIMODAL_TYPES) or _is_video_content_part(obj): return [obj], True if hasattr(obj, "__class__") and hasattr(obj.__class__, "model_fields"): diff --git a/atomic-agents/tests/agents/test_atomic_agent.py b/atomic-agents/tests/agents/test_atomic_agent.py index c626791a..e8f86694 100644 --- a/atomic-agents/tests/agents/test_atomic_agent.py +++ b/atomic-agents/tests/agents/test_atomic_agent.py @@ -1120,6 +1120,29 @@ class MultimodalInputSchema(BaseIOSchema): assert image_entry["image_url"]["url"] == "https://example.com/test.png" +def test_serialize_history_for_token_count_preserves_content_part(agent, mock_history): + video_part = { + "type": "video_url", + "video_url": { + "url": "mm_file://video-file", + "detail": "default", + }, + } + mock_history.get_history.return_value = [ + { + "role": "user", + "content": ['{"prompt":"Summarize the video"}', video_part], + } + ] + + serialized = agent._serialize_history_for_token_count() + + assert serialized[0]["content"] == [ + {"type": "text", "text": '{"prompt":"Summarize the video"}'}, + video_part, + ] + + # --- Tests for tool_result_role and Gemini system message remapping (issue #221) --- diff --git a/atomic-agents/tests/context/test_chat_history.py b/atomic-agents/tests/context/test_chat_history.py index 4089aff2..d6e4a7c3 100644 --- a/atomic-agents/tests/context/test_chat_history.py +++ b/atomic-agents/tests/context/test_chat_history.py @@ -646,6 +646,33 @@ class MessageInput(BaseIOSchema): assert img2 in result[0]["content"] +def test_get_history_video_content_part(history): + """Native video content parts are separated from structured text.""" + + class VideoInput(BaseIOSchema): + """Input containing a video content part.""" + + prompt: str = Field(..., description="Instruction for the video") + video: Dict[str, object] = Field(..., description="Video content part") + + video_part = { + "type": "video_url", + "video_url": { + "url": "mm_file://video-file", + "detail": "default", + "fps": 1, + }, + } + history.add_message("user", VideoInput(prompt="Summarize the video", video=video_part)) + + result = history.get_history() + + assert len(result) == 1 + assert isinstance(result[0]["content"], list) + assert json.loads(result[0]["content"][0]) == {"prompt": "Summarize the video"} + assert result[0]["content"][1] == video_part + + def test_get_history_list_of_nested_schemas_with_multimodal(history): """Multiple nested schemas each containing multimodal objects"""