From e606bfe31dcd687071ae3b2b727df75a9290e936 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 May 2025 20:17:07 -0700 Subject: [PATCH] [Feat - Contributor PR] Add Video support for Bedrock Converse (#11166) * feat: add video support for bedrock converse api (#11043) * fixes: bedrock add video support * fixes: bedrock add video support --------- Co-authored-by: yytdfc --- .../prompt_templates/factory.py | 26 ++++++++++++++++-- .../bedrock/chat/converse_transformation.py | 5 +++- litellm/types/llms/bedrock.py | 9 +++++++ ...llm_core_utils_prompt_templates_factory.py | 27 +++++++++++++++++++ 4 files changed, 64 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 2386e82d4a..7d862af6ae 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -2267,6 +2267,7 @@ from litellm.types.llms.bedrock import ( ) from litellm.types.llms.bedrock import ToolSpecBlock as BedrockToolSpecBlock from litellm.types.llms.bedrock import ToolUseBlock as BedrockToolUseBlock +from litellm.types.llms.bedrock import VideoBlock as BedrockVideoBlock def _parse_content_type(content_type: str) -> str: @@ -2356,9 +2357,15 @@ class BedrockImageProcessor: supported_doc_formats = ( litellm.AmazonConverseConfig().get_supported_document_types() ) + supported_video_formats = ( + litellm.AmazonConverseConfig().get_supported_video_types() + ) document_types = ["application", "text"] is_document = any(mime_type.startswith(doc_type) for doc_type in document_types) + supported_image_and_video_formats: List[str] = ( + supported_video_formats + supported_image_formats + ) if is_document: potential_extensions = mimetypes.guess_all_extensions(mime_type) @@ -2376,9 +2383,12 @@ class BedrockImageProcessor: # Use first valid extension instead of provided image_format return valid_extensions[0] else: - if image_format not in supported_image_formats: + ######################################################### + # Check if image_format is an image or video + ######################################################### + if image_format not in supported_image_and_video_formats: raise ValueError( - f"Unsupported image format: {image_format}. Supported formats: {supported_image_formats}" + f"Unsupported image format: {image_format}. Supported formats: {supported_image_and_video_formats}" ) return image_format @@ -2392,6 +2402,14 @@ class BedrockImageProcessor: document_types = ["application", "text"] is_document = any(mime_type.startswith(doc_type) for doc_type in document_types) + supported_video_formats = ( + litellm.AmazonConverseConfig().get_supported_video_types() + ) + is_video = any( + image_format.startswith(video_type) + for video_type in supported_video_formats + ) + if is_document: return BedrockContentBlock( document=BedrockDocumentBlock( @@ -2400,6 +2418,10 @@ class BedrockImageProcessor: name=f"DocumentPDFmessages_{str(uuid.uuid4())}", ) ) + elif is_video: + return BedrockContentBlock( + video=BedrockVideoBlock(source=_blob, format=image_format) + ) else: return BedrockContentBlock( image=BedrockImageBlock(source=_blob, format=image_format) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index b0e7703fe7..2fc3020bea 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -191,8 +191,11 @@ class AmazonConverseConfig(BaseConfig): def get_supported_document_types(self) -> List[str]: return ["pdf", "csv", "doc", "docx", "xls", "xlsx", "html", "txt", "md"] + def get_supported_video_types(self) -> List[str]: + return ["mp4", "mov", "mkv", "webm", "flv", "mpeg", "mpg", "wmv", "3gp"] + def get_all_supported_content_types(self) -> List[str]: - return self.get_supported_image_types() + self.get_supported_document_types() + return self.get_supported_image_types() + self.get_supported_document_types() + self.get_supported_video_types() def _create_json_tool_call_for_response_format( self, diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py index 8c95349e4a..0d9655ac71 100644 --- a/litellm/types/llms/bedrock.py +++ b/litellm/types/llms/bedrock.py @@ -36,6 +36,14 @@ class ImageBlock(TypedDict): source: SourceBlock +BedrockVideoTypes = Literal["mp4", "mov", "mkv", "webm", "flv", "mpeg", "mpg", "wmv", "3gp"] + + +class VideoBlock(TypedDict): + format: Union[BedrockVideoTypes, str] + source: SourceBlock + + BedrockDocumentTypes = Literal[ "pdf", "csv", "doc", "docx", "xls", "xlsx", "html", "txt", "md" ] @@ -85,6 +93,7 @@ class BedrockConverseReasoningContentBlockDelta(TypedDict, total=False): class ContentBlock(TypedDict, total=False): text: str image: ImageBlock + video: VideoBlock document: DocumentBlock toolResult: ToolResultBlock toolUse: ToolUseBlock diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index a8b9cd8a44..fcb2e664ef 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -7,6 +7,7 @@ import litellm from litellm.litellm_core_utils.prompt_templates.factory import ( BAD_MESSAGE_ERROR_STR, BedrockConverseMessagesProcessor, + BedrockImageProcessor, ollama_pt, ) @@ -82,6 +83,32 @@ async def test_anthropic_bedrock_thinking_blocks_with_none_content(): ) +def test_bedrock_validate_format_image_or_video(): + """Test the _validate_format method for images, videos, and documents""" + + # Test valid image formats + valid_image_formats = ["png", "jpeg", "gif", "webp"] + for format in valid_image_formats: + result = BedrockImageProcessor._validate_format(f"image/{format}", format) + assert result == format, f"Expected {format}, got {result}" + + # Test valid video formats + valid_video_formats = [ + "mp4", + "mov", + "mkv", + "webm", + "flv", + "mpeg", + "mpg", + "wmv", + "3gp", + ] + for format in valid_video_formats: + result = BedrockImageProcessor._validate_format(f"video/{format}", format) + assert result == format, f"Expected {format}, got {result}" + + # def test_ollama_pt_consecutive_system_messages(): # """Test handling consecutive system messages""" # messages = [