[Feat - Contributor PR] Add Video support for Bedrock Converse (#11166)

* feat: add video support for bedrock converse api (#11043)

* fixes: bedrock add video support

* fixes: bedrock add video support

---------

Co-authored-by: yytdfc <fuchen@foxmail.com>

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -2267,6 +2267,7 @@ from litellm.types.llms.bedrock import (
 )
 from litellm.types.llms.bedrock import ToolSpecBlock as BedrockToolSpecBlock
 from litellm.types.llms.bedrock import ToolUseBlock as BedrockToolUseBlock
+from litellm.types.llms.bedrock import VideoBlock as BedrockVideoBlock
 
 
 def _parse_content_type(content_type: str) -> str:
@@ -2356,9 +2357,15 @@ class BedrockImageProcessor:
         supported_doc_formats = (
             litellm.AmazonConverseConfig().get_supported_document_types()
         )
+        supported_video_formats = (
+            litellm.AmazonConverseConfig().get_supported_video_types()
+        )
 
         document_types = ["application", "text"]
         is_document = any(mime_type.startswith(doc_type) for doc_type in document_types)
+        supported_image_and_video_formats: List[str] = (
+            supported_video_formats + supported_image_formats
+        )
 
         if is_document:
             potential_extensions = mimetypes.guess_all_extensions(mime_type)
@@ -2376,9 +2383,12 @@ class BedrockImageProcessor:
             # Use first valid extension instead of provided image_format
             return valid_extensions[0]
         else:
-            if image_format not in supported_image_formats:
+            #########################################################
+            # Check if image_format is an image or video
+            #########################################################
+            if image_format not in supported_image_and_video_formats:
                 raise ValueError(
-                    f"Unsupported image format: {image_format}. Supported formats: {supported_image_formats}"
+                    f"Unsupported image format: {image_format}. Supported formats: {supported_image_and_video_formats}"
                 )
             return image_format
 
@@ -2392,6 +2402,14 @@ class BedrockImageProcessor:
         document_types = ["application", "text"]
         is_document = any(mime_type.startswith(doc_type) for doc_type in document_types)
 
+        supported_video_formats = (
+            litellm.AmazonConverseConfig().get_supported_video_types()
+        )
+        is_video = any(
+            image_format.startswith(video_type)
+            for video_type in supported_video_formats
+        )
+
         if is_document:
             return BedrockContentBlock(
                 document=BedrockDocumentBlock(
@@ -2400,6 +2418,10 @@ class BedrockImageProcessor:
                     name=f"DocumentPDFmessages_{str(uuid.uuid4())}",
                 )
             )
+        elif is_video:
+            return BedrockContentBlock(
+                video=BedrockVideoBlock(source=_blob, format=image_format)
+            )
         else:
             return BedrockContentBlock(
                 image=BedrockImageBlock(source=_blob, format=image_format)

litellm/llms/bedrock/chat/converse_transformation.py

@@ -191,8 +191,11 @@ class AmazonConverseConfig(BaseConfig):
     def get_supported_document_types(self) -> List[str]:
         return ["pdf", "csv", "doc", "docx", "xls", "xlsx", "html", "txt", "md"]
 
+    def get_supported_video_types(self) -> List[str]:
+        return ["mp4", "mov", "mkv", "webm", "flv", "mpeg", "mpg", "wmv", "3gp"]
+
     def get_all_supported_content_types(self) -> List[str]:
-        return self.get_supported_image_types() + self.get_supported_document_types()
+        return self.get_supported_image_types() + self.get_supported_document_types() + self.get_supported_video_types()
 
     def _create_json_tool_call_for_response_format(
         self,

litellm/types/llms/bedrock.py

@@ -36,6 +36,14 @@ class ImageBlock(TypedDict):
     source: SourceBlock
 
 
+BedrockVideoTypes = Literal["mp4", "mov", "mkv", "webm", "flv", "mpeg", "mpg", "wmv", "3gp"]
+
+
+class VideoBlock(TypedDict):
+    format: Union[BedrockVideoTypes, str]
+    source: SourceBlock
+
+
 BedrockDocumentTypes = Literal[
     "pdf", "csv", "doc", "docx", "xls", "xlsx", "html", "txt", "md"
 ]
@@ -85,6 +93,7 @@ class BedrockConverseReasoningContentBlockDelta(TypedDict, total=False):
 class ContentBlock(TypedDict, total=False):
     text: str
     image: ImageBlock
+    video: VideoBlock
     document: DocumentBlock
     toolResult: ToolResultBlock
     toolUse: ToolUseBlock

tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py

@@ -7,6 +7,7 @@ import litellm
 from litellm.litellm_core_utils.prompt_templates.factory import (
     BAD_MESSAGE_ERROR_STR,
     BedrockConverseMessagesProcessor,
+    BedrockImageProcessor,
     ollama_pt,
 )
 
@@ -82,6 +83,32 @@ async def test_anthropic_bedrock_thinking_blocks_with_none_content():
     )
 
 
+def test_bedrock_validate_format_image_or_video():
+    """Test the _validate_format method for images, videos, and documents"""
+
+    # Test valid image formats
+    valid_image_formats = ["png", "jpeg", "gif", "webp"]
+    for format in valid_image_formats:
+        result = BedrockImageProcessor._validate_format(f"image/{format}", format)
+        assert result == format, f"Expected {format}, got {result}"
+
+    # Test valid video formats
+    valid_video_formats = [
+        "mp4",
+        "mov",
+        "mkv",
+        "webm",
+        "flv",
+        "mpeg",
+        "mpg",
+        "wmv",
+        "3gp",
+    ]
+    for format in valid_video_formats:
+        result = BedrockImageProcessor._validate_format(f"video/{format}", format)
+        assert result == format, f"Expected {format}, got {result}"
+
+
 # def test_ollama_pt_consecutive_system_messages():
 #     """Test handling consecutive system messages"""
 #     messages = [