diff --git a/docs/my-website/docs/anthropic_unified/index.md b/docs/my-website/docs/anthropic_unified/index.md index 9981547ce1..f8a50e14da 100644 --- a/docs/my-website/docs/anthropic_unified/index.md +++ b/docs/my-website/docs/anthropic_unified/index.md @@ -506,12 +506,15 @@ Request body will be in the Anthropic messages API format. **litellm follows the A system prompt providing context or specific instructions to the model. - **temperature** (number): Controls randomness in the model's responses. Valid range: `0 < temperature < 1`. -- **thinking** (object): +- **thinking** (object): Configuration for enabling extended thinking. If enabled, it includes: - - **budget_tokens** (integer): + - **budget_tokens** (integer): Minimum of 1024 tokens (and less than `max_tokens`). - - **type** (enum): + - **type** (enum): E.g., `"enabled"`. + - **summary** (string, optional): + Enables the summary style for thinking blocks. Possible values: `"auto"`, `"concise"`, `"detailed"`, `"disabled"`. + When routing to non-Anthropic providers (e.g., `openai/gpt-5.1`), the `summary` value is preserved and forwarded to the downstream API. - **tool_choice** (object): Instructs how the model should utilize any provided tools. - **tools** (array of objects): diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 16752b030b..b1d52b506e 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -813,7 +813,7 @@ router_settings: | LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development) | LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers | LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60 -| LITELLM_REASONING_AUTO_SUMMARY | If set to "true", automatically enables detailed reasoning summaries for reasoning models (e.g., o1, o3-mini, deepseek-reasoner). When enabled, adds `summary: "detailed"` to reasoning effort configurations. Default is "false" +| LITELLM_REASONING_AUTO_SUMMARY | If set to "true", automatically enables detailed reasoning summaries (`summary: "detailed"`) for reasoning models across all translation paths (Anthropic adapter, Responses API, etc.). Default is "false" | LITELLM_SALT_KEY | Salt key for encryption in LiteLLM | LITELLM_SSL_CIPHERS | SSL/TLS cipher configuration for faster handshakes. Controls cipher suite preferences for OpenSSL connections. | LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM diff --git a/docs/my-website/docs/reasoning_content.md b/docs/my-website/docs/reasoning_content.md index 313228d511..3f7b76dab6 100644 --- a/docs/my-website/docs/reasoning_content.md +++ b/docs/my-website/docs/reasoning_content.md @@ -700,3 +700,69 @@ response = litellm.completion( reasoning_effort={"effort": "low", "summary": "detailed"}, # Explicit control ) ``` + +### Summary Preservation via `/v1/messages` Adapter + +When using the Anthropic `/v1/messages` adapter to route non-Claude models (e.g., `openai/gpt-5.1`), the `thinking.summary` value is preserved and forwarded to the downstream provider. For example: + +```python +import litellm + +response = await litellm.anthropic.messages.acreate( + model="openai/gpt-5.1", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=8096, + thinking={"type": "enabled", "budget_tokens": 5000, "summary": "concise"}, +) +# The summary="concise" is preserved when routing to OpenAI's Responses API +``` + +### Enabling Default Summary Injection for `/v1/messages` Adapter + +When the Anthropic `/v1/messages` adapter translates `thinking` parameters to OpenAI `reasoning_effort` for non-Claude models, you can opt-in to automatic `summary="detailed"` injection using the `reasoning_auto_summary` flag. This ensures that reasoning text is returned in the response (matching the Anthropic thinking behavior). + +To **enable** this default injection, use the `reasoning_auto_summary` flag: + + + + +```python +import litellm + +# Enable default summary="detailed" injection +litellm.reasoning_auto_summary = True + +response = await litellm.anthropic.messages.acreate( + model="openai/gpt-5.1", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=8096, + thinking={"type": "enabled", "budget_tokens": 5000}, +) +# summary="detailed" will be automatically added to reasoning_effort +``` + + + + + +```bash +export LITELLM_REASONING_AUTO_SUMMARY=true +``` + + + + + +```yaml +litellm_settings: + reasoning_auto_summary: true +``` + + + + +:::info + +This flag only affects the automatic injection of `summary="detailed"` when no user-provided summary is present. If you explicitly pass `thinking.summary` (e.g., `"concise"` or `"auto"`), your value is always preserved regardless of this flag. + +::: diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index f5856ab1f4..ee4cdbcdf3 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -240,10 +240,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if key in ("max_tokens", "max_completion_tokens"): responses_api_request["max_output_tokens"] = value elif key == "tools" and value is not None: - responses_api_request["tools"] = ( - self._convert_tools_to_responses_format( - cast(List[Dict[str, Any]], value) - ) + responses_api_request[ + "tools" + ] = self._convert_tools_to_responses_format( + cast(List[Dict[str, Any]], value) ) elif key == "response_format": text_format = self._transform_response_format_to_text_format(value) @@ -1072,9 +1072,9 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): ) if provider_specific_fields: - function_chunk["provider_specific_fields"] = ( - provider_specific_fields - ) + function_chunk[ + "provider_specific_fields" + ] = provider_specific_fields tool_call_index = parsed_chunk.get("output_index", 0) tool_call_chunk = ChatCompletionToolCallChunk( @@ -1147,9 +1147,9 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): # Add provider_specific_fields to function if present if provider_specific_fields: - function_chunk["provider_specific_fields"] = ( - provider_specific_fields - ) + function_chunk[ + "provider_specific_fields" + ] = provider_specific_fields tool_call_index = parsed_chunk.get("output_index", 0) tool_call_chunk = ChatCompletionToolCallChunk( diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index ad4bee0834..b931d7ecfe 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -83,7 +83,9 @@ class LangsmithLogger(CustomBatchLogger): if _batch_size: self.batch_size = int(_batch_size) self.log_queue: List[LangsmithQueueObject] = [] - self._flush_task: Optional[asyncio.Task[Any]] = self._start_periodic_flush_task() + self._flush_task: Optional[ + asyncio.Task[Any] + ] = self._start_periodic_flush_task() def _start_periodic_flush_task(self) -> Optional[asyncio.Task[Any]]: """Start the periodic flush task only when an event loop is already running.""" diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 56f7f305dc..5323f692b8 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1686,7 +1686,9 @@ class Logging(LiteLLMLoggingBaseClass): ) return logging_result - def _merge_hidden_params_from_response_into_metadata(self, logging_result: Any) -> None: + def _merge_hidden_params_from_response_into_metadata( + self, logging_result: Any + ) -> None: """ Copy response._hidden_params into litellm_params.metadata['hidden_params']. @@ -1704,9 +1706,9 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["litellm_params"].setdefault("metadata", {}) if self.model_call_details["litellm_params"]["metadata"] is None: self.model_call_details["litellm_params"]["metadata"] = {} - self.model_call_details["litellm_params"]["metadata"]["hidden_params"] = getattr( - logging_result, "_hidden_params", {} - ) + self.model_call_details["litellm_params"]["metadata"][ + "hidden_params" + ] = getattr(logging_result, "_hidden_params", {}) def _process_hidden_params_and_response_cost( self, diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index 8b1b21a0f9..897ca3bf89 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -15,6 +15,9 @@ import litellm from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( AnthropicAdapter, ) +from litellm.llms.anthropic.experimental_pass_through.utils import ( + is_reasoning_auto_summary_enabled, +) from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) @@ -44,8 +47,9 @@ class LiteLLMMessagesToCompletionTransformationHandler: For OpenAI models, Chat Completions typically does not return reasoning text (only token accounting). To return a thinking-like content block in the - Anthropic response format, we route the request through OpenAI's Responses API - and request a reasoning summary. + Anthropic response format, we route the request through OpenAI's Responses API. + If the user provides a `summary` field in the thinking dict, it is passed + through to the OpenAI reasoning params (opt-in per OpenAI spec). """ custom_llm_provider = completion_kwargs.get("custom_llm_provider") if custom_llm_provider is None: @@ -78,20 +82,29 @@ class LiteLLMMessagesToCompletionTransformationHandler: # Prefix model with "responses/" to route to OpenAI Responses API completion_kwargs["model"] = f"responses/{model}" + auto_summary = is_reasoning_auto_summary_enabled() + reasoning_effort = completion_kwargs.get("reasoning_effort") + summary = thinking.get("summary") if isinstance(reasoning_effort, str) and reasoning_effort: - completion_kwargs["reasoning_effort"] = { - "effort": reasoning_effort, - "summary": "detailed", - } + reasoning_dict: Dict[str, Any] = {"effort": reasoning_effort} + if summary: + reasoning_dict["summary"] = summary + elif auto_summary: + reasoning_dict["summary"] = "detailed" + completion_kwargs["reasoning_effort"] = reasoning_dict elif isinstance(reasoning_effort, dict): if ( "summary" not in reasoning_effort and "generate_summary" not in reasoning_effort ): - updated_reasoning_effort = dict(reasoning_effort) - updated_reasoning_effort["summary"] = "detailed" - completion_kwargs["reasoning_effort"] = updated_reasoning_effort + effective_summary = ( + summary if summary else ("detailed" if auto_summary else None) + ) + if effective_summary: + updated_reasoning_effort = dict(reasoning_effort) + updated_reasoning_effort["summary"] = effective_summary + completion_kwargs["reasoning_effort"] = updated_reasoning_effort @staticmethod def _prepare_completion_kwargs( diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index b9a8d1de48..ed49943b7f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -14,6 +14,10 @@ from typing import ( cast, ) +from litellm.llms.anthropic.experimental_pass_through.utils import ( + is_reasoning_auto_summary_enabled, +) + # OpenAI has a 64-character limit for function/tool names # Anthropic does not have this limit, so we need to truncate long names OPENAI_MAX_TOOL_NAME_LENGTH = 64 @@ -734,6 +738,24 @@ class LiteLLMAnthropicMessagesAdapter: thinking ) if reasoning_effort: + summary = ( + thinking.get("summary") if isinstance(thinking, dict) else None + ) + auto_summary = is_reasoning_auto_summary_enabled() + if summary: + return { + "reasoning_effort": { + "effort": reasoning_effort, + "summary": summary, + } + } + elif auto_summary: + return { + "reasoning_effort": { + "effort": reasoning_effort, + "summary": "detailed", + } + } return {"reasoning_effort": reasoning_effort} return {} @@ -869,19 +891,25 @@ class LiteLLMAnthropicMessagesAdapter: # Handle array items if "items" in schema: - LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(schema["items"]) + LiteLLMAnthropicMessagesAdapter._add_additional_properties_false( + schema["items"] + ) # Handle anyOf/oneOf/allOf for key in ("anyOf", "oneOf", "allOf"): if key in schema: for sub_schema in schema[key]: - LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(sub_schema) + LiteLLMAnthropicMessagesAdapter._add_additional_properties_false( + sub_schema + ) # Handle $defs / definitions for key in ("$defs", "definitions"): if key in schema: for def_schema in schema[key].values(): - LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(def_schema) + LiteLLMAnthropicMessagesAdapter._add_additional_properties_false( + def_schema + ) def _add_system_message_to_messages( self, @@ -918,6 +946,144 @@ class LiteLLMAnthropicMessagesAdapter: ChatCompletionSystemMessage(role="system", content=openai_system_content), # type: ignore ) + def _translate_metadata_to_openai( + self, + anthropic_message_request: AnthropicMessagesRequest, + new_kwargs: ChatCompletionRequest, + ) -> None: + """Translate metadata fields from Anthropic request to OpenAI request.""" + if "metadata" in anthropic_message_request: + metadata = anthropic_message_request["metadata"] + if metadata and "user_id" in metadata: + new_kwargs["user"] = metadata["user_id"] + + if "litellm_metadata" in anthropic_message_request: + # metadata will be passed to litellm.acompletion(), it's a litellm_param + new_kwargs["metadata"] = anthropic_message_request.pop("litellm_metadata") + + def _translate_tool_choice_to_openai( + self, + anthropic_message_request: AnthropicMessagesRequest, + new_kwargs: ChatCompletionRequest, + ) -> None: + """Translate Anthropic tool_choice to OpenAI format.""" + if "tool_choice" not in anthropic_message_request: + return + tool_choice = anthropic_message_request["tool_choice"] + if not tool_choice: + return + new_kwargs["tool_choice"] = self.translate_anthropic_tool_choice_to_openai( + tool_choice=cast(AnthropicMessagesToolChoice, tool_choice) + ) + + def _translate_tools_to_openai( + self, + anthropic_message_request: AnthropicMessagesRequest, + new_kwargs: ChatCompletionRequest, + ) -> Dict[str, str]: + """Translate tools and extract web_search_options when needed.""" + if "tools" not in anthropic_message_request: + return {} + + tools = anthropic_message_request["tools"] + if not tools: + return {} + + web_search_tools: List[AllAnthropicToolsValues] = [] + regular_tools: List[AllAnthropicToolsValues] = [] + for tool in tools: + cast_tool = cast(Dict[str, Any], tool) + if self._is_web_search_tool(cast_tool): + web_search_tools.append(cast(AllAnthropicToolsValues, tool)) + else: + regular_tools.append(cast(AllAnthropicToolsValues, tool)) + + if web_search_tools: + new_kwargs["web_search_options"] = {} # type: ignore + + if not regular_tools: + return {} + + translated_tools, tool_name_mapping = self.translate_anthropic_tools_to_openai( + tools=regular_tools, + model=new_kwargs.get("model"), + ) + new_kwargs["tools"] = translated_tools + return tool_name_mapping + + def _translate_thinking_to_openai( + self, + anthropic_message_request: AnthropicMessagesRequest, + new_kwargs: ChatCompletionRequest, + ) -> None: + """Translate Anthropic thinking to either thinking or reasoning_effort.""" + if "thinking" not in anthropic_message_request: + return + + thinking = anthropic_message_request["thinking"] + if not thinking: + return + + model = new_kwargs.get("model", "") + if self.is_anthropic_claude_model(model): + new_kwargs["thinking"] = thinking # type: ignore + return + + reasoning_effort = self.translate_anthropic_thinking_to_reasoning_effort( + cast(Dict[str, Any], thinking) + ) + if not reasoning_effort: + return + + summary = thinking.get("summary") if isinstance(thinking, dict) else None + auto_summary = is_reasoning_auto_summary_enabled() + if summary: + new_kwargs["reasoning_effort"] = cast( + Any, + { + "effort": reasoning_effort, + "summary": summary, + }, + ) + elif auto_summary: + new_kwargs["reasoning_effort"] = cast( + Any, + { + "effort": reasoning_effort, + "summary": "detailed", + }, + ) + else: + new_kwargs["reasoning_effort"] = reasoning_effort + + def _translate_output_format_to_openai( + self, + anthropic_message_request: AnthropicMessagesRequest, + new_kwargs: ChatCompletionRequest, + ) -> None: + """Translate output_format to response_format when applicable.""" + if "output_format" not in anthropic_message_request: + return + output_format = anthropic_message_request["output_format"] + if not output_format: + return + response_format = self.translate_anthropic_output_format_to_openai( + output_format=output_format + ) + if response_format: + new_kwargs["response_format"] = response_format + + def _copy_untranslated_anthropic_params( + self, + anthropic_message_request: AnthropicMessagesRequest, + new_kwargs: ChatCompletionRequest, + ) -> None: + """Copy through anthropic params that do not require translation.""" + translatable_params = self.translatable_anthropic_params() + for k, v in anthropic_message_request.items(): + if k not in translatable_params: # pass remaining params as is + new_kwargs[k] = v # type: ignore + def translate_anthropic_to_openai( self, anthropic_message_request: AnthropicMessagesRequest ) -> Tuple[ChatCompletionRequest, Dict[str, str]]: @@ -958,83 +1124,35 @@ class LiteLLMAnthropicMessagesAdapter: "model": anthropic_message_request["model"], "messages": new_messages, } - ## CONVERT METADATA (user_id) - if "metadata" in anthropic_message_request: - metadata = anthropic_message_request["metadata"] - if metadata and "user_id" in metadata: - new_kwargs["user"] = metadata["user_id"] - - # Pass litellm proxy specific metadata - if "litellm_metadata" in anthropic_message_request: - # metadata will be passed to litellm.acompletion(), it's a litellm_param - new_kwargs["metadata"] = anthropic_message_request.pop("litellm_metadata") - + ## CONVERT METADATA (user_id + litellm metadata) + self._translate_metadata_to_openai( + anthropic_message_request=anthropic_message_request, + new_kwargs=new_kwargs, + ) ## CONVERT TOOL CHOICE - if "tool_choice" in anthropic_message_request: - tool_choice = anthropic_message_request["tool_choice"] - if tool_choice: - new_kwargs[ - "tool_choice" - ] = self.translate_anthropic_tool_choice_to_openai( - tool_choice=cast(AnthropicMessagesToolChoice, tool_choice) - ) + self._translate_tool_choice_to_openai( + anthropic_message_request=anthropic_message_request, + new_kwargs=new_kwargs, + ) ## CONVERT TOOLS - if "tools" in anthropic_message_request: - tools = anthropic_message_request["tools"] - if tools: - # Separate web search tools from regular tools - web_search_tools = [] - regular_tools = [] - for tool in tools: - if self._is_web_search_tool(cast(Dict[str, Any], tool)): - web_search_tools.append(tool) - else: - regular_tools.append(tool) - - # If web search tools are present, add web_search_options parameter - if web_search_tools: - new_kwargs["web_search_options"] = {} # type: ignore - - # Only translate regular tools (non-web-search) - if regular_tools: - ( - new_kwargs["tools"], - tool_name_mapping, - ) = self.translate_anthropic_tools_to_openai( - tools=cast(List[AllAnthropicToolsValues], regular_tools), - model=new_kwargs.get("model"), - ) - + tool_name_mapping = self._translate_tools_to_openai( + anthropic_message_request=anthropic_message_request, + new_kwargs=new_kwargs, + ) ## CONVERT THINKING - if "thinking" in anthropic_message_request: - thinking = anthropic_message_request["thinking"] - if thinking: - model = new_kwargs.get("model", "") - if self.is_anthropic_claude_model(model): - new_kwargs["thinking"] = thinking # type: ignore - else: - reasoning_effort = ( - self.translate_anthropic_thinking_to_reasoning_effort( - cast(Dict[str, Any], thinking) - ) - ) - if reasoning_effort: - new_kwargs["reasoning_effort"] = reasoning_effort - + self._translate_thinking_to_openai( + anthropic_message_request=anthropic_message_request, + new_kwargs=new_kwargs, + ) ## CONVERT OUTPUT_FORMAT to RESPONSE_FORMAT - if "output_format" in anthropic_message_request: - output_format = anthropic_message_request["output_format"] - if output_format: - response_format = self.translate_anthropic_output_format_to_openai( - output_format=output_format - ) - if response_format: - new_kwargs["response_format"] = response_format - - translatable_params = self.translatable_anthropic_params() - for k, v in anthropic_message_request.items(): - if k not in translatable_params: # pass remaining params as is - new_kwargs[k] = v # type: ignore + self._translate_output_format_to_openai( + anthropic_message_request=anthropic_message_request, + new_kwargs=new_kwargs, + ) + self._copy_untranslated_anthropic_params( + anthropic_message_request=anthropic_message_request, + new_kwargs=new_kwargs, + ) return new_kwargs, tool_name_mapping diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index ddd514146d..dae7044a5b 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -8,6 +8,9 @@ path used for OpenAI and Azure models. import json from typing import Any, Dict, List, Optional, Union, cast +from litellm.llms.anthropic.experimental_pass_through.utils import ( + is_reasoning_auto_summary_enabled, +) from litellm.types.llms.anthropic import ( AllAnthropicToolsValues, AnthopicMessagesAssistantMessageParam, @@ -94,7 +97,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ) elif btype == "image": url = self._translate_anthropic_image_source_to_url( - block.get("source", {}) + cast(dict, block.get("source", {})) ) if url: user_parts.append( @@ -267,7 +270,14 @@ class LiteLLMAnthropicToResponsesAPIAdapter: effort = "low" else: effort = "minimal" - return {"effort": effort, "summary": "detailed"} + auto_summary = is_reasoning_auto_summary_enabled() + result: Dict[str, Any] = {"effort": effort} + summary = thinking.get("summary") + if summary: + result["summary"] = summary + elif auto_summary: + result["summary"] = "detailed" + return result def translate_request( self, diff --git a/litellm/llms/anthropic/experimental_pass_through/utils.py b/litellm/llms/anthropic/experimental_pass_through/utils.py new file mode 100644 index 0000000000..6c1db6017b --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/utils.py @@ -0,0 +1,11 @@ +import os + +import litellm + + +def is_reasoning_auto_summary_enabled() -> bool: + """Check whether the default 'summary: detailed' injection is enabled (opt-in).""" + return ( + litellm.reasoning_auto_summary + or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true" + ) diff --git a/litellm/llms/gemini/image_generation/transformation.py b/litellm/llms/gemini/image_generation/transformation.py index 3e3f6162fc..b094fc133d 100644 --- a/litellm/llms/gemini/image_generation/transformation.py +++ b/litellm/llms/gemini/image_generation/transformation.py @@ -88,12 +88,15 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): tokens_details = usage_metadata.get("promptTokensDetails", []) for details in tokens_details: if isinstance(details, dict): - modality = details.get("modality") - token_count = details.get("tokenCount", 0) + modality = str(details.get("modality", "")).upper() + raw_token_count = details.get( + "tokenCount", details.get("token_count", 0) + ) + token_count = raw_token_count if isinstance(raw_token_count, int) else 0 if modality == "TEXT": - input_tokens_details.text_tokens = token_count + input_tokens_details.text_tokens += token_count elif modality == "IMAGE": - input_tokens_details.image_tokens = token_count + input_tokens_details.image_tokens += token_count return ImageUsage( input_tokens=usage_metadata.get("promptTokenCount", 0), diff --git a/litellm/llms/moonshot/chat/transformation.py b/litellm/llms/moonshot/chat/transformation.py index c97bd6c4e1..e4d7b5f033 100644 --- a/litellm/llms/moonshot/chat/transformation.py +++ b/litellm/llms/moonshot/chat/transformation.py @@ -168,7 +168,9 @@ class MoonshotChatConfig(OpenAIGPTConfig): if ( msg.get("role") == "assistant" and msg.get("tool_calls") - and not msg.get("reasoning_content") # Check using .get() which works for both dicts and Pydantic models + and not msg.get( + "reasoning_content" + ) # Check using .get() which works for both dicts and Pydantic models ): patched = dict(cast(dict, msg)) provider_fields = patched.get("provider_specific_fields") or {} diff --git a/litellm/llms/ovhcloud/chat/transformation.py b/litellm/llms/ovhcloud/chat/transformation.py index 84090fafd3..1416b782f1 100644 --- a/litellm/llms/ovhcloud/chat/transformation.py +++ b/litellm/llms/ovhcloud/chat/transformation.py @@ -28,9 +28,7 @@ class OVHCloudChatConfig(OpenAIGPTConfig): """ supports_function_calling: Optional[bool] = None try: - model_info = _get_model_info_helper( - model, custom_llm_provider="ovhcloud" - ) + model_info = _get_model_info_helper(model, custom_llm_provider="ovhcloud") supports_function_calling = model_info.get( "supports_function_calling", None ) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index f6310778c7..7945c44d44 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -555,7 +555,9 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 } } if "thought_signature" in invocation: - tc_part["thoughtSignature"] = invocation["thought_signature"] + tc_part["thoughtSignature"] = invocation[ + "thought_signature" + ] assistant_content.append(tc_part) # type: ignore # Re-inject toolResponse part if response is present @@ -566,11 +568,11 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 } if invocation.get("tool_type"): tr_dict["toolType"] = invocation["tool_type"] - tr_part: Dict[str, Any] = { - "toolResponse": tr_dict - } + tr_part: Dict[str, Any] = {"toolResponse": tr_dict} if "thought_signature" in invocation: - tr_part["thoughtSignature"] = invocation["thought_signature"] + tr_part["thoughtSignature"] = invocation[ + "thought_signature" + ] assistant_content.append(tr_part) # type: ignore msg_i += 1 diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 1cae1e512f..36f51c5b2f 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -12,6 +12,7 @@ from typing import ( Dict, List, Literal, + Mapping, Optional, Tuple, Type, @@ -1696,6 +1697,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): response_tokens: Optional[int] = None response_tokens_details: Optional[CompletionTokensDetailsWrapper] = None usage_metadata = completion_response["usageMetadata"] + + def _get_token_count(detail: Mapping[str, Any]) -> int: + raw_token_count = detail.get("tokenCount", detail.get("token_count", 0)) + return raw_token_count if isinstance(raw_token_count, int) else 0 + if "cachedContentTokenCount" in usage_metadata: cached_tokens = usage_metadata["cachedContentTokenCount"] @@ -1705,10 +1711,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if "responseTokensDetails" in usage_metadata: response_tokens_details = CompletionTokensDetailsWrapper() for detail in usage_metadata["responseTokensDetails"]: - if detail["modality"] == "TEXT": - response_tokens_details.text_tokens = detail.get("tokenCount", 0) - elif detail["modality"] == "AUDIO": - response_tokens_details.audio_tokens = detail.get("tokenCount", 0) + modality = str(detail.get("modality", "")).upper() + token_count = _get_token_count(detail) + if modality == "TEXT": + response_tokens_details.text_tokens = ( + response_tokens_details.text_tokens or 0 + ) + token_count + elif modality == "AUDIO": + response_tokens_details.audio_tokens = ( + response_tokens_details.audio_tokens or 0 + ) + token_count ######################################################### @@ -1717,16 +1729,24 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if response_tokens_details is None: response_tokens_details = CompletionTokensDetailsWrapper() for detail in usage_metadata["candidatesTokensDetails"]: - modality = detail.get("modality") - token_count = detail.get("tokenCount", 0) + modality = str(detail.get("modality", "")).upper() + token_count = _get_token_count(detail) if modality == "TEXT": - response_tokens_details.text_tokens = token_count + response_tokens_details.text_tokens = ( + response_tokens_details.text_tokens or 0 + ) + token_count elif modality == "AUDIO": - response_tokens_details.audio_tokens = token_count + response_tokens_details.audio_tokens = ( + response_tokens_details.audio_tokens or 0 + ) + token_count elif modality == "IMAGE": - response_tokens_details.image_tokens = token_count + response_tokens_details.image_tokens = ( + response_tokens_details.image_tokens or 0 + ) + token_count elif modality == "VIDEO": - response_tokens_details.video_tokens = token_count + response_tokens_details.video_tokens = ( + response_tokens_details.video_tokens or 0 + ) + token_count # Calculate text_tokens if not explicitly provided in candidatesTokensDetails # candidatesTokenCount includes all modalities, so: text = total - (image + audio + video) @@ -1750,14 +1770,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ## Parse promptTokensDetails (total tokens by modality, includes cached + non-cached) if "promptTokensDetails" in usage_metadata: for detail in usage_metadata["promptTokensDetails"]: - if detail["modality"] == "AUDIO": - prompt_audio_tokens = detail.get("tokenCount", 0) - elif detail["modality"] == "TEXT": - prompt_text_tokens = detail.get("tokenCount", 0) - elif detail["modality"] == "IMAGE": - prompt_image_tokens = detail.get("tokenCount", 0) - elif detail["modality"] == "VIDEO": - prompt_video_tokens = detail.get("tokenCount", 0) + modality = str(detail.get("modality", "")).upper() + token_count = _get_token_count(detail) + if modality == "AUDIO": + prompt_audio_tokens = (prompt_audio_tokens or 0) + token_count + elif modality == "TEXT": + prompt_text_tokens = (prompt_text_tokens or 0) + token_count + elif modality == "IMAGE": + prompt_image_tokens = (prompt_image_tokens or 0) + token_count + elif modality == "VIDEO": + prompt_video_tokens = (prompt_video_tokens or 0) + token_count ## Parse cacheTokensDetails (breakdown of cached tokens by modality) ## When explicit caching is used, Gemini provides this field to show which modalities were cached @@ -1768,14 +1790,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if "cacheTokensDetails" in usage_metadata: for detail in usage_metadata["cacheTokensDetails"]: - if detail["modality"] == "AUDIO": - cached_audio_tokens = detail.get("tokenCount", 0) - elif detail["modality"] == "TEXT": - cached_text_tokens = detail.get("tokenCount", 0) - elif detail["modality"] == "IMAGE": - cached_image_tokens = detail.get("tokenCount", 0) - elif detail["modality"] == "VIDEO": - cached_video_tokens = detail.get("tokenCount", 0) + modality = str(detail.get("modality", "")).upper() + token_count = _get_token_count(detail) + if modality == "AUDIO": + cached_audio_tokens = (cached_audio_tokens or 0) + token_count + elif modality == "TEXT": + cached_text_tokens = (cached_text_tokens or 0) + token_count + elif modality == "IMAGE": + cached_image_tokens = (cached_image_tokens or 0) + token_count + elif modality == "VIDEO": + cached_video_tokens = (cached_video_tokens or 0) + token_count ## Calculate non-cached tokens by subtracting cached from total (per modality) ## This is necessary because promptTokensDetails includes both cached and non-cached tokens diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index 82076ff360..5d94cd4212 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -105,11 +105,13 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): # Extract Vertex AI credentials and settings vertex_credentials = self.get_vertex_ai_credentials(litellm_params) vertex_project = self.get_vertex_ai_project(litellm_params) - + # Check for count_tokens specific location override - vertex_count_tokens_location = litellm_params.get("vertex_count_tokens_location") + vertex_count_tokens_location = litellm_params.get( + "vertex_count_tokens_location" + ) vertex_location_raw = self.get_vertex_ai_location(litellm_params) - + # Determine final location with precedence: # 1. vertex_count_tokens_location (if provided) # 2. vertex_location (if provided) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index e2f06abc52..800cca21db 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -920,9 +920,9 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 route=route, ) if _end_user_object is not None: - end_user_params["allowed_model_region"] = ( - _end_user_object.allowed_model_region - ) + end_user_params[ + "allowed_model_region" + ] = _end_user_object.allowed_model_region if _end_user_object.litellm_budget_table is not None: _apply_budget_limits_to_end_user_params( end_user_params=end_user_params, @@ -1499,9 +1499,9 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 if _end_user_object is not None: valid_token_dict.update(end_user_params) - valid_token_dict["end_user_object_permission"] = ( - _end_user_object.object_permission - ) + valid_token_dict[ + "end_user_object_permission" + ] = _end_user_object.object_permission # check if token is from litellm-ui, litellm ui makes keys to allow users to login with sso. These keys can only be used for LiteLLM UI functions # sso/login, ui/login, /key functions and /user functions diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index bad70d30da..8147d17c10 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1004,7 +1004,8 @@ class ProxyBaseLLMRequestProcessing: self.data["deployment"] = llm_router.get_deployment(model_id=model_id) asyncio.create_task( proxy_logging_obj.update_request_status( - litellm_call_id=self.data.get("litellm_call_id", ""), status="success" + litellm_call_id=self.data.get("litellm_call_id", ""), + status="success", ) ) if self._is_streaming_request( @@ -1029,11 +1030,13 @@ class ProxyBaseLLMRequestProcessing: ) # Call response headers hook for streaming success - callback_headers = await proxy_logging_obj.post_call_response_headers_hook( - data=self.data, - user_api_key_dict=user_api_key_dict, - response=response, - request_headers=dict(request.headers), + callback_headers = ( + await proxy_logging_obj.post_call_response_headers_hook( + data=self.data, + user_api_key_dict=user_api_key_dict, + response=response, + request_headers=dict(request.headers), + ) ) if callback_headers: custom_headers.update(callback_headers) @@ -1439,7 +1442,8 @@ class ProxyBaseLLMRequestProcessing: )["guardrail_blocked"] = True except Exception as e: verbose_proxy_logger.exception( - "Error in deferred streaming guardrail initialization: %s", e, + "Error in deferred streaming guardrail initialization: %s", + e, ) finally: try: @@ -1453,7 +1457,8 @@ class ProxyBaseLLMRequestProcessing: ) except Exception as e: verbose_proxy_logger.exception( - "Error in deferred streaming async logging: %s", e, + "Error in deferred streaming async logging: %s", + e, ) try: @@ -1466,7 +1471,8 @@ class ProxyBaseLLMRequestProcessing: ) except Exception as e: verbose_proxy_logger.exception( - "Error in deferred streaming sync logging: %s", e, + "Error in deferred streaming sync logging: %s", + e, ) async def _handle_llm_api_exception( diff --git a/litellm/proxy/guardrails/guardrail_hooks/akto/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/akto/__init__.py index 4ae2675540..c4aaea709b 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/akto/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/akto/__init__.py @@ -17,7 +17,9 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" akto_api_key=getattr(litellm_params, "akto_api_key", None), akto_account_id=getattr(litellm_params, "akto_account_id", None), akto_vxlan_id=getattr(litellm_params, "akto_vxlan_id", None), - unreachable_fallback=getattr(litellm_params, "unreachable_fallback", "fail_closed"), + unreachable_fallback=getattr( + litellm_params, "unreachable_fallback", "fail_closed" + ), guardrail_timeout=getattr(litellm_params, "guardrail_timeout", None), guardrail_name=guardrail.get("guardrail_name", ""), event_hook=litellm_params.mode, diff --git a/litellm/proxy/guardrails/guardrail_hooks/akto/akto.py b/litellm/proxy/guardrails/guardrail_hooks/akto/akto.py index be9c9cb1be..5058ee348d 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/akto/akto.py +++ b/litellm/proxy/guardrails/guardrail_hooks/akto/akto.py @@ -77,17 +77,27 @@ class AktoGuardrail(CustomGuardrail): ) self.background_tasks: set = set() - self.akto_base_url = (akto_base_url or os.environ.get("AKTO_GUARDRAIL_API_BASE", "")).rstrip("/") + self.akto_base_url = ( + akto_base_url or os.environ.get("AKTO_GUARDRAIL_API_BASE", "") + ).rstrip("/") if not self.akto_base_url: - raise ValueError("akto_base_url is required. Set AKTO_GUARDRAIL_API_BASE or pass it in litellm_params.") + raise ValueError( + "akto_base_url is required. Set AKTO_GUARDRAIL_API_BASE or pass it in litellm_params." + ) self.akto_api_key = akto_api_key or os.environ.get("AKTO_API_KEY", "") if not self.akto_api_key: - raise ValueError("akto_api_key is required. Set AKTO_API_KEY or pass it in litellm_params.") + raise ValueError( + "akto_api_key is required. Set AKTO_API_KEY or pass it in litellm_params." + ) - self.unreachable_fallback: Literal["fail_closed", "fail_open"] = unreachable_fallback + self.unreachable_fallback: Literal[ + "fail_closed", "fail_open" + ] = unreachable_fallback self.guardrail_timeout = guardrail_timeout or DEFAULT_GUARDRAIL_TIMEOUT - self.akto_account_id = akto_account_id or os.environ.get("AKTO_ACCOUNT_ID", "1000000") + self.akto_account_id = akto_account_id or os.environ.get( + "AKTO_ACCOUNT_ID", "1000000" + ) self.akto_vxlan_id = akto_vxlan_id or os.environ.get("AKTO_VXLAN_ID", "0") kwargs["supported_event_hooks"] = [ @@ -173,7 +183,9 @@ class AktoGuardrail(CustomGuardrail): body["model"] = request_data["model"] else: texts = inputs.get("texts", []) - body["messages"] = [{"role": "user", "content": t} for t in texts] if texts else [] + body["messages"] = ( + [{"role": "user", "content": t} for t in texts] if texts else [] + ) tools = inputs.get("tools") if tools: @@ -199,15 +211,23 @@ class AktoGuardrail(CustomGuardrail): texts = inputs.get("texts", []) if texts: - return {"choices": [{"message": {"content": t, "role": "assistant"}} for t in texts]} + return { + "choices": [ + {"message": {"content": t, "role": "assistant"}} for t in texts + ] + } return {} @staticmethod def build_tag_metadata(request_data: dict) -> Dict[str, str]: """Build tag/metadata dict with user_id and team_id for Akto tracking.""" tag: Dict[str, str] = {"gen-ai": "Gen AI"} - user_id = AktoGuardrail.resolve_metadata_value(request_data, "user_api_key_user_id") - team_id = AktoGuardrail.resolve_metadata_value(request_data, "user_api_key_team_id") + user_id = AktoGuardrail.resolve_metadata_value( + request_data, "user_api_key_user_id" + ) + team_id = AktoGuardrail.resolve_metadata_value( + request_data, "user_api_key_team_id" + ) if user_id: tag["user_id"] = user_id if team_id: @@ -236,15 +256,23 @@ class AktoGuardrail(CustomGuardrail): response_headers: Dict[str, str] = {} if include_response: response_body = self.build_response_body(inputs, request_data) - response_payload = json.dumps({"body": json.dumps(response_body)}) # Double-encoded + response_payload = json.dumps( + {"body": json.dumps(response_body)} + ) # Double-encoded response_headers = {"content-type": "application/json"} # Extract client IP from proxy headers ip = "" proxy_req = request_data.get("proxy_server_request", {}) - proxy_headers = proxy_req.get("headers", {}) if isinstance(proxy_req, dict) else {} + proxy_headers = ( + proxy_req.get("headers", {}) if isinstance(proxy_req, dict) else {} + ) if isinstance(proxy_headers, dict): - ip = proxy_headers.get("x-forwarded-for") or proxy_headers.get("x-real-ip") or "" + ip = ( + proxy_headers.get("x-forwarded-for") + or proxy_headers.get("x-real-ip") + or "" + ) if "," in ip: ip = ip.split(",")[0].strip() @@ -253,7 +281,9 @@ class AktoGuardrail(CustomGuardrail): "requestHeaders": json.dumps(request_headers), "responseHeaders": json.dumps(response_headers), "method": "POST", - "requestPayload": json.dumps({"body": json.dumps(request_body)}), # Double-encoded + "requestPayload": json.dumps( + {"body": json.dumps(request_body)} + ), # Double-encoded "responsePayload": response_payload, "ip": ip, "destIp": "127.0.0.1", @@ -393,7 +423,9 @@ class AktoGuardrail(CustomGuardrail): if input_type == "request": # Pre_call: awaited guardrail check (no ingestion) - payload = self.build_akto_payload(inputs, request_data, include_response=False) + payload = self.build_akto_payload( + inputs, request_data, include_response=False + ) try: response = await self.send_request( guardrails=True, @@ -419,7 +451,9 @@ class AktoGuardrail(CustomGuardrail): ) blocked_payload["responsePayload"] = json.dumps( { - "body": json.dumps({"x-blocked-by": "Akto Proxy", "reason": reason}), + "body": json.dumps( + {"x-blocked-by": "Akto Proxy", "reason": reason} + ), } ) blocked_payload["responseHeaders"] = json.dumps( @@ -442,7 +476,9 @@ class AktoGuardrail(CustomGuardrail): elif input_type == "response": # Post_call: fire-and-forget combined guardrail + ingest - payload = self.build_akto_payload(inputs, request_data, include_response=True) + payload = self.build_akto_payload( + inputs, request_data, include_response=True + ) task = asyncio.create_task( self.fire_and_forget_request( guardrails=True, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ecf25d73cb..5aa0b9ea4a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -639,9 +639,9 @@ except ImportError: server_root_path = get_server_root_path() _license_check = LicenseCheck() premium_user: bool = _license_check.is_premium() -premium_user_data: Optional["EnterpriseLicenseData"] = ( - _license_check.airgapped_license_data -) +premium_user_data: Optional[ + "EnterpriseLicenseData" +] = _license_check.airgapped_license_data global_max_parallel_request_retries_env: Optional[str] = os.getenv( "LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES" ) @@ -1524,9 +1524,9 @@ master_key: Optional[str] = None config_agents: Optional[List[AgentConfig]] = None otel_logging = False prisma_client: Optional[PrismaClient] = None -shared_aiohttp_session: Optional["ClientSession"] = ( - None # Global shared session for connection reuse -) +shared_aiohttp_session: Optional[ + "ClientSession" +] = None # Global shared session for connection reuse user_api_key_cache = DualCache( default_in_memory_ttl=UserAPIKeyCacheTTLEnum.in_memory_cache_ttl.value ) @@ -1534,13 +1534,13 @@ model_max_budget_limiter = _PROXY_VirtualKeyModelMaxBudgetLimiter( dual_cache=user_api_key_cache ) litellm.logging_callback_manager.add_litellm_callback(model_max_budget_limiter) -redis_usage_cache: Optional[RedisCache] = ( - None # redis cache used for tracking spend, tpm/rpm limits -) +redis_usage_cache: Optional[ + RedisCache +] = None # redis cache used for tracking spend, tpm/rpm limits polling_via_cache_enabled: Union[Literal["all"], List[str], bool] = False -native_background_mode: List[str] = ( - [] -) # Models that should use native provider background mode instead of polling +native_background_mode: List[ + str +] = [] # Models that should use native provider background mode instead of polling polling_cache_ttl: int = 3600 # Default 1 hour TTL for polling cache user_custom_auth = None user_custom_key_generate = None @@ -1900,9 +1900,9 @@ async def update_cache( # noqa: PLR0915 _id = "team_id:{}".format(team_id) try: # Fetch the existing cost for the given user - existing_spend_obj: Optional[LiteLLM_TeamTable] = ( - await user_api_key_cache.async_get_cache(key=_id) - ) + existing_spend_obj: Optional[ + LiteLLM_TeamTable + ] = await user_api_key_cache.async_get_cache(key=_id) if existing_spend_obj is None: # do nothing if team not in api key cache return @@ -2023,9 +2023,11 @@ def run_ollama_serve(): with open(os.devnull, "w") as devnull: subprocess.Popen(command, stdout=devnull, stderr=devnull) except Exception as e: - verbose_proxy_logger.debug(f""" + verbose_proxy_logger.debug( + f""" LiteLLM Warning: proxy started with `ollama` model\n`ollama serve` failed with Exception{e}. \nEnsure you run `ollama serve` - """) + """ + ) def _get_process_rss_mb() -> Optional[float]: @@ -4976,10 +4978,10 @@ class ProxyConfig: ) try: - guardrails_in_db: List[Guardrail] = ( - await GuardrailRegistry.get_all_guardrails_from_db( - prisma_client=prisma_client - ) + guardrails_in_db: List[ + Guardrail + ] = await GuardrailRegistry.get_all_guardrails_from_db( + prisma_client=prisma_client ) verbose_proxy_logger.debug( "guardrails from the DB %s", str(guardrails_in_db) @@ -5361,9 +5363,9 @@ async def initialize( # noqa: PLR0915 user_api_base = api_base dynamic_config[user_model]["api_base"] = api_base if api_version: - os.environ["AZURE_API_VERSION"] = ( - api_version # set this for azure - litellm can read this from the env - ) + os.environ[ + "AZURE_API_VERSION" + ] = api_version # set this for azure - litellm can read this from the env if max_tokens: # model-specific param dynamic_config[user_model]["max_tokens"] = max_tokens if temperature: # model-specific param @@ -5700,9 +5702,9 @@ class ProxyStartupEvent: """ from litellm.secret_managers.main import str_to_bool - _use_redis_transaction_buffer: Optional[Union[bool, str]] = ( - general_settings.get("use_redis_transaction_buffer", False) - ) + _use_redis_transaction_buffer: Optional[ + Union[bool, str] + ] = general_settings.get("use_redis_transaction_buffer", False) if isinstance(_use_redis_transaction_buffer, str): _use_redis_transaction_buffer = str_to_bool(_use_redis_transaction_buffer) @@ -12297,9 +12299,9 @@ async def get_config_list( hasattr(sub_field_info, "description") and sub_field_info.description is not None ): - nested_fields[idx].field_description = ( - sub_field_info.description - ) + nested_fields[ + idx + ].field_description = sub_field_info.description idx += 1 _stored_in_db = None diff --git a/tests/llm_translation/test_gemini_image_usage.py b/tests/llm_translation/test_gemini_image_usage.py index 8c7f05d38e..0497d7fd9d 100644 --- a/tests/llm_translation/test_gemini_image_usage.py +++ b/tests/llm_translation/test_gemini_image_usage.py @@ -4,9 +4,11 @@ Test for Gemini image generation usage metadata extraction. This test verifies the fix for issue #18323 where image_generation() was returning usage=0 while completion() returned proper token usage. """ +import os import pytest from unittest.mock import patch, MagicMock import litellm +from litellm.llms.gemini.image_generation.transformation import GoogleImageGenConfig from litellm.types.utils import ImageResponse, ImageObject, ImageUsage @@ -211,3 +213,56 @@ def test_gemini_imagen_models_no_usage_extraction(): # For Imagen models, we don't extract usage from the predictions format # This test just ensures we don't crash + + +def test_gemini_image_generation_accumulates_multiple_image_prompt_token_details(): + """ + Regression test: promptTokensDetails can include multiple IMAGE entries. + These must be accumulated instead of overwritten. + """ + previous_local_model_cost_map = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") + previous_model_cost = litellm.model_cost + try: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "gemini/gemini-3-pro-image-preview" + config = GoogleImageGenConfig() + + usage_metadata = { + "promptTokenCount": 200, + "candidatesTokenCount": 0, + "totalTokenCount": 200, + "promptTokensDetails": [ + {"modality": "TEXT", "tokenCount": 10}, + {"modality": "IMAGE", "tokenCount": 90}, + {"modality": "IMAGE", "tokenCount": 100}, + ], + } + + parsed_usage = config._transform_image_usage(usage_metadata) + image_response = ImageResponse( + data=[ImageObject(b64_json="fake_image_data")], + usage=parsed_usage, + ) + + observed_cost = litellm.completion_cost( + completion_response=image_response, + model=model, + custom_llm_provider="gemini", + ) + + model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") + expected_image_tokens = 190 + expected_total_prompt_tokens = 200 + expected_prompt_cost = expected_total_prompt_tokens * model_info["input_cost_per_token"] + + assert parsed_usage.input_tokens_details.image_tokens == expected_image_tokens + assert parsed_usage.input_tokens_details.text_tokens == 10 + assert observed_cost == pytest.approx(expected_prompt_cost, rel=1e-12) + finally: + if previous_local_model_cost_map is None: + os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) + else: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = previous_local_model_cost_map + litellm.model_cost = previous_model_cost diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 636e84fe79..33628e1d19 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -31,7 +31,7 @@ def test_anthropic_experimental_pass_through_messages_handler(): model="openai/claude-3-5-sonnet-20240620", api_key="test-api-key", ) - except Exception as e: + except (ValueError, TypeError, AttributeError) as e: print(f"Error: {e}") mock_responses.assert_called_once() assert mock_responses.call_args.kwargs["api_key"] == "test-api-key" @@ -56,7 +56,7 @@ def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_an api_base="test-api-base", custom_key="custom_value", ) - except Exception as e: + except (ValueError, TypeError, AttributeError) as e: print(f"Error: {e}") mock_completion.assert_called_once() assert mock_completion.call_args.kwargs["api_key"] == "test-api-key" @@ -81,7 +81,7 @@ def test_anthropic_experimental_pass_through_messages_handler_custom_llm_provide custom_llm_provider="my-custom-llm", api_key="test-api-key", ) - except Exception as e: + except (ValueError, TypeError, AttributeError) as e: print(f"Error: {e}") # Assert that litellm.completion was called when using a custom LLM provider @@ -125,24 +125,29 @@ async def test_bedrock_converse_budget_tokens_preserved(): max_tokens=1024, messages=[{"role": "user", "content": "What is 2+2?"}], model="bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0", - thinking={ - "budget_tokens": 1024, - "type": "enabled" - }, + thinking={"budget_tokens": 1024, "type": "enabled"}, ) - except Exception: + except (ValueError, TypeError, AttributeError): pass # Expected due to response format conversion mock_acompletion.assert_called_once() call_kwargs = mock_acompletion.call_args.kwargs - print("acompletion call kwargs: ", json.dumps(call_kwargs, indent=4, default=str)) + print( + "acompletion call kwargs: ", json.dumps(call_kwargs, indent=4, default=str) + ) # Verify thinking parameter is passed through with budget_tokens preserved thinking_param = call_kwargs.get("thinking") - assert thinking_param is not None, "thinking parameter should be passed to acompletion" - assert thinking_param.get("type") == "enabled", "thinking.type should be 'enabled'" - assert thinking_param.get("budget_tokens") == 1024, f"thinking.budget_tokens should be 1024, but got {thinking_param.get('budget_tokens')}" + assert ( + thinking_param is not None + ), "thinking parameter should be passed to acompletion" + assert ( + thinking_param.get("type") == "enabled" + ), "thinking.type should be 'enabled'" + assert ( + thinking_param.get("budget_tokens") == 1024 + ), f"thinking.budget_tokens should be 1024, but got {thinking_param.get('budget_tokens')}" def test_openai_model_with_thinking_converts_to_reasoning(): @@ -164,12 +169,9 @@ def test_openai_model_with_thinking_converts_to_reasoning(): messages=[{"role": "user", "content": "What is 2+2?"}], model="openai/gpt-5.2", api_key="test-api-key", - thinking={ - "type": "enabled", - "budget_tokens": 1024 - }, + thinking={"type": "enabled", "budget_tokens": 1024}, ) - except Exception as e: + except (ValueError, TypeError, AttributeError) as e: print(f"Error: {e}") mock_responses.assert_called_once() @@ -177,17 +179,23 @@ def test_openai_model_with_thinking_converts_to_reasoning(): call_kwargs = mock_responses.call_args.kwargs # Verify reasoning is set (converted from thinking) - assert "reasoning" in call_kwargs, "reasoning should be passed to litellm.responses" + assert ( + "reasoning" in call_kwargs + ), "reasoning should be passed to litellm.responses" # budget_tokens=1024 -> effort="minimal" (< 2000 threshold) - expected_reasoning = {"effort": "minimal", "summary": "detailed"} + # reasoning_auto_summary is False by default, so no summary key + expected_reasoning = {"effort": "minimal"} assert call_kwargs["reasoning"] == expected_reasoning, ( f"reasoning should be {expected_reasoning} for budget_tokens=1024, " f"got {call_kwargs.get('reasoning')}" ) + assert "summary" not in call_kwargs["reasoning"] # Verify thinking is NOT passed directly to the Responses API - assert "thinking" not in call_kwargs, "thinking should NOT be passed directly to litellm.responses" + assert ( + "thinking" not in call_kwargs + ), "thinking should NOT be passed directly to litellm.responses" class TestThinkingParameterTransformation: @@ -198,13 +206,13 @@ class TestThinkingParameterTransformation: from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) - + thinking = {"type": "enabled", "budget_tokens": 5000} result = LiteLLMAnthropicMessagesAdapter.translate_thinking_for_model( thinking=thinking, model="bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0", ) - + assert result == {"thinking": thinking} assert result["thinking"]["budget_tokens"] == 5000 @@ -213,12 +221,281 @@ class TestThinkingParameterTransformation: from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) - + thinking = {"type": "enabled", "budget_tokens": 1024} result = LiteLLMAnthropicMessagesAdapter.translate_thinking_for_model( thinking=thinking, model="openai/gpt-5.2", ) - + + # reasoning_auto_summary is False by default, so no summary key assert result == {"reasoning_effort": "minimal"} assert "thinking" not in result + assert "summary" not in str(result["reasoning_effort"]) + + def test_translate_thinking_for_model_summary_when_enabled(self): + """When reasoning_auto_summary is True, summary='detailed' is injected.""" + import litellm + from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( + LiteLLMAnthropicMessagesAdapter, + ) + + original = litellm.reasoning_auto_summary + try: + litellm.reasoning_auto_summary = True + thinking = {"type": "enabled", "budget_tokens": 5000} + result = LiteLLMAnthropicMessagesAdapter.translate_thinking_for_model( + thinking=thinking, + model="openai/gpt-5.2", + ) + assert result == { + "reasoning_effort": {"effort": "medium", "summary": "detailed"} + } + finally: + litellm.reasoning_auto_summary = original + + def test_translate_thinking_for_model_preserves_user_summary(self): + """User-provided summary is always preserved regardless of flag.""" + from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( + LiteLLMAnthropicMessagesAdapter, + ) + + thinking = {"type": "enabled", "budget_tokens": 10000, "summary": "concise"} + result = LiteLLMAnthropicMessagesAdapter.translate_thinking_for_model( + thinking=thinking, + model="openai/gpt-5.2", + ) + assert result == {"reasoning_effort": {"effort": "high", "summary": "concise"}} + + +class TestThinkingSummaryPreservation: + """Tests for thinking.summary preservation and reasoning_auto_summary flag.""" + + def test_thinking_summary_concise_preserved_for_openai(self): + """User-provided summary='concise' should not be replaced with 'detailed'.""" + from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, + ) + + thinking = {"type": "enabled", "budget_tokens": 5000, "summary": "concise"} + completion_kwargs = {"model": "openai/gpt-5.1", "reasoning_effort": "medium"} + LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( + completion_kwargs, thinking=thinking + ) + assert completion_kwargs["reasoning_effort"] == { + "effort": "medium", + "summary": "concise", + } + + def test_thinking_summary_auto_preserved_for_openai(self): + """User-provided summary='auto' should be preserved.""" + from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, + ) + + thinking = {"type": "enabled", "budget_tokens": 10000, "summary": "auto"} + completion_kwargs = {"model": "openai/gpt-5.1", "reasoning_effort": "high"} + LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( + completion_kwargs, thinking=thinking + ) + assert completion_kwargs["reasoning_effort"] == { + "effort": "high", + "summary": "auto", + } + + def test_summary_added_when_auto_summary_enabled(self): + """When reasoning_auto_summary is True, summary='detailed' is added.""" + import litellm + from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, + ) + + original = litellm.reasoning_auto_summary + try: + litellm.reasoning_auto_summary = True + completion_kwargs = { + "model": "responses/gpt-5.2", + "custom_llm_provider": "openai", + "reasoning_effort": "medium", + } + LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( + completion_kwargs, thinking={"type": "enabled", "budget_tokens": 5000} + ) + assert completion_kwargs["reasoning_effort"] == { + "effort": "medium", + "summary": "detailed", + } + finally: + litellm.reasoning_auto_summary = original + + def test_no_summary_by_default_string_reasoning(self): + """By default (reasoning_auto_summary=False), summary is not added for string reasoning_effort.""" + import litellm + from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, + ) + + original = litellm.reasoning_auto_summary + try: + litellm.reasoning_auto_summary = False + completion_kwargs = { + "model": "responses/gpt-5.2", + "custom_llm_provider": "openai", + "reasoning_effort": "high", + } + LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( + completion_kwargs, thinking={"type": "enabled", "budget_tokens": 10000} + ) + assert completion_kwargs["reasoning_effort"] == {"effort": "high"} + assert "summary" not in completion_kwargs["reasoning_effort"] + finally: + litellm.reasoning_auto_summary = original + + def test_no_summary_by_default_dict_reasoning(self): + """By default (reasoning_auto_summary=False), summary is not injected into dict reasoning_effort.""" + import litellm + from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, + ) + + original = litellm.reasoning_auto_summary + try: + litellm.reasoning_auto_summary = False + completion_kwargs = { + "model": "responses/gpt-5.2", + "custom_llm_provider": "openai", + "reasoning_effort": {"effort": "medium"}, + } + LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( + completion_kwargs, thinking={"type": "enabled", "budget_tokens": 5000} + ) + assert completion_kwargs["reasoning_effort"] == {"effort": "medium"} + assert "summary" not in completion_kwargs["reasoning_effort"] + finally: + litellm.reasoning_auto_summary = original + + def test_summary_added_when_env_var_set(self): + """When LITELLM_REASONING_AUTO_SUMMARY env var is true, summary is added.""" + import litellm + from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, + ) + + original = litellm.reasoning_auto_summary + try: + litellm.reasoning_auto_summary = False + os.environ["LITELLM_REASONING_AUTO_SUMMARY"] = "true" + completion_kwargs = { + "model": "responses/gpt-5.2", + "custom_llm_provider": "openai", + "reasoning_effort": "high", + } + LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( + completion_kwargs, thinking={"type": "enabled", "budget_tokens": 10000} + ) + assert completion_kwargs["reasoning_effort"] == { + "effort": "high", + "summary": "detailed", + } + finally: + litellm.reasoning_auto_summary = original + os.environ.pop("LITELLM_REASONING_AUTO_SUMMARY", None) + + def test_user_provided_summary_preserved_even_when_flag_off(self): + """When user already set summary in dict reasoning_effort, it's preserved regardless of flag.""" + import litellm + from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, + ) + + original = litellm.reasoning_auto_summary + try: + litellm.reasoning_auto_summary = False + completion_kwargs = { + "model": "responses/gpt-5.2", + "custom_llm_provider": "openai", + "reasoning_effort": {"effort": "high", "summary": "concise"}, + } + LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( + completion_kwargs, thinking={"type": "enabled", "budget_tokens": 10000} + ) + assert completion_kwargs["reasoning_effort"]["summary"] == "concise" + finally: + litellm.reasoning_auto_summary = original + + def test_openai_model_with_thinking_summary_end_to_end(self): + """End-to-end: anthropic_messages_handler should preserve thinking.summary for OpenAI models.""" + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + anthropic_messages_handler, + ) + + with patch("litellm.responses", return_value="test-response") as mock_responses: + try: + anthropic_messages_handler( + max_tokens=1024, + messages=[{"role": "user", "content": "What is 2+2?"}], + model="openai/gpt-5.2", + api_key="test-api-key", + thinking={ + "type": "enabled", + "budget_tokens": 5000, + "summary": "concise", + }, + ) + except (ValueError, TypeError, AttributeError): + pass + + mock_responses.assert_called_once() + call_kwargs = mock_responses.call_args.kwargs + reasoning = call_kwargs["reasoning"] + assert ( + reasoning["summary"] == "concise" + ), f"Expected summary='concise', got summary='{reasoning.get('summary')}'" + + def test_responses_adapter_preserves_summary(self): + """translate_thinking_to_reasoning should include summary when user provides it.""" + from litellm.llms.anthropic.experimental_pass_through.responses_adapters.transformation import ( + LiteLLMAnthropicToResponsesAPIAdapter, + ) + + thinking = {"type": "enabled", "budget_tokens": 5000, "summary": "concise"} + result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning( + thinking + ) + assert result == {"effort": "medium", "summary": "concise"} + + def test_responses_adapter_no_summary_by_default(self): + """translate_thinking_to_reasoning should not include summary by default (opt-in).""" + import litellm + from litellm.llms.anthropic.experimental_pass_through.responses_adapters.transformation import ( + LiteLLMAnthropicToResponsesAPIAdapter, + ) + + original = litellm.reasoning_auto_summary + try: + litellm.reasoning_auto_summary = False + thinking = {"type": "enabled", "budget_tokens": 5000} + result = ( + LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning( + thinking + ) + ) + assert result == {"effort": "medium"} + assert result is not None and "summary" not in result + finally: + litellm.reasoning_auto_summary = original + + def test_translate_thinking_for_model_preserves_summary(self): + """translate_thinking_for_model should include summary in reasoning_effort dict when user provides it.""" + from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( + LiteLLMAnthropicMessagesAdapter, + ) + + thinking = {"type": "enabled", "budget_tokens": 5000, "summary": "concise"} + result = LiteLLMAnthropicMessagesAdapter.translate_thinking_for_model( + thinking=thinking, + model="openai/gpt-5.2", + ) + assert result == { + "reasoning_effort": {"effort": "medium", "summary": "concise"} + } diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 252ba230ff..02b817cd33 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -170,6 +170,7 @@ class TestOutputConfigStructuredOutput: # translate_messages_to_responses_input # --------------------------------------------------------------------------- + # Helper: cast plain dicts to the expected type so call sites stay clean. def _translate_messages(messages: List[Any]) -> List[Dict[str, Any]]: return _ADAPTER.translate_messages_to_responses_input(messages) # type: ignore[arg-type] @@ -274,7 +275,11 @@ class TestTranslateMessagesToResponsesInput: "content": [ { "type": "image", - "source": {"type": "base64", "media_type": "image/jpeg", "data": ""}, + "source": { + "type": "base64", + "media_type": "image/jpeg", + "data": "", + }, } ], } @@ -462,7 +467,10 @@ class TestTranslateMessagesToResponsesInput: ] result = _translate_messages(messages) assert len(result) == 1 - assert result[0]["content"][0] == {"type": "input_text", "text": "Describe this image:"} + assert result[0]["content"][0] == { + "type": "input_text", + "text": "Describe this image:", + } assert result[0]["content"][1] == { "type": "input_image", "image_url": "https://example.com/cat.jpg", @@ -606,7 +614,9 @@ class TestTranslateThinkingToReasoning: result = _ADAPTER.translate_thinking_to_reasoning( {"type": "enabled", "budget_tokens": 10000} ) - assert result == {"effort": "high", "summary": "detailed"} + # Default (reasoning_auto_summary=False): only effort, no summary + assert result == {"effort": "high"} + assert result is not None and "summary" not in result def test_budget_above_threshold_high_effort(self): result = _ADAPTER.translate_thinking_to_reasoning( @@ -614,24 +624,28 @@ class TestTranslateThinkingToReasoning: ) assert result is not None assert result["effort"] == "high" + assert "summary" not in result def test_budget_medium_effort(self): result = _ADAPTER.translate_thinking_to_reasoning( {"type": "enabled", "budget_tokens": 7500} ) - assert result == {"effort": "medium", "summary": "detailed"} + assert result == {"effort": "medium"} + assert result is not None and "summary" not in result def test_budget_low_effort(self): result = _ADAPTER.translate_thinking_to_reasoning( {"type": "enabled", "budget_tokens": 3000} ) - assert result == {"effort": "low", "summary": "detailed"} + assert result == {"effort": "low"} + assert result is not None and "summary" not in result def test_budget_minimal_effort(self): result = _ADAPTER.translate_thinking_to_reasoning( {"type": "enabled", "budget_tokens": 500} ) - assert result == {"effort": "minimal", "summary": "detailed"} + assert result == {"effort": "minimal"} + assert result is not None and "summary" not in result def test_budget_at_exact_thresholds(self): result_medium = _ADAPTER.translate_thinking_to_reasoning( @@ -639,11 +653,13 @@ class TestTranslateThinkingToReasoning: ) assert result_medium is not None assert result_medium["effort"] == "medium" + assert "summary" not in result_medium result_low = _ADAPTER.translate_thinking_to_reasoning( {"type": "enabled", "budget_tokens": 2000} ) assert result_low is not None assert result_low["effort"] == "low" + assert "summary" not in result_low def test_disabled_type_returns_none(self): result = _ADAPTER.translate_thinking_to_reasoning({"type": "disabled"}) @@ -656,7 +672,38 @@ class TestTranslateThinkingToReasoning: def test_missing_budget_defaults_to_minimal(self): """Missing budget_tokens defaults to 0, which is < 2000 -> minimal.""" result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled"}) - assert result == {"effort": "minimal", "summary": "detailed"} + assert result == {"effort": "minimal"} + assert result is not None and "summary" not in result + + def test_summary_added_when_auto_summary_enabled(self): + """When reasoning_auto_summary is True, summary='detailed' is included.""" + import litellm + + original = litellm.reasoning_auto_summary + try: + litellm.reasoning_auto_summary = True + result = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 10000} + ) + assert result == {"effort": "high", "summary": "detailed"} + finally: + litellm.reasoning_auto_summary = original + + def test_summary_added_when_env_var_set(self): + """When LITELLM_REASONING_AUTO_SUMMARY env var is true, summary is included.""" + import litellm + + original = litellm.reasoning_auto_summary + try: + litellm.reasoning_auto_summary = False + os.environ["LITELLM_REASONING_AUTO_SUMMARY"] = "true" + result = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 5000} + ) + assert result == {"effort": "medium", "summary": "detailed"} + finally: + litellm.reasoning_auto_summary = original + os.environ.pop("LITELLM_REASONING_AUTO_SUMMARY", None) # --------------------------------------------------------------------------- @@ -715,7 +762,9 @@ class TestTranslateRequestBroaderCoverage: def test_tools_translated(self): req = _make_request( - tools=[{"name": "calculator", "description": "Does math.", "input_schema": {}}] + tools=[ + {"name": "calculator", "description": "Does math.", "input_schema": {}} + ] ) kwargs = _ADAPTER.translate_request(req) assert len(kwargs["tools"]) == 1 @@ -732,7 +781,9 @@ class TestTranslateRequestBroaderCoverage: def test_thinking_translated_to_reasoning(self): req = _make_request(thinking={"type": "enabled", "budget_tokens": 12000}) kwargs = _ADAPTER.translate_request(req) - assert kwargs["reasoning"] == {"effort": "high", "summary": "detailed"} + # reasoning_auto_summary is False by default, so no summary key + assert kwargs["reasoning"] == {"effort": "high"} + assert "summary" not in kwargs["reasoning"] def test_disabled_thinking_not_included_in_kwargs(self): req = _make_request(thinking={"type": "disabled"}) @@ -753,8 +804,17 @@ class TestTranslateRequestBroaderCoverage: def test_no_optional_fields_does_not_add_spurious_keys(self): req = _make_request() kwargs = _ADAPTER.translate_request(req) - for key in ("instructions", "temperature", "top_p", "tools", "tool_choice", - "reasoning", "text", "context_management", "user"): + for key in ( + "instructions", + "temperature", + "top_p", + "tools", + "tool_choice", + "reasoning", + "text", + "context_management", + "user", + ): assert key not in kwargs, f"unexpected key: {key}" @@ -801,9 +861,7 @@ def _make_output_message(texts: List[str]) -> MagicMock: return msg -def _make_function_call_item( - call_id: str, name: str, arguments: str -) -> MagicMock: +def _make_function_call_item(call_id: str, name: str, arguments: str) -> MagicMock: """Build a mock ResponseFunctionToolCall.""" from openai.types.responses import ResponseFunctionToolCall # type: ignore[import] diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 965fc03a33..3102a69596 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -860,6 +860,42 @@ def test_vertex_ai_usage_metadata_with_image_tokens_in_prompt(): ) +def test_vertex_ai_usage_metadata_accumulates_duplicate_modalities(): + """Ensure _calculate_usage accumulates repeated modality entries.""" + v = VertexGeminiConfig() + usage_metadata = { + "promptTokenCount": 210, + "candidatesTokenCount": 50, + "totalTokenCount": 260, + "promptTokensDetails": [ + {"modality": "TEXT", "tokenCount": 20}, + {"modality": "IMAGE", "tokenCount": 90}, + {"modality": "IMAGE", "token_count": 100}, + ], + "candidatesTokensDetails": [ + {"modality": "IMAGE", "tokenCount": 30}, + {"modality": "TEXT", "tokenCount": 15}, + {"modality": "TEXT", "token_count": 5}, + ], + "cacheTokensDetails": [ + {"modality": "TEXT", "tokenCount": 4}, + {"modality": "IMAGE", "tokenCount": 40}, + {"modality": "IMAGE", "token_count": 10}, + ], + } + usage_metadata = UsageMetadata(**usage_metadata) + result = v._calculate_usage(completion_response={"usageMetadata": usage_metadata}) + + # prompt details are total - cached per modality + assert result.prompt_tokens_details.text_tokens == 16 # 20 - 4 + assert result.prompt_tokens_details.image_tokens == 140 # (90 + 100) - (40 + 10) + + # candidates details accumulate duplicate modalities + assert result.completion_tokens_details.text_tokens == 20 # 15 + 5 + assert result.completion_tokens_details.image_tokens == 30 + assert result.completion_tokens == 50 + + def test_vertex_ai_map_thinking_param_with_budget_tokens_0(): """ If budget_tokens is 0, do not set includeThoughts to True