From f6e3baafc519a38eec8eb5079b2d8b684a4351b9 Mon Sep 17 00:00:00 2001 From: Chesars Date: Tue, 17 Feb 2026 22:52:23 -0300 Subject: [PATCH 01/15] fix(anthropic): preserve thinking.summary when routing to OpenAI Responses API Read summary from the original thinking dict instead of hardcoding "detailed" in _route_openai_thinking_to_responses_api_if_needed(). This preserves the user's chosen summary value (e.g. "concise", "auto") for non-Claude models routed through the Anthropic Messages adapter to OpenAI's Responses API. Fixes #20998 --- .../adapters/handler.py | 3 +- ...erimental_pass_through_messages_handler.py | 75 ++++++++++++++++++- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index 73e74c228b..c0b77798f2 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -78,9 +78,10 @@ class LiteLLMMessagesToCompletionTransformationHandler: reasoning_effort = completion_kwargs.get("reasoning_effort") if isinstance(reasoning_effort, str) and reasoning_effort: + summary = thinking.get("summary", "detailed") if isinstance(thinking, dict) else "detailed" completion_kwargs["reasoning_effort"] = { "effort": reasoning_effort, - "summary": "detailed", + "summary": summary, } elif isinstance(reasoning_effort, dict): if ( diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 376d14416a..e2639a3126 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -209,12 +209,83 @@ class TestThinkingParameterTransformation: from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) - + thinking = {"type": "enabled", "budget_tokens": 1024} result = LiteLLMAnthropicMessagesAdapter.translate_thinking_for_model( thinking=thinking, model="openai/gpt-5.2", ) - + assert result == {"reasoning_effort": "minimal"} assert "thinking" not in result + + +class TestThinkingSummaryPreservation: + """Tests for issue #20998: thinking.summary must be preserved when routing to OpenAI Responses API.""" + + def test_thinking_summary_concise_preserved_for_openai(self): + """User-provided summary='concise' should not be replaced with 'detailed'.""" + from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, + ) + + thinking = {"type": "enabled", "budget_tokens": 5000, "summary": "concise"} + completion_kwargs = {"model": "openai/gpt-5.1", "reasoning_effort": "medium"} + LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( + completion_kwargs, thinking=thinking + ) + assert completion_kwargs["reasoning_effort"] == {"effort": "medium", "summary": "concise"} + + def test_thinking_summary_auto_preserved_for_openai(self): + """User-provided summary='auto' should be preserved.""" + from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, + ) + + thinking = {"type": "enabled", "budget_tokens": 10000, "summary": "auto"} + completion_kwargs = {"model": "openai/gpt-5.1", "reasoning_effort": "high"} + LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( + completion_kwargs, thinking=thinking + ) + assert completion_kwargs["reasoning_effort"] == {"effort": "high", "summary": "auto"} + + def test_thinking_without_summary_defaults_to_detailed(self): + """When no summary is provided, default 'detailed' should still be used.""" + from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, + ) + + thinking = {"type": "enabled", "budget_tokens": 5000} + completion_kwargs = {"model": "openai/gpt-5.1", "reasoning_effort": "medium"} + LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( + completion_kwargs, thinking=thinking + ) + assert completion_kwargs["reasoning_effort"] == {"effort": "medium", "summary": "detailed"} + + def test_openai_model_with_thinking_summary_end_to_end(self): + """End-to-end: anthropic_messages_handler should preserve thinking.summary for OpenAI models.""" + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + anthropic_messages_handler, + ) + + with patch("litellm.completion", return_value="test-response") as mock_completion: + try: + anthropic_messages_handler( + max_tokens=1024, + messages=[{"role": "user", "content": "What is 2+2?"}], + model="openai/gpt-5.2", + api_key="test-api-key", + thinking={ + "type": "enabled", + "budget_tokens": 5000, + "summary": "concise", + }, + ) + except Exception: + pass + + mock_completion.assert_called_once() + call_kwargs = mock_completion.call_args.kwargs + reasoning_effort = call_kwargs["reasoning_effort"] + assert reasoning_effort["summary"] == "concise", \ + f"Expected summary='concise', got summary='{reasoning_effort.get('summary')}'" From ece032523498929d7bde4f52368b18ec247a8750 Mon Sep 17 00:00:00 2001 From: Chesars Date: Wed, 4 Mar 2026 10:45:28 -0300 Subject: [PATCH 02/15] fix(anthropic): make thinking.summary opt-in, don't hardcode default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove hardcoded summary="detailed" injection — summary is opt-in per OpenAI spec and increases costs. Users opt-in per-request via LiteLLM extension: thinking={"type": "enabled", "budget_tokens": N, "summary": "concise"}. Also preserve summary in translate_thinking_for_model() which previously dropped it when converting thinking → reasoning_effort for non-Claude models. Fixes #20998 --- .../adapters/handler.py | 20 +++++----- .../adapters/transformation.py | 3 ++ ...erimental_pass_through_messages_handler.py | 37 ++++++++++++++++--- 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index c0b77798f2..01c8f39ee8 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -44,8 +44,9 @@ class LiteLLMMessagesToCompletionTransformationHandler: For OpenAI models, Chat Completions typically does not return reasoning text (only token accounting). To return a thinking-like content block in the - Anthropic response format, we route the request through OpenAI's Responses API - and request a reasoning summary. + Anthropic response format, we route the request through OpenAI's Responses API. + If the user provides a `summary` field in the thinking dict, it is passed + through to the OpenAI reasoning params (opt-in per OpenAI spec). """ custom_llm_provider = completion_kwargs.get("custom_llm_provider") if custom_llm_provider is None: @@ -77,19 +78,20 @@ class LiteLLMMessagesToCompletionTransformationHandler: completion_kwargs["model"] = f"responses/{model}" reasoning_effort = completion_kwargs.get("reasoning_effort") + summary = thinking.get("summary") if isinstance(thinking, dict) else None if isinstance(reasoning_effort, str) and reasoning_effort: - summary = thinking.get("summary", "detailed") if isinstance(thinking, dict) else "detailed" - completion_kwargs["reasoning_effort"] = { - "effort": reasoning_effort, - "summary": summary, - } + reasoning_dict: Dict[str, Any] = {"effort": reasoning_effort} + if summary: + reasoning_dict["summary"] = summary + completion_kwargs["reasoning_effort"] = reasoning_dict elif isinstance(reasoning_effort, dict): if ( - "summary" not in reasoning_effort + summary + and "summary" not in reasoning_effort and "generate_summary" not in reasoning_effort ): updated_reasoning_effort = dict(reasoning_effort) - updated_reasoning_effort["summary"] = "detailed" + updated_reasoning_effort["summary"] = summary completion_kwargs["reasoning_effort"] = updated_reasoning_effort @staticmethod diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index efbac13735..2af5e35112 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -673,6 +673,9 @@ class LiteLLMAnthropicMessagesAdapter: thinking ) if reasoning_effort: + summary = thinking.get("summary") if isinstance(thinking, dict) else None + if summary: + return {"reasoning_effort": {"effort": reasoning_effort, "summary": summary}} return {"reasoning_effort": reasoning_effort} return {} diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index e2639a3126..c6541ccac4 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -177,8 +177,8 @@ def test_openai_model_with_thinking_converts_to_reasoning_effort(): # Verify reasoning_effort is set (converted from thinking) assert "reasoning_effort" in call_kwargs, "reasoning_effort should be passed to completion" - # reasoning_effort is transformed into a dict with effort and summary fields - expected_reasoning_effort = {"effort": "minimal", "summary": "detailed"} + # reasoning_effort is a dict with effort only (summary is opt-in per OpenAI spec) + expected_reasoning_effort = {"effort": "minimal"} assert call_kwargs["reasoning_effort"] == expected_reasoning_effort, \ f"reasoning_effort should be {expected_reasoning_effort} for budget_tokens=1024, got {call_kwargs.get('reasoning_effort')}" @@ -249,8 +249,8 @@ class TestThinkingSummaryPreservation: ) assert completion_kwargs["reasoning_effort"] == {"effort": "high", "summary": "auto"} - def test_thinking_without_summary_defaults_to_detailed(self): - """When no summary is provided, default 'detailed' should still be used.""" + def test_thinking_without_summary_does_not_inject_summary(self): + """When no summary is provided, no summary should be injected (opt-in per OpenAI spec).""" from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) @@ -260,7 +260,8 @@ class TestThinkingSummaryPreservation: LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( completion_kwargs, thinking=thinking ) - assert completion_kwargs["reasoning_effort"] == {"effort": "medium", "summary": "detailed"} + assert completion_kwargs["reasoning_effort"] == {"effort": "medium"} + assert "summary" not in completion_kwargs["reasoning_effort"] def test_openai_model_with_thinking_summary_end_to_end(self): """End-to-end: anthropic_messages_handler should preserve thinking.summary for OpenAI models.""" @@ -289,3 +290,29 @@ class TestThinkingSummaryPreservation: reasoning_effort = call_kwargs["reasoning_effort"] assert reasoning_effort["summary"] == "concise", \ f"Expected summary='concise', got summary='{reasoning_effort.get('summary')}'" + + def test_translate_thinking_for_model_preserves_summary(self): + """translate_thinking_for_model should include summary in reasoning_effort dict when user provides it.""" + from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( + LiteLLMAnthropicMessagesAdapter, + ) + + thinking = {"type": "enabled", "budget_tokens": 5000, "summary": "concise"} + result = LiteLLMAnthropicMessagesAdapter.translate_thinking_for_model( + thinking=thinking, + model="openai/gpt-5.2", + ) + assert result == {"reasoning_effort": {"effort": "medium", "summary": "concise"}} + + def test_translate_thinking_for_model_no_summary_when_not_provided(self): + """translate_thinking_for_model should return plain string reasoning_effort when no summary provided.""" + from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( + LiteLLMAnthropicMessagesAdapter, + ) + + thinking = {"type": "enabled", "budget_tokens": 5000} + result = LiteLLMAnthropicMessagesAdapter.translate_thinking_for_model( + thinking=thinking, + model="openai/gpt-5.2", + ) + assert result == {"reasoning_effort": "medium"} From ba5d32b6b81c8dd707d3949a10f80bbbe14f2885 Mon Sep 17 00:00:00 2001 From: Chesars Date: Wed, 4 Mar 2026 10:51:59 -0300 Subject: [PATCH 03/15] =?UTF-8?q?fix:=20address=20review=20feedback=20?= =?UTF-8?q?=E2=80=94=20remove=20redundant=20guard,=20preserve=20summary=20?= =?UTF-8?q?in=20translate=5Fanthropic=5Fto=5Fopenai?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove redundant isinstance(thinking, dict) check in handler.py since early return on line 64 guarantees thinking is a dict at that point - Preserve summary in translate_anthropic_to_openai() for consistency across all code paths (adapter, guardrail, main.py) --- .../anthropic/experimental_pass_through/adapters/handler.py | 2 +- .../experimental_pass_through/adapters/transformation.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index 01c8f39ee8..4935c65f40 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -78,7 +78,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: completion_kwargs["model"] = f"responses/{model}" reasoning_effort = completion_kwargs.get("reasoning_effort") - summary = thinking.get("summary") if isinstance(thinking, dict) else None + summary = thinking.get("summary") if isinstance(reasoning_effort, str) and reasoning_effort: reasoning_dict: Dict[str, Any] = {"effort": reasoning_effort} if summary: diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 2af5e35112..ca1a94237a 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -892,7 +892,11 @@ class LiteLLMAnthropicMessagesAdapter: cast(Dict[str, Any], thinking) ) if reasoning_effort: - new_kwargs["reasoning_effort"] = reasoning_effort + summary = thinking.get("summary") if isinstance(thinking, dict) else None + if summary: + new_kwargs["reasoning_effort"] = {"effort": reasoning_effort, "summary": summary} + else: + new_kwargs["reasoning_effort"] = reasoning_effort ## CONVERT OUTPUT_FORMAT to RESPONSE_FORMAT if "output_format" in anthropic_message_request: From 57c0b466e1c78350d24e8a5ecb90feb472358750 Mon Sep 17 00:00:00 2001 From: Chesars Date: Wed, 4 Mar 2026 22:04:15 -0300 Subject: [PATCH 04/15] docs: add thinking.summary field to /v1/messages and reasoning_content docs Document the `summary` optional field in the `thinking` object for the Anthropic `/v1/messages` adapter, and add a section on summary preservation when routing to non-Anthropic providers via the adapter. --- docs/my-website/docs/anthropic_unified/index.md | 9 ++++++--- docs/my-website/docs/reasoning_content.md | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/anthropic_unified/index.md b/docs/my-website/docs/anthropic_unified/index.md index 9981547ce1..f8a50e14da 100644 --- a/docs/my-website/docs/anthropic_unified/index.md +++ b/docs/my-website/docs/anthropic_unified/index.md @@ -506,12 +506,15 @@ Request body will be in the Anthropic messages API format. **litellm follows the A system prompt providing context or specific instructions to the model. - **temperature** (number): Controls randomness in the model's responses. Valid range: `0 < temperature < 1`. -- **thinking** (object): +- **thinking** (object): Configuration for enabling extended thinking. If enabled, it includes: - - **budget_tokens** (integer): + - **budget_tokens** (integer): Minimum of 1024 tokens (and less than `max_tokens`). - - **type** (enum): + - **type** (enum): E.g., `"enabled"`. + - **summary** (string, optional): + Enables the summary style for thinking blocks. Possible values: `"auto"`, `"concise"`, `"detailed"`, `"disabled"`. + When routing to non-Anthropic providers (e.g., `openai/gpt-5.1`), the `summary` value is preserved and forwarded to the downstream API. - **tool_choice** (object): Instructs how the model should utilize any provided tools. - **tools** (array of objects): diff --git a/docs/my-website/docs/reasoning_content.md b/docs/my-website/docs/reasoning_content.md index b5a5809bd4..05c374f38d 100644 --- a/docs/my-website/docs/reasoning_content.md +++ b/docs/my-website/docs/reasoning_content.md @@ -675,3 +675,19 @@ response = litellm.completion( reasoning_effort={"effort": "low", "summary": "detailed"}, # Explicit control ) ``` + +### Summary Preservation via `/v1/messages` Adapter + +When using the Anthropic `/v1/messages` adapter to route non-Claude models (e.g., `openai/gpt-5.1`), the `thinking.summary` value is preserved and forwarded to the downstream provider. For example: + +```python +import litellm + +response = await litellm.anthropic.messages.acreate( + model="openai/gpt-5.1", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=8096, + thinking={"type": "enabled", "budget_tokens": 5000, "summary": "concise"}, +) +# The summary="concise" is preserved when routing to OpenAI's Responses API +``` From b3a17596fe6b214b4bf0c7136336eada4c23358e Mon Sep 17 00:00:00 2001 From: Gustavo Martin Alvarez <55332916+gustipardo@users.noreply.github.com> Date: Wed, 4 Mar 2026 23:52:08 -0300 Subject: [PATCH 05/15] fix(gemini): resolve image token undercounting in usage metadata (#22608) * fix(gemini): ensure image token accumulation in usage metadata Fixed an issue where image tokens were being overwritten instead of accumulated in Gemini responses. Added support for both camelCase and snake_case token count keys. Fixes #22082. * test: add regression test for image token accumulation and cleanup files * fix(gemini): ensure consistent accumulation for responseTokensDetails * fix(gemini): harden token count parsing and add vertex accumulation test Parse tokenCount/token_count as int-safe values to satisfy mypy and avoid None/object arithmetic. Add regression test for duplicate modality accumulation in Vertex _calculate_usage. --- .../gemini/image_generation/transformation.py | 13 ++-- .../vertex_and_google_ai_studio_gemini.py | 75 ++++++++++++------- .../test_gemini_image_usage.py | 55 ++++++++++++++ ...test_vertex_and_google_ai_studio_gemini.py | 37 ++++++++- 4 files changed, 148 insertions(+), 32 deletions(-) diff --git a/litellm/llms/gemini/image_generation/transformation.py b/litellm/llms/gemini/image_generation/transformation.py index 73aef15e4c..6716d9a138 100644 --- a/litellm/llms/gemini/image_generation/transformation.py +++ b/litellm/llms/gemini/image_generation/transformation.py @@ -92,12 +92,15 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): tokens_details = usage_metadata.get("promptTokensDetails", []) for details in tokens_details: if isinstance(details, dict): - modality = details.get("modality") - token_count = details.get("tokenCount", 0) + modality = str(details.get("modality", "")).upper() + raw_token_count = details.get( + "tokenCount", details.get("token_count", 0) + ) + token_count = raw_token_count if isinstance(raw_token_count, int) else 0 if modality == "TEXT": - input_tokens_details.text_tokens = token_count + input_tokens_details.text_tokens += token_count elif modality == "IMAGE": - input_tokens_details.image_tokens = token_count + input_tokens_details.image_tokens += token_count return ImageUsage( input_tokens=usage_metadata.get("promptTokenCount", 0), @@ -274,4 +277,4 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): b64_json=prediction.get("bytesBase64Encoded", None), url=None, # Google AI returns base64, not URLs )) - return model_response \ No newline at end of file + return model_response diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index eb2d5ad51c..4d10dbf7a0 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -1623,6 +1623,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): response_tokens: Optional[int] = None response_tokens_details: Optional[CompletionTokensDetailsWrapper] = None usage_metadata = completion_response["usageMetadata"] + + def _get_token_count(detail: dict) -> int: + raw_token_count = detail.get("tokenCount", detail.get("token_count", 0)) + return raw_token_count if isinstance(raw_token_count, int) else 0 + if "cachedContentTokenCount" in usage_metadata: cached_tokens = usage_metadata["cachedContentTokenCount"] @@ -1632,10 +1637,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if "responseTokensDetails" in usage_metadata: response_tokens_details = CompletionTokensDetailsWrapper() for detail in usage_metadata["responseTokensDetails"]: - if detail["modality"] == "TEXT": - response_tokens_details.text_tokens = detail.get("tokenCount", 0) - elif detail["modality"] == "AUDIO": - response_tokens_details.audio_tokens = detail.get("tokenCount", 0) + modality = str(detail.get("modality", "")).upper() + token_count = _get_token_count(detail) + if modality == "TEXT": + response_tokens_details.text_tokens = ( + response_tokens_details.text_tokens or 0 + ) + token_count + elif modality == "AUDIO": + response_tokens_details.audio_tokens = ( + response_tokens_details.audio_tokens or 0 + ) + token_count ######################################################### @@ -1644,16 +1655,24 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if response_tokens_details is None: response_tokens_details = CompletionTokensDetailsWrapper() for detail in usage_metadata["candidatesTokensDetails"]: - modality = detail.get("modality") - token_count = detail.get("tokenCount", 0) + modality = str(detail.get("modality", "")).upper() + token_count = _get_token_count(detail) if modality == "TEXT": - response_tokens_details.text_tokens = token_count + response_tokens_details.text_tokens = ( + response_tokens_details.text_tokens or 0 + ) + token_count elif modality == "AUDIO": - response_tokens_details.audio_tokens = token_count + response_tokens_details.audio_tokens = ( + response_tokens_details.audio_tokens or 0 + ) + token_count elif modality == "IMAGE": - response_tokens_details.image_tokens = token_count + response_tokens_details.image_tokens = ( + response_tokens_details.image_tokens or 0 + ) + token_count elif modality == "VIDEO": - response_tokens_details.video_tokens = token_count + response_tokens_details.video_tokens = ( + response_tokens_details.video_tokens or 0 + ) + token_count # Calculate text_tokens if not explicitly provided in candidatesTokensDetails # candidatesTokenCount includes all modalities, so: text = total - (image + audio + video) @@ -1677,14 +1696,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ## Parse promptTokensDetails (total tokens by modality, includes cached + non-cached) if "promptTokensDetails" in usage_metadata: for detail in usage_metadata["promptTokensDetails"]: - if detail["modality"] == "AUDIO": - prompt_audio_tokens = detail.get("tokenCount", 0) - elif detail["modality"] == "TEXT": - prompt_text_tokens = detail.get("tokenCount", 0) - elif detail["modality"] == "IMAGE": - prompt_image_tokens = detail.get("tokenCount", 0) - elif detail["modality"] == "VIDEO": - prompt_video_tokens = detail.get("tokenCount", 0) + modality = str(detail.get("modality", "")).upper() + token_count = _get_token_count(detail) + if modality == "AUDIO": + prompt_audio_tokens = (prompt_audio_tokens or 0) + token_count + elif modality == "TEXT": + prompt_text_tokens = (prompt_text_tokens or 0) + token_count + elif modality == "IMAGE": + prompt_image_tokens = (prompt_image_tokens or 0) + token_count + elif modality == "VIDEO": + prompt_video_tokens = (prompt_video_tokens or 0) + token_count ## Parse cacheTokensDetails (breakdown of cached tokens by modality) ## When explicit caching is used, Gemini provides this field to show which modalities were cached @@ -1695,14 +1716,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if "cacheTokensDetails" in usage_metadata: for detail in usage_metadata["cacheTokensDetails"]: - if detail["modality"] == "AUDIO": - cached_audio_tokens = detail.get("tokenCount", 0) - elif detail["modality"] == "TEXT": - cached_text_tokens = detail.get("tokenCount", 0) - elif detail["modality"] == "IMAGE": - cached_image_tokens = detail.get("tokenCount", 0) - elif detail["modality"] == "VIDEO": - cached_video_tokens = detail.get("tokenCount", 0) + modality = str(detail.get("modality", "")).upper() + token_count = _get_token_count(detail) + if modality == "AUDIO": + cached_audio_tokens = (cached_audio_tokens or 0) + token_count + elif modality == "TEXT": + cached_text_tokens = (cached_text_tokens or 0) + token_count + elif modality == "IMAGE": + cached_image_tokens = (cached_image_tokens or 0) + token_count + elif modality == "VIDEO": + cached_video_tokens = (cached_video_tokens or 0) + token_count ## Calculate non-cached tokens by subtracting cached from total (per modality) ## This is necessary because promptTokensDetails includes both cached and non-cached tokens diff --git a/tests/llm_translation/test_gemini_image_usage.py b/tests/llm_translation/test_gemini_image_usage.py index 8c7f05d38e..0497d7fd9d 100644 --- a/tests/llm_translation/test_gemini_image_usage.py +++ b/tests/llm_translation/test_gemini_image_usage.py @@ -4,9 +4,11 @@ Test for Gemini image generation usage metadata extraction. This test verifies the fix for issue #18323 where image_generation() was returning usage=0 while completion() returned proper token usage. """ +import os import pytest from unittest.mock import patch, MagicMock import litellm +from litellm.llms.gemini.image_generation.transformation import GoogleImageGenConfig from litellm.types.utils import ImageResponse, ImageObject, ImageUsage @@ -211,3 +213,56 @@ def test_gemini_imagen_models_no_usage_extraction(): # For Imagen models, we don't extract usage from the predictions format # This test just ensures we don't crash + + +def test_gemini_image_generation_accumulates_multiple_image_prompt_token_details(): + """ + Regression test: promptTokensDetails can include multiple IMAGE entries. + These must be accumulated instead of overwritten. + """ + previous_local_model_cost_map = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") + previous_model_cost = litellm.model_cost + try: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "gemini/gemini-3-pro-image-preview" + config = GoogleImageGenConfig() + + usage_metadata = { + "promptTokenCount": 200, + "candidatesTokenCount": 0, + "totalTokenCount": 200, + "promptTokensDetails": [ + {"modality": "TEXT", "tokenCount": 10}, + {"modality": "IMAGE", "tokenCount": 90}, + {"modality": "IMAGE", "tokenCount": 100}, + ], + } + + parsed_usage = config._transform_image_usage(usage_metadata) + image_response = ImageResponse( + data=[ImageObject(b64_json="fake_image_data")], + usage=parsed_usage, + ) + + observed_cost = litellm.completion_cost( + completion_response=image_response, + model=model, + custom_llm_provider="gemini", + ) + + model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") + expected_image_tokens = 190 + expected_total_prompt_tokens = 200 + expected_prompt_cost = expected_total_prompt_tokens * model_info["input_cost_per_token"] + + assert parsed_usage.input_tokens_details.image_tokens == expected_image_tokens + assert parsed_usage.input_tokens_details.text_tokens == 10 + assert observed_cost == pytest.approx(expected_prompt_cost, rel=1e-12) + finally: + if previous_local_model_cost_map is None: + os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) + else: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = previous_local_model_cost_map + litellm.model_cost = previous_model_cost diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 8beb19bf1a..0f8ae71543 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -862,6 +862,42 @@ def test_vertex_ai_usage_metadata_with_image_tokens_in_prompt(): ) +def test_vertex_ai_usage_metadata_accumulates_duplicate_modalities(): + """Ensure _calculate_usage accumulates repeated modality entries.""" + v = VertexGeminiConfig() + usage_metadata = { + "promptTokenCount": 210, + "candidatesTokenCount": 50, + "totalTokenCount": 260, + "promptTokensDetails": [ + {"modality": "TEXT", "tokenCount": 20}, + {"modality": "IMAGE", "tokenCount": 90}, + {"modality": "IMAGE", "token_count": 100}, + ], + "candidatesTokensDetails": [ + {"modality": "IMAGE", "tokenCount": 30}, + {"modality": "TEXT", "tokenCount": 15}, + {"modality": "TEXT", "token_count": 5}, + ], + "cacheTokensDetails": [ + {"modality": "TEXT", "tokenCount": 4}, + {"modality": "IMAGE", "tokenCount": 40}, + {"modality": "IMAGE", "token_count": 10}, + ], + } + usage_metadata = UsageMetadata(**usage_metadata) + result = v._calculate_usage(completion_response={"usageMetadata": usage_metadata}) + + # prompt details are total - cached per modality + assert result.prompt_tokens_details.text_tokens == 16 # 20 - 4 + assert result.prompt_tokens_details.image_tokens == 140 # (90 + 100) - (40 + 10) + + # candidates details accumulate duplicate modalities + assert result.completion_tokens_details.text_tokens == 20 # 15 + 5 + assert result.completion_tokens_details.image_tokens == 30 + assert result.completion_tokens == 50 + + def test_vertex_ai_map_thinking_param_with_budget_tokens_0(): """ If budget_tokens is 0, do not set includeThoughts to True @@ -3723,4 +3759,3 @@ def test_vertex_ai_usage_metadata_video_tokens_with_caching(): "Prompt video tokens should be 10240 - 5120 (cached) = 5120" assert result.prompt_tokens_details.text_tokens == 9 assert result.prompt_tokens_details.audio_tokens == 200 - From 607a9683a4a5d3d8cdb1b2f5463eaa875966d68e Mon Sep 17 00:00:00 2001 From: Chesars Date: Thu, 5 Mar 2026 14:02:07 +0000 Subject: [PATCH 06/15] feat(anthropic): add opt-out flag for default reasoning summary Add `litellm.disable_default_reasoning_summary` flag (default False) and env var `LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY` to allow users to opt out of the automatic `summary="detailed"` injection when routing Anthropic thinking requests to OpenAI's Responses API. Default behavior is preserved (summary="detailed" is always added), but users who don't want to pay for summary tokens can now disable it. https://claude.ai/code/session_01VJU9EwVvgvmeCe3Yu1aULa --- litellm/__init__.py | 1 + .../adapters/handler.py | 19 ++- .../responses_adapters/transformation.py | 9 ++ ...erimental_pass_through_messages_handler.py | 134 +++++++++++++----- .../test_responses_adapters_transformation.py | 32 +++++ 5 files changed, 156 insertions(+), 39 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 57e9cb25f4..ceadb983bd 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -293,6 +293,7 @@ llm_guard_mode: Literal["all", "key-specific", "request-specific"] = "all" guardrail_name_config_map: Dict[str, GuardrailItem] = {} include_cost_in_streaming_usage: bool = False reasoning_auto_summary: bool = False +disable_default_reasoning_summary: bool = False ### PROMPTS #### from litellm.types.prompts.init_prompts import PromptSpec diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index 4935c65f40..a0954d765c 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -1,3 +1,4 @@ +import os from typing import ( TYPE_CHECKING, Any, @@ -77,22 +78,30 @@ class LiteLLMMessagesToCompletionTransformationHandler: # Prefix model with "responses/" to route to OpenAI Responses API completion_kwargs["model"] = f"responses/{model}" + summary_disabled = ( + litellm.disable_default_reasoning_summary + or os.getenv("LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY", "false").lower() == "true" + ) + reasoning_effort = completion_kwargs.get("reasoning_effort") summary = thinking.get("summary") if isinstance(reasoning_effort, str) and reasoning_effort: reasoning_dict: Dict[str, Any] = {"effort": reasoning_effort} if summary: reasoning_dict["summary"] = summary + elif not summary_disabled: + reasoning_dict["summary"] = "detailed" completion_kwargs["reasoning_effort"] = reasoning_dict elif isinstance(reasoning_effort, dict): if ( - summary - and "summary" not in reasoning_effort + "summary" not in reasoning_effort and "generate_summary" not in reasoning_effort ): - updated_reasoning_effort = dict(reasoning_effort) - updated_reasoning_effort["summary"] = summary - completion_kwargs["reasoning_effort"] = updated_reasoning_effort + effective_summary = summary if summary else ("detailed" if not summary_disabled else None) + if effective_summary: + updated_reasoning_effort = dict(reasoning_effort) + updated_reasoning_effort["summary"] = effective_summary + completion_kwargs["reasoning_effort"] = updated_reasoning_effort @staticmethod def _prepare_completion_kwargs( diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 497809b05f..ec855acd23 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -6,8 +6,11 @@ path used for OpenAI and Azure models. """ import json +import os from typing import Any, Dict, List, Optional, Union, cast +import litellm + from litellm.types.llms.anthropic import ( AllAnthropicToolsValues, AnthopicMessagesAssistantMessageParam, @@ -241,10 +244,16 @@ class LiteLLMAnthropicToResponsesAPIAdapter: effort = "low" else: effort = "minimal" + summary_disabled = ( + litellm.disable_default_reasoning_summary + or os.getenv("LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY", "false").lower() == "true" + ) result: Dict[str, Any] = {"effort": effort} summary = thinking.get("summary") if summary: result["summary"] = summary + elif not summary_disabled: + result["summary"] = "detailed" return result def translate_request( diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 7864c5f59a..384c4a97c1 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -180,7 +180,8 @@ def test_openai_model_with_thinking_converts_to_reasoning(): assert "reasoning" in call_kwargs, "reasoning should be passed to litellm.responses" # budget_tokens=1024 -> effort="minimal" (< 2000 threshold) - expected_reasoning = {"effort": "minimal"} + # summary="detailed" added by default unless disable_default_reasoning_summary is set + expected_reasoning = {"effort": "minimal", "summary": "detailed"} assert call_kwargs["reasoning"] == expected_reasoning, ( f"reasoning should be {expected_reasoning} for budget_tokens=1024, " f"got {call_kwargs.get('reasoning')}" @@ -225,7 +226,7 @@ class TestThinkingParameterTransformation: class TestThinkingSummaryPreservation: - """Tests for issue #20998: thinking.summary must be preserved when routing to OpenAI Responses API.""" + """Tests for thinking.summary preservation and disable_default_reasoning_summary flag.""" def test_thinking_summary_concise_preserved_for_openai(self): """User-provided summary='concise' should not be replaced with 'detailed'.""" @@ -253,26 +254,98 @@ class TestThinkingSummaryPreservation: ) assert completion_kwargs["reasoning_effort"] == {"effort": "high", "summary": "auto"} - def test_thinking_without_summary_does_not_inject_summary(self): - """When no summary is provided, no summary should be injected (opt-in per OpenAI spec).""" + def test_summary_added_by_default_when_no_user_summary(self): + """When no user summary and flag is off, summary='detailed' is added by default.""" + import litellm from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) - thinking = {"type": "enabled", "budget_tokens": 5000} - completion_kwargs = {"model": "openai/gpt-5.1", "reasoning_effort": "medium"} - LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( - completion_kwargs, thinking=thinking + original = litellm.disable_default_reasoning_summary + try: + litellm.disable_default_reasoning_summary = False + completion_kwargs = { + "model": "responses/gpt-5.2", + "custom_llm_provider": "openai", + "reasoning_effort": "medium", + } + LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( + completion_kwargs, thinking={"type": "enabled", "budget_tokens": 5000} + ) + assert completion_kwargs["reasoning_effort"] == {"effort": "medium", "summary": "detailed"} + finally: + litellm.disable_default_reasoning_summary = original + + def test_summary_excluded_when_disable_flag_set_string_reasoning(self): + """When disable_default_reasoning_summary is True, summary is not added for string reasoning_effort.""" + import litellm + from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, ) - assert completion_kwargs["reasoning_effort"] == {"effort": "medium"} - assert "summary" not in completion_kwargs["reasoning_effort"] + + original = litellm.disable_default_reasoning_summary + try: + litellm.disable_default_reasoning_summary = True + completion_kwargs = { + "model": "responses/gpt-5.2", + "custom_llm_provider": "openai", + "reasoning_effort": "high", + } + LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( + completion_kwargs, thinking={"type": "enabled", "budget_tokens": 10000} + ) + assert completion_kwargs["reasoning_effort"] == {"effort": "high"} + assert "summary" not in completion_kwargs["reasoning_effort"] + finally: + litellm.disable_default_reasoning_summary = original + + def test_summary_excluded_when_disable_flag_set_dict_reasoning(self): + """When disable_default_reasoning_summary is True, summary is not injected into dict reasoning_effort.""" + import litellm + from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, + ) + + original = litellm.disable_default_reasoning_summary + try: + litellm.disable_default_reasoning_summary = True + completion_kwargs = { + "model": "responses/gpt-5.2", + "custom_llm_provider": "openai", + "reasoning_effort": {"effort": "medium"}, + } + LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( + completion_kwargs, thinking={"type": "enabled", "budget_tokens": 5000} + ) + assert completion_kwargs["reasoning_effort"] == {"effort": "medium"} + assert "summary" not in completion_kwargs["reasoning_effort"] + finally: + litellm.disable_default_reasoning_summary = original + + def test_user_provided_summary_preserved_even_when_flag_off(self): + """When user already set summary in dict reasoning_effort, it's preserved regardless of flag.""" + import litellm + from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, + ) + + original = litellm.disable_default_reasoning_summary + try: + litellm.disable_default_reasoning_summary = False + completion_kwargs = { + "model": "responses/gpt-5.2", + "custom_llm_provider": "openai", + "reasoning_effort": {"effort": "high", "summary": "concise"}, + } + LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( + completion_kwargs, thinking={"type": "enabled", "budget_tokens": 10000} + ) + assert completion_kwargs["reasoning_effort"]["summary"] == "concise" + finally: + litellm.disable_default_reasoning_summary = original def test_openai_model_with_thinking_summary_end_to_end(self): - """End-to-end: anthropic_messages_handler should preserve thinking.summary for OpenAI models. - - OpenAI models are routed to litellm.responses(), so we verify the - reasoning dict passed to it contains the user's summary value. - """ + """End-to-end: anthropic_messages_handler should preserve thinking.summary for OpenAI models.""" from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages_handler, ) @@ -309,16 +382,22 @@ class TestThinkingSummaryPreservation: result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning(thinking) assert result == {"effort": "medium", "summary": "concise"} - def test_responses_adapter_no_summary_when_not_provided(self): - """translate_thinking_to_reasoning should not include summary when not provided.""" + def test_responses_adapter_no_summary_when_disabled(self): + """translate_thinking_to_reasoning should not include summary when flag is set and no user summary.""" + import litellm from litellm.llms.anthropic.experimental_pass_through.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) - thinking = {"type": "enabled", "budget_tokens": 5000} - result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning(thinking) - assert result == {"effort": "medium"} - assert "summary" not in result + original = litellm.disable_default_reasoning_summary + try: + litellm.disable_default_reasoning_summary = True + thinking = {"type": "enabled", "budget_tokens": 5000} + result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning(thinking) + assert result == {"effort": "medium"} + assert "summary" not in result + finally: + litellm.disable_default_reasoning_summary = original def test_translate_thinking_for_model_preserves_summary(self): """translate_thinking_for_model should include summary in reasoning_effort dict when user provides it.""" @@ -332,16 +411,3 @@ class TestThinkingSummaryPreservation: model="openai/gpt-5.2", ) assert result == {"reasoning_effort": {"effort": "medium", "summary": "concise"}} - - def test_translate_thinking_for_model_no_summary_when_not_provided(self): - """translate_thinking_for_model should return plain string reasoning_effort when no summary provided.""" - from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( - LiteLLMAnthropicMessagesAdapter, - ) - - thinking = {"type": "enabled", "budget_tokens": 5000} - result = LiteLLMAnthropicMessagesAdapter.translate_thinking_for_model( - thinking=thinking, - model="openai/gpt-5.2", - ) - assert result == {"reasoning_effort": "medium"} diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 252ba230ff..7f77394552 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -658,6 +658,38 @@ class TestTranslateThinkingToReasoning: result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled"}) assert result == {"effort": "minimal", "summary": "detailed"} + def test_summary_excluded_when_disable_flag_set(self): + """When disable_default_reasoning_summary is True, summary is not included.""" + import litellm + + original = litellm.disable_default_reasoning_summary + try: + litellm.disable_default_reasoning_summary = True + result = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 10000} + ) + assert result == {"effort": "high"} + assert "summary" not in result + finally: + litellm.disable_default_reasoning_summary = original + + def test_summary_excluded_when_env_var_set(self): + """When LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY env var is true, summary is not included.""" + import litellm + + original = litellm.disable_default_reasoning_summary + try: + litellm.disable_default_reasoning_summary = False + os.environ["LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY"] = "true" + result = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 5000} + ) + assert result == {"effort": "medium"} + assert "summary" not in result + finally: + litellm.disable_default_reasoning_summary = original + os.environ.pop("LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY", None) + # --------------------------------------------------------------------------- # translate_request – broader coverage From 3e9ea6f49bf9b552834ba0a21a827fecc131854a Mon Sep 17 00:00:00 2001 From: Chesars Date: Thu, 5 Mar 2026 11:24:18 -0300 Subject: [PATCH 07/15] refactor: extract summary_disabled logic into shared helper and add missing env var test - Extract duplicated summary_disabled evaluation from handler.py and transformation.py into a shared is_default_reasoning_summary_disabled() helper in utils.py to prevent future divergence. - Add test_summary_excluded_when_env_var_set to handler test class to close env-var test coverage gap flagged by Greptile. --- .../adapters/handler.py | 9 +++---- .../responses_adapters/transformation.py | 9 +++---- .../experimental_pass_through/utils.py | 12 +++++++++ ...erimental_pass_through_messages_handler.py | 25 +++++++++++++++++++ 4 files changed, 45 insertions(+), 10 deletions(-) create mode 100644 litellm/llms/anthropic/experimental_pass_through/utils.py diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index a0954d765c..e7effb77ca 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -1,4 +1,3 @@ -import os from typing import ( TYPE_CHECKING, Any, @@ -13,6 +12,9 @@ from typing import ( ) import litellm +from litellm.llms.anthropic.experimental_pass_through.utils import ( + is_default_reasoning_summary_disabled, +) from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( AnthropicAdapter, ) @@ -78,10 +80,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: # Prefix model with "responses/" to route to OpenAI Responses API completion_kwargs["model"] = f"responses/{model}" - summary_disabled = ( - litellm.disable_default_reasoning_summary - or os.getenv("LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY", "false").lower() == "true" - ) + summary_disabled = is_default_reasoning_summary_disabled() reasoning_effort = completion_kwargs.get("reasoning_effort") summary = thinking.get("summary") diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index ec855acd23..5b4f2a19f2 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -6,10 +6,12 @@ path used for OpenAI and Azure models. """ import json -import os from typing import Any, Dict, List, Optional, Union, cast import litellm +from litellm.llms.anthropic.experimental_pass_through.utils import ( + is_default_reasoning_summary_disabled, +) from litellm.types.llms.anthropic import ( AllAnthropicToolsValues, @@ -244,10 +246,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: effort = "low" else: effort = "minimal" - summary_disabled = ( - litellm.disable_default_reasoning_summary - or os.getenv("LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY", "false").lower() == "true" - ) + summary_disabled = is_default_reasoning_summary_disabled() result: Dict[str, Any] = {"effort": effort} summary = thinking.get("summary") if summary: diff --git a/litellm/llms/anthropic/experimental_pass_through/utils.py b/litellm/llms/anthropic/experimental_pass_through/utils.py new file mode 100644 index 0000000000..4a40e4629e --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/utils.py @@ -0,0 +1,12 @@ +import os + +import litellm + + +def is_default_reasoning_summary_disabled() -> bool: + """Check whether the default 'summary: detailed' injection should be suppressed.""" + return ( + litellm.disable_default_reasoning_summary + or os.getenv("LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY", "false").lower() + == "true" + ) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 384c4a97c1..0bc28cc086 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -322,6 +322,31 @@ class TestThinkingSummaryPreservation: finally: litellm.disable_default_reasoning_summary = original + def test_summary_excluded_when_env_var_set(self): + """When LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY env var is true, summary is not added.""" + import litellm + from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, + ) + + original = litellm.disable_default_reasoning_summary + try: + litellm.disable_default_reasoning_summary = False + os.environ["LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY"] = "true" + completion_kwargs = { + "model": "responses/gpt-5.2", + "custom_llm_provider": "openai", + "reasoning_effort": "high", + } + LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( + completion_kwargs, thinking={"type": "enabled", "budget_tokens": 10000} + ) + assert completion_kwargs["reasoning_effort"] == {"effort": "high"} + assert "summary" not in completion_kwargs["reasoning_effort"] + finally: + litellm.disable_default_reasoning_summary = original + os.environ.pop("LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY", None) + def test_user_provided_summary_preserved_even_when_flag_off(self): """When user already set summary in dict reasoning_effort, it's preserved regardless of flag.""" import litellm From 5b904f6054849640df07d2d4c05284ac0420970a Mon Sep 17 00:00:00 2001 From: Chesars Date: Thu, 5 Mar 2026 12:22:18 -0300 Subject: [PATCH 08/15] fix(anthropic): align translate_thinking_for_model with default summary injection + add docs - Update translate_thinking_for_model (3rd code path) to inject summary="detailed" by default, consistent with the other two paths - Add disable_default_reasoning_summary flag check via shared helper - Add tests for flag enabled/disabled and user-provided summary - Document disable_default_reasoning_summary in reasoning_content.md --- docs/my-website/docs/reasoning_content.md | 50 +++++++++++++++++++ .../adapters/transformation.py | 7 +++ ...erimental_pass_through_messages_handler.py | 34 ++++++++++++- 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/reasoning_content.md b/docs/my-website/docs/reasoning_content.md index 05c374f38d..3693ab3315 100644 --- a/docs/my-website/docs/reasoning_content.md +++ b/docs/my-website/docs/reasoning_content.md @@ -691,3 +691,53 @@ response = await litellm.anthropic.messages.acreate( ) # The summary="concise" is preserved when routing to OpenAI's Responses API ``` + +### Default Summary Injection for `/v1/messages` Adapter + +When the Anthropic `/v1/messages` adapter translates `thinking` parameters to OpenAI `reasoning_effort` for non-Claude models, `summary="detailed"` is automatically injected by default. This ensures that reasoning text is returned in the response (matching the Anthropic thinking behavior). + +To **disable** this default injection, use the `disable_default_reasoning_summary` flag: + + + + +```python +import litellm + +# Disable default summary="detailed" injection +litellm.disable_default_reasoning_summary = True + +response = await litellm.anthropic.messages.acreate( + model="openai/gpt-5.1", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=8096, + thinking={"type": "enabled", "budget_tokens": 5000}, +) +# No summary will be injected — only reasoning_effort is forwarded +``` + + + + + +```bash +export LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY=true +``` + + + + + +```yaml +litellm_settings: + disable_default_reasoning_summary: true +``` + + + + +:::info + +This flag only affects the automatic injection of `summary="detailed"` when no user-provided summary is present. If you explicitly pass `thinking.summary` (e.g., `"concise"` or `"auto"`), your value is always preserved regardless of this flag. + +::: diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index b0138ecbf5..049b876332 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -13,6 +13,10 @@ from typing import ( cast, ) +from litellm.llms.anthropic.experimental_pass_through.utils import ( + is_default_reasoning_summary_disabled, +) + # OpenAI has a 64-character limit for function/tool names # Anthropic does not have this limit, so we need to truncate long names OPENAI_MAX_TOOL_NAME_LENGTH = 64 @@ -694,8 +698,11 @@ class LiteLLMAnthropicMessagesAdapter: ) if reasoning_effort: summary = thinking.get("summary") if isinstance(thinking, dict) else None + summary_disabled = is_default_reasoning_summary_disabled() if summary: return {"reasoning_effort": {"effort": reasoning_effort, "summary": summary}} + elif not summary_disabled: + return {"reasoning_effort": {"effort": reasoning_effort, "summary": "detailed"}} return {"reasoning_effort": reasoning_effort} return {} diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 0bc28cc086..203b6dacea 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -221,9 +221,41 @@ class TestThinkingParameterTransformation: model="openai/gpt-5.2", ) - assert result == {"reasoning_effort": "minimal"} + assert result == {"reasoning_effort": {"effort": "minimal", "summary": "detailed"}} assert "thinking" not in result + def test_translate_thinking_for_model_no_summary_when_disabled(self): + """When disable_default_reasoning_summary is True, no summary is injected.""" + import litellm + from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( + LiteLLMAnthropicMessagesAdapter, + ) + + original = litellm.disable_default_reasoning_summary + try: + litellm.disable_default_reasoning_summary = True + thinking = {"type": "enabled", "budget_tokens": 5000} + result = LiteLLMAnthropicMessagesAdapter.translate_thinking_for_model( + thinking=thinking, + model="openai/gpt-5.2", + ) + assert result == {"reasoning_effort": "medium"} + finally: + litellm.disable_default_reasoning_summary = original + + def test_translate_thinking_for_model_preserves_user_summary(self): + """User-provided summary is always preserved regardless of flag.""" + from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( + LiteLLMAnthropicMessagesAdapter, + ) + + thinking = {"type": "enabled", "budget_tokens": 10000, "summary": "concise"} + result = LiteLLMAnthropicMessagesAdapter.translate_thinking_for_model( + thinking=thinking, + model="openai/gpt-5.2", + ) + assert result == {"reasoning_effort": {"effort": "high", "summary": "concise"}} + class TestThinkingSummaryPreservation: """Tests for thinking.summary preservation and disable_default_reasoning_summary flag.""" From b74571214fd0c123ca9379232f1519e4751dc001 Mon Sep 17 00:00:00 2001 From: Chesars Date: Sat, 14 Mar 2026 00:53:56 -0300 Subject: [PATCH 09/15] chore: remove debug scripts and unused import Remove 8 development scripts from scripts/ that were accidentally committed. Remove unused `import litellm` from responses_adapters/transformation.py. --- .../responses_adapters/transformation.py | 1 - scripts/test_gpt54_reasoning_tools.py | 77 -------- scripts/test_perplexity_regression.py | 83 -------- scripts/test_perplexity_responses.py | 179 ------------------ scripts/test_reasoning_none_tools.py | 39 ---- scripts/test_reasoning_tools.py | 19 -- scripts/test_tool_choice_responses.py | 45 ----- scripts/test_tool_search_chat.py | 57 ------ scripts/test_tool_search_responses.py | 101 ---------- 9 files changed, 601 deletions(-) delete mode 100644 scripts/test_gpt54_reasoning_tools.py delete mode 100644 scripts/test_perplexity_regression.py delete mode 100644 scripts/test_perplexity_responses.py delete mode 100644 scripts/test_reasoning_none_tools.py delete mode 100644 scripts/test_reasoning_tools.py delete mode 100644 scripts/test_tool_choice_responses.py delete mode 100644 scripts/test_tool_search_chat.py delete mode 100644 scripts/test_tool_search_responses.py diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 54dd86eaba..2fef5dee46 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -8,7 +8,6 @@ path used for OpenAI and Azure models. import json from typing import Any, Dict, List, Optional, Union, cast -import litellm from litellm.llms.anthropic.experimental_pass_through.utils import ( is_default_reasoning_summary_disabled, ) diff --git a/scripts/test_gpt54_reasoning_tools.py b/scripts/test_gpt54_reasoning_tools.py deleted file mode 100644 index df6dea9ce7..0000000000 --- a/scripts/test_gpt54_reasoning_tools.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -Repro script: verify that gpt-5.4 drops reasoning_effort when tools are present. -Expected: the call succeeds (reasoning_effort is silently dropped). -If the bug were still present, OpenAI would return an error like: - "reasoning_effort is not supported with function calling" -""" - -import os -from dotenv import load_dotenv - -load_dotenv() - -import litellm - -litellm.set_verbose = True - -tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current weather for a city", - "parameters": { - "type": "object", - "properties": { - "city": {"type": "string", "description": "City name"}, - }, - "required": ["city"], - }, - }, - } -] - -print("=== Test: gpt-5.4 + reasoning_effort='medium' + tools ===") -try: - response = litellm.completion( - model="gpt-5.4", - messages=[{"role": "user", "content": "What's the weather in Buenos Aires?"}], - reasoning_effort="medium", - tools=tools, - drop_params=True, - ) - print(f"SUCCESS - model: {response.model}") - print(f"Choice: {response.choices[0].message}") - if response.choices[0].message.tool_calls: - print(f"Tool calls: {response.choices[0].message.tool_calls}") - print("\nreasoning_effort was correctly dropped (no error from OpenAI)") -except Exception as e: - print(f"FAILED: {e}") - -print("\n=== Test: gpt-5.4 + reasoning_effort='high' + tools ===") -try: - response = litellm.completion( - model="gpt-5.4", - messages=[{"role": "user", "content": "What's 2+2?"}], - reasoning_effort="high", - tools=tools, - drop_params=True, - ) - print(f"SUCCESS - model: {response.model}") - print(f"reasoning_effort was correctly dropped (no error from OpenAI)") -except Exception as e: - print(f"FAILED: {e}") - -print("\n=== Test: gpt-5.4 + reasoning_effort='none' + tools (should KEEP reasoning_effort) ===") -try: - response = litellm.completion( - model="gpt-5.4", - messages=[{"role": "user", "content": "Say hello"}], - reasoning_effort="none", - tools=tools, - drop_params=True, - ) - print(f"SUCCESS - model: {response.model}") - print(f"reasoning_effort='none' correctly kept (OpenAI allows this)") -except Exception as e: - print(f"FAILED: {e}") diff --git a/scripts/test_perplexity_regression.py b/scripts/test_perplexity_regression.py deleted file mode 100644 index 67a01e2dcd..0000000000 --- a/scripts/test_perplexity_regression.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -Simple regression test: call Perplexity through LiteLLM -to verify chat completions and responses API both work. -""" -import os -import sys -from dotenv import load_dotenv - -load_dotenv() - -import litellm - -# Show which branch we're on -branch = os.popen("git rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown").read().strip() -print(f"=== Branch: {branch} ===\n") - -# 1. Chat completions -print("--- Test 1: Chat Completions ---") -try: - resp = litellm.completion( - model="perplexity/sonar", - messages=[{"role": "user", "content": "Say hello in 3 words"}], - max_tokens=20, - ) - print(f"OK: {resp.choices[0].message.content[:80]}") - print(f" model: {resp.model}") - print(f" usage: {resp.usage}") -except Exception as e: - print(f"FAIL: {e}") - -# 2. Responses API (string input) -print("\n--- Test 2: Responses API (string input) ---") -try: - resp = litellm.responses( - model="perplexity/sonar", - input="Say hello in 3 words", - max_output_tokens=20, - ) - print(f"OK: {resp.output[0].content[0].text[:80]}") - print(f" model: {resp.model}") -except Exception as e: - print(f"FAIL: {e}") - -# 3. Responses API (list input - the _format_input concern) -print("\n--- Test 3: Responses API (list input without type field) ---") -try: - resp = litellm.responses( - model="perplexity/sonar", - input=[{"role": "user", "content": "Say hello in 3 words"}], - max_output_tokens=20, - ) - print(f"OK: {resp.output[0].content[0].text[:80]}") -except Exception as e: - print(f"FAIL: {e}") - -# 4. Check which config class is resolved for chat -print("\n--- Test 4: Config class resolution ---") -from litellm.utils import ProviderConfigManager -from litellm.types.utils import LlmProviders - -chat_config = ProviderConfigManager.get_provider_chat_config( - model="perplexity/sonar", provider=LlmProviders.PERPLEXITY -) -print(f"Chat config class: {type(chat_config).__name__}") -print(f" module: {type(chat_config).__module__}") - -resp_config = ProviderConfigManager.get_provider_responses_api_config( - provider=LlmProviders.PERPLEXITY -) -print(f"Responses config class: {type(resp_config).__name__}") -print(f" module: {type(resp_config).__module__}") - -# 5. Check supported params include preset/models for responses -print("\n--- Test 5: Supported params ---") -if resp_config: - params = resp_config.get_supported_openai_params("sonar") - print(f"Responses supported params: {params}") - has_preset = "preset" in params - has_models = "models" in params - print(f" Has 'preset': {has_preset}") - print(f" Has 'models': {has_models}") - -print("\n=== Done ===") diff --git a/scripts/test_perplexity_responses.py b/scripts/test_perplexity_responses.py deleted file mode 100644 index 5616ada380..0000000000 --- a/scripts/test_perplexity_responses.py +++ /dev/null @@ -1,179 +0,0 @@ -""" -Live test: Perplexity Responses API via LiteLLM. -Tests: non-streaming, streaming, preset models, models fallback param, -chat completions (regression check), and cost dict→float parsing. - -DO NOT COMMIT this file. -""" - -import os -import traceback - -from dotenv import load_dotenv - -load_dotenv() - -import litellm - -# litellm.set_verbose = True - - -def test_non_streaming_preset(): - """Test non-streaming with preset model.""" - print("=" * 60) - print("TEST 1: Non-streaming preset/pro-search") - print("=" * 60) - - response = litellm.responses( - model="perplexity/preset/pro-search", - input="What is 2 + 2? Answer in one word.", - ) - - print(f" Response ID: {response.id}") - print(f" Model: {response.model}") - print(f" Status: {response.status}") - - assert response.status == "completed", f"FAIL: status={response.status}" - assert response.output, "FAIL: no output" - print(" PASS: non-streaming preset works") - - if response.usage and response.usage.cost is not None: - assert isinstance(response.usage.cost, (int, float)), ( - f"FAIL: cost is {type(response.usage.cost)}: {response.usage.cost}" - ) - print(f" PASS: cost={response.usage.cost} (float, not dict)") - print() - - -def test_streaming_preset(): - """Test streaming with preset model.""" - print("=" * 60) - print("TEST 2: Streaming preset/pro-search") - print("=" * 60) - - response = litellm.responses( - model="perplexity/preset/pro-search", - input="What is the capital of France? One word.", - stream=True, - ) - - chunks = 0 - completed = False - for chunk in response: - chunks += 1 - event_type = getattr(chunk, "type", "unknown") - if event_type == "response.output_text.delta": - print(f" delta: {chunk.delta}", end="", flush=True) - elif event_type == "response.completed": - completed = True - print(f"\n [completed] model={chunk.response.model}") - if chunk.response.usage and chunk.response.usage.cost is not None: - cost = chunk.response.usage.cost - assert isinstance(cost, (int, float)), ( - f"FAIL: streaming cost is {type(cost)}: {cost}" - ) - print(f" PASS: streaming cost={cost} (float)") - - assert chunks > 0, "FAIL: no chunks received" - assert completed, "FAIL: never got response.completed event" - print(f" Total chunks: {chunks}") - print(" PASS: streaming preset works") - print() - - -def test_models_fallback_param(): - """Test that 'models' param (Perplexity fallback chain) is forwarded.""" - print("=" * 60) - print("TEST 3: models param (fallback chain)") - print("=" * 60) - - response = litellm.responses( - model="perplexity/openai/gpt-5.1", - input="Say 'hello' and nothing else.", - models=["openai/gpt-5-mini", "openai/gpt-5.1"], - ) - - print(f" Response ID: {response.id}") - print(f" Model used: {response.model}") - print(f" Status: {response.status}") - - assert response.status == "completed", f"FAIL: status={response.status}" - print(" PASS: models fallback param works") - print() - - -def test_chat_completions_not_broken(): - """Regression: Perplexity chat completions must still use PerplexityChatConfig.""" - print("=" * 60) - print("TEST 4: Chat completions regression check") - print("=" * 60) - - response = litellm.completion( - model="perplexity/sonar", - messages=[{"role": "user", "content": "Say 'hi' and nothing else."}], - max_tokens=10, - ) - - print(f" Model: {response.model}") - print(f" Content: {response.choices[0].message.content[:50]}") - - assert response.choices, "FAIL: no choices" - assert response.choices[0].message.content, "FAIL: empty content" - print(" PASS: chat completions still work (no regression)") - print() - - -def test_with_instructions(): - """Test instructions param.""" - print("=" * 60) - print("TEST 5: instructions param") - print("=" * 60) - - response = litellm.responses( - model="perplexity/preset/pro-search", - input="What is Python?", - instructions="Answer in exactly 5 words.", - ) - - print(f" Status: {response.status}") - # Extract text from output - for item in response.output: - if hasattr(item, "content"): - for c in item.content: - if hasattr(c, "text"): - print(f" Answer: {c.text}") - break - - assert response.status == "completed", f"FAIL: status={response.status}" - print(" PASS: instructions param works") - print() - - -if __name__ == "__main__": - api_key = os.environ.get("PERPLEXITYAI_API_KEY", "NOT SET") - print(f"Using PERPLEXITYAI_API_KEY: {api_key[:10]}...") - print() - - tests = [ - test_non_streaming_preset, - test_streaming_preset, - test_models_fallback_param, - test_chat_completions_not_broken, - test_with_instructions, - ] - - passed = 0 - failed = 0 - for test in tests: - try: - test() - passed += 1 - except Exception as e: - failed += 1 - print(f" FAIL: {e}") - traceback.print_exc() - print() - - print("=" * 60) - print(f"Results: {passed} passed, {failed} failed out of {len(tests)}") - print("=" * 60) diff --git a/scripts/test_reasoning_none_tools.py b/scripts/test_reasoning_none_tools.py deleted file mode 100644 index 30d7bd1dab..0000000000 --- a/scripts/test_reasoning_none_tools.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -Repro: gpt-5.4 + reasoning_effort='none' + tools -Current behavior: reasoning_effort='none' is NOT dropped, but OpenAI rejects it. -""" - -import os -from dotenv import load_dotenv - -load_dotenv() - -import litellm - -tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current weather", - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - "required": ["city"], - }, - }, - } -] - -print("=== gpt-5.4 + reasoning_effort='none' + tools ===") -try: - response = litellm.completion( - model="gpt-5.4", - messages=[{"role": "user", "content": "What's the weather in Buenos Aires?"}], - reasoning_effort="none", - tools=tools, - ) - print(f"SUCCESS - model: {response.model}") - print(f"Choice: {response.choices[0].message}") -except Exception as e: - print(f"FAILED: {e}") diff --git a/scripts/test_reasoning_tools.py b/scripts/test_reasoning_tools.py deleted file mode 100644 index b925d0ade2..0000000000 --- a/scripts/test_reasoning_tools.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Test gpt-5.4 with reasoning_effort + tools to see OpenAI's response.""" -import os -from dotenv import load_dotenv -load_dotenv() - -import litellm - -try: - response = litellm.completion( - model="gpt-5.4", - messages=[{"role": "user", "content": "What's the weather in SF?"}], - tools=[{"type": "function", "function": {"name": "get_weather", "description": "Get weather", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}}], - reasoning_effort="high", - ) - print("SUCCESS:") - print(response) -except Exception as e: - print(f"ERROR ({type(e).__name__}):") - print(e) diff --git a/scripts/test_tool_choice_responses.py b/scripts/test_tool_choice_responses.py deleted file mode 100644 index dab599933a..0000000000 --- a/scripts/test_tool_choice_responses.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Post-fix verification for #23423: tool_choice with responses/ prefix.""" -import os -from dotenv import load_dotenv -load_dotenv() - -import litellm - -# Verify supports_tool_choice resolves correctly -from litellm.utils import supports_tool_choice -print("supports_tool_choice('gpt-5.4'):", supports_tool_choice("gpt-5.4")) -print("supports_tool_choice('openai/responses/gpt-5.4'):", supports_tool_choice("openai/responses/gpt-5.4")) - -# Verify tool_choice is in supported params -params = litellm.get_supported_openai_params(model="openai/responses/gpt-5.4", custom_llm_provider="openai") -print("tool_choice in supported params:", "tool_choice" in params) - -# Real API call with tool_choice -tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the weather for a city", - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - "required": ["city"], - }, - }, - } -] - -response = litellm.completion( - model="openai/responses/gpt-4.1-nano", # cheaper model - messages=[{"role": "user", "content": "What's the weather in Buenos Aires?"}], - tools=tools, - tool_choice="required", -) - -print("\nResponse:") -print(" tool_calls:", response.choices[0].message.tool_calls) -print(" finish_reason:", response.choices[0].finish_reason) - -has_tool_call = response.choices[0].message.tool_calls is not None -print("\nVERDICT:", "PASS - tool_choice works" if has_tool_call else "FAIL - tool_choice dropped") diff --git a/scripts/test_tool_search_chat.py b/scripts/test_tool_search_chat.py deleted file mode 100644 index 9cfecae983..0000000000 --- a/scripts/test_tool_search_chat.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Test the Chat Completions Bridge tool search example from docs (line 856-887)""" -import os -from dotenv import load_dotenv -load_dotenv() - -import litellm - -try: - response = litellm.completion( - model="openai/responses/gpt-5.4", - messages=[{"role": "user", "content": "Look up invoice INV-2024-001"}], - tools=[ - {"type": "tool_search"}, - { - "type": "namespace", - "name": "billing", - "description": "Billing and invoicing tools", - "tools": [ - { - "type": "function", - "name": "get_invoice", - "description": "Get an invoice by ID", - "parameters": { - "type": "object", - "properties": {"invoice_id": {"type": "string"}}, - "required": ["invoice_id"], - }, - "defer_loading": True, - }, - ], - }, - ], - ) - - print("=== Raw response ===") - print(f"tool_calls value: {response.choices[0].message.tool_calls}") - print(f"tool_calls is None? {response.choices[0].message.tool_calls is None}") - print() - - # Test the docs code exactly as written - print("=== Testing docs code (no None guard) ===") - try: - for tool_call in response.choices[0].message.tool_calls: - print(f"Called: {tool_call.function.name}({tool_call.function.arguments})") - except TypeError as e: - print(f" !!! TypeError: {e}") - print(f" Greptile was RIGHT - need 'or []' guard") - - # Test with the fix - print() - print("=== Testing with fix (or [] guard) ===") - for tool_call in (response.choices[0].message.tool_calls or []): - print(f"Called: {tool_call.function.name}({tool_call.function.arguments})") - print(" OK - no crash") - -except Exception as e: - print(f"API Error: {type(e).__name__}: {e}") diff --git a/scripts/test_tool_search_responses.py b/scripts/test_tool_search_responses.py deleted file mode 100644 index 58c7f1e2ac..0000000000 --- a/scripts/test_tool_search_responses.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Test the Responses API tool search example from docs (line 705-783)""" -import os -from dotenv import load_dotenv -load_dotenv() - -import litellm -import json - -# Define namespaces with deferred tools -tools = [ - {"type": "tool_search"}, # Enable tool search - { - "type": "namespace", - "name": "crm", - "description": "CRM tools for customer management", - "tools": [ - { - "type": "function", - "name": "get_customer", - "description": "Get customer details by ID", - "parameters": { - "type": "object", - "properties": { - "customer_id": {"type": "string"} - }, - "required": ["customer_id"], - }, - "defer_loading": True, - }, - { - "type": "function", - "name": "list_customers", - "description": "List customers with optional filters", - "parameters": { - "type": "object", - "properties": { - "status": {"type": "string", "enum": ["active", "inactive"]}, - }, - }, - "defer_loading": True, - }, - ], - }, - { - "type": "namespace", - "name": "billing", - "description": "Billing and invoicing tools", - "tools": [ - { - "type": "function", - "name": "get_invoice", - "description": "Get an invoice by ID", - "parameters": { - "type": "object", - "properties": { - "invoice_id": {"type": "string"} - }, - "required": ["invoice_id"], - }, - "defer_loading": True, - }, - ], - }, -] - -try: - response = litellm.responses( - model="openai/gpt-5.4", - input="Look up invoice INV-2024-001 from the billing system", - tools=tools, - ) - - print("=== Raw response.output ===") - print(response.output) - print() - - # Test the parsing code from the docs - print("=== Parsing output items ===") - for item in response.output: - print(f" item type: {type(item)}") - if isinstance(item, dict): - print(f" dict keys: {item.keys()}") - if item["type"] == "tool_search_call": - print(f"Searched namespaces: {item['arguments']['paths']}") - elif item["type"] == "tool_search_output": - print(f"Loaded {len(item['tools'])} tool(s)") - elif item["type"] == "function_call": - print(f"Called: {item.get('namespace', '')}.{item['name']}({item['arguments']})") - else: - print(f" object attrs: {dir(item)}") - if item.type == "function_call": - # Greptile says this will fail if namespace is missing - print(f" Has 'namespace' attr? {hasattr(item, 'namespace')}") - try: - print(f"Called: {item.namespace}.{item.name}({item.arguments})") - except AttributeError as e: - print(f" !!! AttributeError: {e}") - print(f" Greptile was RIGHT - need getattr fallback") - -except Exception as e: - print(f"API Error: {type(e).__name__}: {e}") From af7e2e687870f133e4a915f231b56b00572d4b5e Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Sat, 21 Mar 2026 00:01:48 +0530 Subject: [PATCH 10/15] Fix ruff PLR0915 error --- .../adapters/transformation.py | 249 ++++++++++++------ .../vertex_and_google_ai_studio_gemini.py | 3 +- 2 files changed, 171 insertions(+), 81 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index f6847c9b1d..932266da8f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -737,12 +737,24 @@ class LiteLLMAnthropicMessagesAdapter: thinking ) if reasoning_effort: - summary = thinking.get("summary") if isinstance(thinking, dict) else None + summary = ( + thinking.get("summary") if isinstance(thinking, dict) else None + ) summary_disabled = is_default_reasoning_summary_disabled() if summary: - return {"reasoning_effort": {"effort": reasoning_effort, "summary": summary}} + return { + "reasoning_effort": { + "effort": reasoning_effort, + "summary": summary, + } + } elif not summary_disabled: - return {"reasoning_effort": {"effort": reasoning_effort, "summary": "detailed"}} + return { + "reasoning_effort": { + "effort": reasoning_effort, + "summary": "detailed", + } + } return {"reasoning_effort": reasoning_effort} return {} @@ -888,6 +900,135 @@ class LiteLLMAnthropicMessagesAdapter: ChatCompletionSystemMessage(role="system", content=openai_system_content), # type: ignore ) + def _translate_metadata_to_openai( + self, + anthropic_message_request: AnthropicMessagesRequest, + new_kwargs: ChatCompletionRequest, + ) -> None: + """Translate metadata fields from Anthropic request to OpenAI request.""" + if "metadata" in anthropic_message_request: + metadata = anthropic_message_request["metadata"] + if metadata and "user_id" in metadata: + new_kwargs["user"] = metadata["user_id"] + + if "litellm_metadata" in anthropic_message_request: + # metadata will be passed to litellm.acompletion(), it's a litellm_param + new_kwargs["metadata"] = anthropic_message_request.pop("litellm_metadata") + + def _translate_tool_choice_to_openai( + self, + anthropic_message_request: AnthropicMessagesRequest, + new_kwargs: ChatCompletionRequest, + ) -> None: + """Translate Anthropic tool_choice to OpenAI format.""" + if "tool_choice" not in anthropic_message_request: + return + tool_choice = anthropic_message_request["tool_choice"] + if not tool_choice: + return + new_kwargs["tool_choice"] = self.translate_anthropic_tool_choice_to_openai( + tool_choice=cast(AnthropicMessagesToolChoice, tool_choice) + ) + + def _translate_tools_to_openai( + self, + anthropic_message_request: AnthropicMessagesRequest, + new_kwargs: ChatCompletionRequest, + ) -> Dict[str, str]: + """Translate tools and extract web_search_options when needed.""" + if "tools" not in anthropic_message_request: + return {} + + tools = anthropic_message_request["tools"] + if not tools: + return {} + + web_search_tools: List[AllAnthropicToolsValues] = [] + regular_tools: List[AllAnthropicToolsValues] = [] + for tool in tools: + cast_tool = cast(Dict[str, Any], tool) + if self._is_web_search_tool(cast_tool): + web_search_tools.append(cast(AllAnthropicToolsValues, tool)) + else: + regular_tools.append(cast(AllAnthropicToolsValues, tool)) + + if web_search_tools: + new_kwargs["web_search_options"] = {} # type: ignore + + if not regular_tools: + return {} + + translated_tools, tool_name_mapping = self.translate_anthropic_tools_to_openai( + tools=regular_tools, + model=new_kwargs.get("model"), + ) + new_kwargs["tools"] = translated_tools + return tool_name_mapping + + def _translate_thinking_to_openai( + self, + anthropic_message_request: AnthropicMessagesRequest, + new_kwargs: ChatCompletionRequest, + ) -> None: + """Translate Anthropic thinking to either thinking or reasoning_effort.""" + if "thinking" not in anthropic_message_request: + return + + thinking = anthropic_message_request["thinking"] + if not thinking: + return + + model = new_kwargs.get("model", "") + if self.is_anthropic_claude_model(model): + new_kwargs["thinking"] = thinking # type: ignore + return + + reasoning_effort = self.translate_anthropic_thinking_to_reasoning_effort( + cast(Dict[str, Any], thinking) + ) + if not reasoning_effort: + return + + summary = thinking.get("summary") if isinstance(thinking, dict) else None + if summary: + new_kwargs["reasoning_effort"] = cast( + Any, + { + "effort": reasoning_effort, + "summary": summary, + }, + ) + else: + new_kwargs["reasoning_effort"] = reasoning_effort + + def _translate_output_format_to_openai( + self, + anthropic_message_request: AnthropicMessagesRequest, + new_kwargs: ChatCompletionRequest, + ) -> None: + """Translate output_format to response_format when applicable.""" + if "output_format" not in anthropic_message_request: + return + output_format = anthropic_message_request["output_format"] + if not output_format: + return + response_format = self.translate_anthropic_output_format_to_openai( + output_format=output_format + ) + if response_format: + new_kwargs["response_format"] = response_format + + def _copy_untranslated_anthropic_params( + self, + anthropic_message_request: AnthropicMessagesRequest, + new_kwargs: ChatCompletionRequest, + ) -> None: + """Copy through anthropic params that do not require translation.""" + translatable_params = self.translatable_anthropic_params() + for k, v in anthropic_message_request.items(): + if k not in translatable_params: # pass remaining params as is + new_kwargs[k] = v # type: ignore + def translate_anthropic_to_openai( self, anthropic_message_request: AnthropicMessagesRequest ) -> Tuple[ChatCompletionRequest, Dict[str, str]]: @@ -928,87 +1069,35 @@ class LiteLLMAnthropicMessagesAdapter: "model": anthropic_message_request["model"], "messages": new_messages, } - ## CONVERT METADATA (user_id) - if "metadata" in anthropic_message_request: - metadata = anthropic_message_request["metadata"] - if metadata and "user_id" in metadata: - new_kwargs["user"] = metadata["user_id"] - - # Pass litellm proxy specific metadata - if "litellm_metadata" in anthropic_message_request: - # metadata will be passed to litellm.acompletion(), it's a litellm_param - new_kwargs["metadata"] = anthropic_message_request.pop("litellm_metadata") - + ## CONVERT METADATA (user_id + litellm metadata) + self._translate_metadata_to_openai( + anthropic_message_request=anthropic_message_request, + new_kwargs=new_kwargs, + ) ## CONVERT TOOL CHOICE - if "tool_choice" in anthropic_message_request: - tool_choice = anthropic_message_request["tool_choice"] - if tool_choice: - new_kwargs[ - "tool_choice" - ] = self.translate_anthropic_tool_choice_to_openai( - tool_choice=cast(AnthropicMessagesToolChoice, tool_choice) - ) + self._translate_tool_choice_to_openai( + anthropic_message_request=anthropic_message_request, + new_kwargs=new_kwargs, + ) ## CONVERT TOOLS - if "tools" in anthropic_message_request: - tools = anthropic_message_request["tools"] - if tools: - # Separate web search tools from regular tools - web_search_tools = [] - regular_tools = [] - for tool in tools: - if self._is_web_search_tool(cast(Dict[str, Any], tool)): - web_search_tools.append(tool) - else: - regular_tools.append(tool) - - # If web search tools are present, add web_search_options parameter - if web_search_tools: - new_kwargs["web_search_options"] = {} # type: ignore - - # Only translate regular tools (non-web-search) - if regular_tools: - ( - new_kwargs["tools"], - tool_name_mapping, - ) = self.translate_anthropic_tools_to_openai( - tools=cast(List[AllAnthropicToolsValues], regular_tools), - model=new_kwargs.get("model"), - ) - + tool_name_mapping = self._translate_tools_to_openai( + anthropic_message_request=anthropic_message_request, + new_kwargs=new_kwargs, + ) ## CONVERT THINKING - if "thinking" in anthropic_message_request: - thinking = anthropic_message_request["thinking"] - if thinking: - model = new_kwargs.get("model", "") - if self.is_anthropic_claude_model(model): - new_kwargs["thinking"] = thinking # type: ignore - else: - reasoning_effort = ( - self.translate_anthropic_thinking_to_reasoning_effort( - cast(Dict[str, Any], thinking) - ) - ) - if reasoning_effort: - summary = thinking.get("summary") if isinstance(thinking, dict) else None - if summary: - new_kwargs["reasoning_effort"] = {"effort": reasoning_effort, "summary": summary} - else: - new_kwargs["reasoning_effort"] = reasoning_effort - + self._translate_thinking_to_openai( + anthropic_message_request=anthropic_message_request, + new_kwargs=new_kwargs, + ) ## CONVERT OUTPUT_FORMAT to RESPONSE_FORMAT - if "output_format" in anthropic_message_request: - output_format = anthropic_message_request["output_format"] - if output_format: - response_format = self.translate_anthropic_output_format_to_openai( - output_format=output_format - ) - if response_format: - new_kwargs["response_format"] = response_format - - translatable_params = self.translatable_anthropic_params() - for k, v in anthropic_message_request.items(): - if k not in translatable_params: # pass remaining params as is - new_kwargs[k] = v # type: ignore + self._translate_output_format_to_openai( + anthropic_message_request=anthropic_message_request, + new_kwargs=new_kwargs, + ) + self._copy_untranslated_anthropic_params( + anthropic_message_request=anthropic_message_request, + new_kwargs=new_kwargs, + ) return new_kwargs, tool_name_mapping diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 4ddbd06113..132821dbc9 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -12,6 +12,7 @@ from typing import ( Dict, List, Literal, + Mapping, Optional, Tuple, Type, @@ -1633,7 +1634,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): response_tokens_details: Optional[CompletionTokensDetailsWrapper] = None usage_metadata = completion_response["usageMetadata"] - def _get_token_count(detail: dict) -> int: + def _get_token_count(detail: Mapping[str, Any]) -> int: raw_token_count = detail.get("tokenCount", detail.get("token_count", 0)) return raw_token_count if isinstance(raw_token_count, int) else 0 From a05824d9bacc56f16cf6985e8ea3e77d946d7b28 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Sat, 21 Mar 2026 00:14:33 +0530 Subject: [PATCH 11/15] Fix code qa --- docs/my-website/docs/proxy/config_settings.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index d7542fc2c3..22fc0d3e91 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -813,6 +813,7 @@ router_settings: | LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development) | LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers | LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60 +| LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY | If set to "true", disables automatic default reasoning summary injection (`summary: "detailed"`) for Anthropic experimental pass-through translations. Default is "false" | LITELLM_REASONING_AUTO_SUMMARY | If set to "true", automatically enables detailed reasoning summaries for reasoning models (e.g., o1, o3-mini, deepseek-reasoner). When enabled, adds `summary: "detailed"` to reasoning effort configurations. Default is "false" | LITELLM_SALT_KEY | Salt key for encryption in LiteLLM | LITELLM_SSL_CIPHERS | SSL/TLS cipher configuration for faster handshakes. Controls cipher suite preferences for OpenSSL connections. From 0091d048dc03bd11e87f71199e10af5de85c2900 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 21 Mar 2026 11:36:26 -0700 Subject: [PATCH 12/15] fix: make reasoning summary opt-in, fix missing injection path, narrow test exceptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address Greptile review feedback: 1. Replace opt-out `disable_default_reasoning_summary` with existing opt-in `reasoning_auto_summary` flag — avoids backwards-incompatible change where all users routing thinking-enabled requests would silently get a changed reasoning_effort shape (string -> dict) on upgrade. 2. Add default summary injection to `_translate_thinking_to_openai` — this path was the only one missing it, causing inconsistent behavior for litellm.completion() callers using the Anthropic adapter. 3. Narrow `except Exception` to `except (ValueError, TypeError, AttributeError)` in tests to avoid masking genuine failures. Co-Authored-By: Claude Opus 4.6 --- docs/my-website/docs/proxy/config_settings.md | 3 +- docs/my-website/docs/reasoning_content.md | 16 +- litellm/__init__.py | 1 - .../adapters/handler.py | 14 +- .../adapters/transformation.py | 27 ++- .../responses_adapters/transformation.py | 9 +- .../experimental_pass_through/utils.py | 9 +- ...odel_prices_and_context_window_backup.json | 8 +- ...erimental_pass_through_messages_handler.py | 173 ++++++++++-------- .../test_responses_adapters_transformation.py | 76 +++++--- 10 files changed, 199 insertions(+), 137 deletions(-) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 22fc0d3e91..6f204b2b0e 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -813,8 +813,7 @@ router_settings: | LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development) | LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers | LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60 -| LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY | If set to "true", disables automatic default reasoning summary injection (`summary: "detailed"`) for Anthropic experimental pass-through translations. Default is "false" -| LITELLM_REASONING_AUTO_SUMMARY | If set to "true", automatically enables detailed reasoning summaries for reasoning models (e.g., o1, o3-mini, deepseek-reasoner). When enabled, adds `summary: "detailed"` to reasoning effort configurations. Default is "false" +| LITELLM_REASONING_AUTO_SUMMARY | If set to "true", automatically enables detailed reasoning summaries (`summary: "detailed"`) for reasoning models across all translation paths (Anthropic adapter, Responses API, etc.). Default is "false" | LITELLM_SALT_KEY | Salt key for encryption in LiteLLM | LITELLM_SSL_CIPHERS | SSL/TLS cipher configuration for faster handshakes. Controls cipher suite preferences for OpenSSL connections. | LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM diff --git a/docs/my-website/docs/reasoning_content.md b/docs/my-website/docs/reasoning_content.md index f112120366..24694063ef 100644 --- a/docs/my-website/docs/reasoning_content.md +++ b/docs/my-website/docs/reasoning_content.md @@ -700,11 +700,11 @@ response = await litellm.anthropic.messages.acreate( # The summary="concise" is preserved when routing to OpenAI's Responses API ``` -### Default Summary Injection for `/v1/messages` Adapter +### Enabling Default Summary Injection for `/v1/messages` Adapter -When the Anthropic `/v1/messages` adapter translates `thinking` parameters to OpenAI `reasoning_effort` for non-Claude models, `summary="detailed"` is automatically injected by default. This ensures that reasoning text is returned in the response (matching the Anthropic thinking behavior). +When the Anthropic `/v1/messages` adapter translates `thinking` parameters to OpenAI `reasoning_effort` for non-Claude models, you can opt-in to automatic `summary="detailed"` injection using the `reasoning_auto_summary` flag. This ensures that reasoning text is returned in the response (matching the Anthropic thinking behavior). -To **disable** this default injection, use the `disable_default_reasoning_summary` flag: +To **enable** this default injection, use the `reasoning_auto_summary` flag: @@ -712,8 +712,8 @@ To **disable** this default injection, use the `disable_default_reasoning_summar ```python import litellm -# Disable default summary="detailed" injection -litellm.disable_default_reasoning_summary = True +# Enable default summary="detailed" injection +litellm.reasoning_auto_summary = True response = await litellm.anthropic.messages.acreate( model="openai/gpt-5.1", @@ -721,7 +721,7 @@ response = await litellm.anthropic.messages.acreate( max_tokens=8096, thinking={"type": "enabled", "budget_tokens": 5000}, ) -# No summary will be injected — only reasoning_effort is forwarded +# summary="detailed" will be automatically added to reasoning_effort ``` @@ -729,7 +729,7 @@ response = await litellm.anthropic.messages.acreate( ```bash -export LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY=true +export LITELLM_REASONING_AUTO_SUMMARY=true ``` @@ -738,7 +738,7 @@ export LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY=true ```yaml litellm_settings: - disable_default_reasoning_summary: true + reasoning_auto_summary: true ``` diff --git a/litellm/__init__.py b/litellm/__init__.py index a55d49690b..7f72e0b0e8 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -305,7 +305,6 @@ llm_guard_mode: Literal["all", "key-specific", "request-specific"] = "all" guardrail_name_config_map: Dict[str, GuardrailItem] = {} include_cost_in_streaming_usage: bool = False reasoning_auto_summary: bool = False -disable_default_reasoning_summary: bool = False ### PROMPTS #### from litellm.types.prompts.init_prompts import PromptSpec diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index c45d6b1858..897ca3bf89 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -12,12 +12,12 @@ from typing import ( ) import litellm -from litellm.llms.anthropic.experimental_pass_through.utils import ( - is_default_reasoning_summary_disabled, -) from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( AnthropicAdapter, ) +from litellm.llms.anthropic.experimental_pass_through.utils import ( + is_reasoning_auto_summary_enabled, +) from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) @@ -82,7 +82,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: # Prefix model with "responses/" to route to OpenAI Responses API completion_kwargs["model"] = f"responses/{model}" - summary_disabled = is_default_reasoning_summary_disabled() + auto_summary = is_reasoning_auto_summary_enabled() reasoning_effort = completion_kwargs.get("reasoning_effort") summary = thinking.get("summary") @@ -90,7 +90,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: reasoning_dict: Dict[str, Any] = {"effort": reasoning_effort} if summary: reasoning_dict["summary"] = summary - elif not summary_disabled: + elif auto_summary: reasoning_dict["summary"] = "detailed" completion_kwargs["reasoning_effort"] = reasoning_dict elif isinstance(reasoning_effort, dict): @@ -98,7 +98,9 @@ class LiteLLMMessagesToCompletionTransformationHandler: "summary" not in reasoning_effort and "generate_summary" not in reasoning_effort ): - effective_summary = summary if summary else ("detailed" if not summary_disabled else None) + effective_summary = ( + summary if summary else ("detailed" if auto_summary else None) + ) if effective_summary: updated_reasoning_effort = dict(reasoning_effort) updated_reasoning_effort["summary"] = effective_summary diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 23c8898622..ed49943b7f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -15,7 +15,7 @@ from typing import ( ) from litellm.llms.anthropic.experimental_pass_through.utils import ( - is_default_reasoning_summary_disabled, + is_reasoning_auto_summary_enabled, ) # OpenAI has a 64-character limit for function/tool names @@ -741,7 +741,7 @@ class LiteLLMAnthropicMessagesAdapter: summary = ( thinking.get("summary") if isinstance(thinking, dict) else None ) - summary_disabled = is_default_reasoning_summary_disabled() + auto_summary = is_reasoning_auto_summary_enabled() if summary: return { "reasoning_effort": { @@ -749,7 +749,7 @@ class LiteLLMAnthropicMessagesAdapter: "summary": summary, } } - elif not summary_disabled: + elif auto_summary: return { "reasoning_effort": { "effort": reasoning_effort, @@ -891,19 +891,25 @@ class LiteLLMAnthropicMessagesAdapter: # Handle array items if "items" in schema: - LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(schema["items"]) + LiteLLMAnthropicMessagesAdapter._add_additional_properties_false( + schema["items"] + ) # Handle anyOf/oneOf/allOf for key in ("anyOf", "oneOf", "allOf"): if key in schema: for sub_schema in schema[key]: - LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(sub_schema) + LiteLLMAnthropicMessagesAdapter._add_additional_properties_false( + sub_schema + ) # Handle $defs / definitions for key in ("$defs", "definitions"): if key in schema: for def_schema in schema[key].values(): - LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(def_schema) + LiteLLMAnthropicMessagesAdapter._add_additional_properties_false( + def_schema + ) def _add_system_message_to_messages( self, @@ -1030,6 +1036,7 @@ class LiteLLMAnthropicMessagesAdapter: return summary = thinking.get("summary") if isinstance(thinking, dict) else None + auto_summary = is_reasoning_auto_summary_enabled() if summary: new_kwargs["reasoning_effort"] = cast( Any, @@ -1038,6 +1045,14 @@ class LiteLLMAnthropicMessagesAdapter: "summary": summary, }, ) + elif auto_summary: + new_kwargs["reasoning_effort"] = cast( + Any, + { + "effort": reasoning_effort, + "summary": "detailed", + }, + ) else: new_kwargs["reasoning_effort"] = reasoning_effort diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 2fef5dee46..dae7044a5b 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -9,9 +9,8 @@ import json from typing import Any, Dict, List, Optional, Union, cast from litellm.llms.anthropic.experimental_pass_through.utils import ( - is_default_reasoning_summary_disabled, + is_reasoning_auto_summary_enabled, ) - from litellm.types.llms.anthropic import ( AllAnthropicToolsValues, AnthopicMessagesAssistantMessageParam, @@ -98,7 +97,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ) elif btype == "image": url = self._translate_anthropic_image_source_to_url( - block.get("source", {}) + cast(dict, block.get("source", {})) ) if url: user_parts.append( @@ -271,12 +270,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter: effort = "low" else: effort = "minimal" - summary_disabled = is_default_reasoning_summary_disabled() + auto_summary = is_reasoning_auto_summary_enabled() result: Dict[str, Any] = {"effort": effort} summary = thinking.get("summary") if summary: result["summary"] = summary - elif not summary_disabled: + elif auto_summary: result["summary"] = "detailed" return result diff --git a/litellm/llms/anthropic/experimental_pass_through/utils.py b/litellm/llms/anthropic/experimental_pass_through/utils.py index 4a40e4629e..6c1db6017b 100644 --- a/litellm/llms/anthropic/experimental_pass_through/utils.py +++ b/litellm/llms/anthropic/experimental_pass_through/utils.py @@ -3,10 +3,9 @@ import os import litellm -def is_default_reasoning_summary_disabled() -> bool: - """Check whether the default 'summary: detailed' injection should be suppressed.""" +def is_reasoning_auto_summary_enabled() -> bool: + """Check whether the default 'summary: detailed' injection is enabled (opt-in).""" return ( - litellm.disable_default_reasoning_summary - or os.getenv("LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY", "false").lower() - == "true" + litellm.reasoning_auto_summary + or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true" ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 879dd42be4..bbf9f6d9dc 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6152,7 +6152,8 @@ "max_query_tokens": 4096, "max_tokens": 32768, "mode": "rerank", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-cohere-rerank-4-0-in-microsoft-foundry/4477076" }, "azure_ai/cohere-rerank-v4.0-fast": { "input_cost_per_query": 0.002, @@ -6163,7 +6164,8 @@ "max_query_tokens": 4096, "max_tokens": 32768, "mode": "rerank", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-cohere-rerank-4-0-in-microsoft-foundry/4477076" }, "azure_ai/deepseek-v3.2": { "input_cost_per_token": 5.8e-07, @@ -6173,6 +6175,7 @@ "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.68e-06, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-deepseek-v3-2-and-deepseek-v3-2-speciale-in-microsoft-foundry/4477549", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -6187,6 +6190,7 @@ "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.68e-06, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-deepseek-v3-2-and-deepseek-v3-2-speciale-in-microsoft-foundry/4477549", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 203b6dacea..24080ca120 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -31,7 +31,7 @@ def test_anthropic_experimental_pass_through_messages_handler(): model="openai/claude-3-5-sonnet-20240620", api_key="test-api-key", ) - except Exception as e: + except (ValueError, TypeError, AttributeError) as e: print(f"Error: {e}") mock_responses.assert_called_once() assert mock_responses.call_args.kwargs["api_key"] == "test-api-key" @@ -56,7 +56,7 @@ def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_an api_base="test-api-base", custom_key="custom_value", ) - except Exception as e: + except (ValueError, TypeError, AttributeError) as e: print(f"Error: {e}") mock_completion.assert_called_once() assert mock_completion.call_args.kwargs["api_key"] == "test-api-key" @@ -81,7 +81,7 @@ def test_anthropic_experimental_pass_through_messages_handler_custom_llm_provide custom_llm_provider="my-custom-llm", api_key="test-api-key", ) - except Exception as e: + except (ValueError, TypeError, AttributeError) as e: print(f"Error: {e}") # Assert that litellm.completion was called when using a custom LLM provider @@ -125,24 +125,29 @@ async def test_bedrock_converse_budget_tokens_preserved(): max_tokens=1024, messages=[{"role": "user", "content": "What is 2+2?"}], model="bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0", - thinking={ - "budget_tokens": 1024, - "type": "enabled" - }, + thinking={"budget_tokens": 1024, "type": "enabled"}, ) - except Exception: + except (ValueError, TypeError, AttributeError): pass # Expected due to response format conversion mock_acompletion.assert_called_once() call_kwargs = mock_acompletion.call_args.kwargs - print("acompletion call kwargs: ", json.dumps(call_kwargs, indent=4, default=str)) + print( + "acompletion call kwargs: ", json.dumps(call_kwargs, indent=4, default=str) + ) # Verify thinking parameter is passed through with budget_tokens preserved thinking_param = call_kwargs.get("thinking") - assert thinking_param is not None, "thinking parameter should be passed to acompletion" - assert thinking_param.get("type") == "enabled", "thinking.type should be 'enabled'" - assert thinking_param.get("budget_tokens") == 1024, f"thinking.budget_tokens should be 1024, but got {thinking_param.get('budget_tokens')}" + assert ( + thinking_param is not None + ), "thinking parameter should be passed to acompletion" + assert ( + thinking_param.get("type") == "enabled" + ), "thinking.type should be 'enabled'" + assert ( + thinking_param.get("budget_tokens") == 1024 + ), f"thinking.budget_tokens should be 1024, but got {thinking_param.get('budget_tokens')}" def test_openai_model_with_thinking_converts_to_reasoning(): @@ -164,12 +169,9 @@ def test_openai_model_with_thinking_converts_to_reasoning(): messages=[{"role": "user", "content": "What is 2+2?"}], model="openai/gpt-5.2", api_key="test-api-key", - thinking={ - "type": "enabled", - "budget_tokens": 1024 - }, + thinking={"type": "enabled", "budget_tokens": 1024}, ) - except Exception as e: + except (ValueError, TypeError, AttributeError) as e: print(f"Error: {e}") mock_responses.assert_called_once() @@ -177,18 +179,22 @@ def test_openai_model_with_thinking_converts_to_reasoning(): call_kwargs = mock_responses.call_args.kwargs # Verify reasoning is set (converted from thinking) - assert "reasoning" in call_kwargs, "reasoning should be passed to litellm.responses" + assert ( + "reasoning" in call_kwargs + ), "reasoning should be passed to litellm.responses" # budget_tokens=1024 -> effort="minimal" (< 2000 threshold) - # summary="detailed" added by default unless disable_default_reasoning_summary is set - expected_reasoning = {"effort": "minimal", "summary": "detailed"} + # reasoning_auto_summary is False by default, so no summary key + expected_reasoning = {"effort": "minimal"} assert call_kwargs["reasoning"] == expected_reasoning, ( f"reasoning should be {expected_reasoning} for budget_tokens=1024, " f"got {call_kwargs.get('reasoning')}" ) # Verify thinking is NOT passed directly to the Responses API - assert "thinking" not in call_kwargs, "thinking should NOT be passed directly to litellm.responses" + assert ( + "thinking" not in call_kwargs + ), "thinking should NOT be passed directly to litellm.responses" class TestThinkingParameterTransformation: @@ -199,13 +205,13 @@ class TestThinkingParameterTransformation: from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) - + thinking = {"type": "enabled", "budget_tokens": 5000} result = LiteLLMAnthropicMessagesAdapter.translate_thinking_for_model( thinking=thinking, model="bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0", ) - + assert result == {"thinking": thinking} assert result["thinking"]["budget_tokens"] == 5000 @@ -221,27 +227,30 @@ class TestThinkingParameterTransformation: model="openai/gpt-5.2", ) - assert result == {"reasoning_effort": {"effort": "minimal", "summary": "detailed"}} + # reasoning_auto_summary is False by default, so no summary key + assert result == {"reasoning_effort": "minimal"} assert "thinking" not in result - def test_translate_thinking_for_model_no_summary_when_disabled(self): - """When disable_default_reasoning_summary is True, no summary is injected.""" + def test_translate_thinking_for_model_summary_when_enabled(self): + """When reasoning_auto_summary is True, summary='detailed' is injected.""" import litellm from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) - original = litellm.disable_default_reasoning_summary + original = litellm.reasoning_auto_summary try: - litellm.disable_default_reasoning_summary = True + litellm.reasoning_auto_summary = True thinking = {"type": "enabled", "budget_tokens": 5000} result = LiteLLMAnthropicMessagesAdapter.translate_thinking_for_model( thinking=thinking, model="openai/gpt-5.2", ) - assert result == {"reasoning_effort": "medium"} + assert result == { + "reasoning_effort": {"effort": "medium", "summary": "detailed"} + } finally: - litellm.disable_default_reasoning_summary = original + litellm.reasoning_auto_summary = original def test_translate_thinking_for_model_preserves_user_summary(self): """User-provided summary is always preserved regardless of flag.""" @@ -258,7 +267,7 @@ class TestThinkingParameterTransformation: class TestThinkingSummaryPreservation: - """Tests for thinking.summary preservation and disable_default_reasoning_summary flag.""" + """Tests for thinking.summary preservation and reasoning_auto_summary flag.""" def test_thinking_summary_concise_preserved_for_openai(self): """User-provided summary='concise' should not be replaced with 'detailed'.""" @@ -271,7 +280,10 @@ class TestThinkingSummaryPreservation: LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( completion_kwargs, thinking=thinking ) - assert completion_kwargs["reasoning_effort"] == {"effort": "medium", "summary": "concise"} + assert completion_kwargs["reasoning_effort"] == { + "effort": "medium", + "summary": "concise", + } def test_thinking_summary_auto_preserved_for_openai(self): """User-provided summary='auto' should be preserved.""" @@ -284,18 +296,21 @@ class TestThinkingSummaryPreservation: LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( completion_kwargs, thinking=thinking ) - assert completion_kwargs["reasoning_effort"] == {"effort": "high", "summary": "auto"} + assert completion_kwargs["reasoning_effort"] == { + "effort": "high", + "summary": "auto", + } - def test_summary_added_by_default_when_no_user_summary(self): - """When no user summary and flag is off, summary='detailed' is added by default.""" + def test_summary_added_when_auto_summary_enabled(self): + """When reasoning_auto_summary is True, summary='detailed' is added.""" import litellm from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) - original = litellm.disable_default_reasoning_summary + original = litellm.reasoning_auto_summary try: - litellm.disable_default_reasoning_summary = False + litellm.reasoning_auto_summary = True completion_kwargs = { "model": "responses/gpt-5.2", "custom_llm_provider": "openai", @@ -304,20 +319,23 @@ class TestThinkingSummaryPreservation: LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( completion_kwargs, thinking={"type": "enabled", "budget_tokens": 5000} ) - assert completion_kwargs["reasoning_effort"] == {"effort": "medium", "summary": "detailed"} + assert completion_kwargs["reasoning_effort"] == { + "effort": "medium", + "summary": "detailed", + } finally: - litellm.disable_default_reasoning_summary = original + litellm.reasoning_auto_summary = original - def test_summary_excluded_when_disable_flag_set_string_reasoning(self): - """When disable_default_reasoning_summary is True, summary is not added for string reasoning_effort.""" + def test_no_summary_by_default_string_reasoning(self): + """By default (reasoning_auto_summary=False), summary is not added for string reasoning_effort.""" import litellm from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) - original = litellm.disable_default_reasoning_summary + original = litellm.reasoning_auto_summary try: - litellm.disable_default_reasoning_summary = True + litellm.reasoning_auto_summary = False completion_kwargs = { "model": "responses/gpt-5.2", "custom_llm_provider": "openai", @@ -329,18 +347,18 @@ class TestThinkingSummaryPreservation: assert completion_kwargs["reasoning_effort"] == {"effort": "high"} assert "summary" not in completion_kwargs["reasoning_effort"] finally: - litellm.disable_default_reasoning_summary = original + litellm.reasoning_auto_summary = original - def test_summary_excluded_when_disable_flag_set_dict_reasoning(self): - """When disable_default_reasoning_summary is True, summary is not injected into dict reasoning_effort.""" + def test_no_summary_by_default_dict_reasoning(self): + """By default (reasoning_auto_summary=False), summary is not injected into dict reasoning_effort.""" import litellm from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) - original = litellm.disable_default_reasoning_summary + original = litellm.reasoning_auto_summary try: - litellm.disable_default_reasoning_summary = True + litellm.reasoning_auto_summary = False completion_kwargs = { "model": "responses/gpt-5.2", "custom_llm_provider": "openai", @@ -352,19 +370,19 @@ class TestThinkingSummaryPreservation: assert completion_kwargs["reasoning_effort"] == {"effort": "medium"} assert "summary" not in completion_kwargs["reasoning_effort"] finally: - litellm.disable_default_reasoning_summary = original + litellm.reasoning_auto_summary = original - def test_summary_excluded_when_env_var_set(self): - """When LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY env var is true, summary is not added.""" + def test_summary_added_when_env_var_set(self): + """When LITELLM_REASONING_AUTO_SUMMARY env var is true, summary is added.""" import litellm from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) - original = litellm.disable_default_reasoning_summary + original = litellm.reasoning_auto_summary try: - litellm.disable_default_reasoning_summary = False - os.environ["LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY"] = "true" + litellm.reasoning_auto_summary = False + os.environ["LITELLM_REASONING_AUTO_SUMMARY"] = "true" completion_kwargs = { "model": "responses/gpt-5.2", "custom_llm_provider": "openai", @@ -373,11 +391,13 @@ class TestThinkingSummaryPreservation: LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed( completion_kwargs, thinking={"type": "enabled", "budget_tokens": 10000} ) - assert completion_kwargs["reasoning_effort"] == {"effort": "high"} - assert "summary" not in completion_kwargs["reasoning_effort"] + assert completion_kwargs["reasoning_effort"] == { + "effort": "high", + "summary": "detailed", + } finally: - litellm.disable_default_reasoning_summary = original - os.environ.pop("LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY", None) + litellm.reasoning_auto_summary = original + os.environ.pop("LITELLM_REASONING_AUTO_SUMMARY", None) def test_user_provided_summary_preserved_even_when_flag_off(self): """When user already set summary in dict reasoning_effort, it's preserved regardless of flag.""" @@ -386,9 +406,9 @@ class TestThinkingSummaryPreservation: LiteLLMMessagesToCompletionTransformationHandler, ) - original = litellm.disable_default_reasoning_summary + original = litellm.reasoning_auto_summary try: - litellm.disable_default_reasoning_summary = False + litellm.reasoning_auto_summary = False completion_kwargs = { "model": "responses/gpt-5.2", "custom_llm_provider": "openai", @@ -399,7 +419,7 @@ class TestThinkingSummaryPreservation: ) assert completion_kwargs["reasoning_effort"]["summary"] == "concise" finally: - litellm.disable_default_reasoning_summary = original + litellm.reasoning_auto_summary = original def test_openai_model_with_thinking_summary_end_to_end(self): """End-to-end: anthropic_messages_handler should preserve thinking.summary for OpenAI models.""" @@ -420,14 +440,15 @@ class TestThinkingSummaryPreservation: "summary": "concise", }, ) - except Exception: + except (ValueError, TypeError, AttributeError): pass mock_responses.assert_called_once() call_kwargs = mock_responses.call_args.kwargs reasoning = call_kwargs["reasoning"] - assert reasoning["summary"] == "concise", \ - f"Expected summary='concise', got summary='{reasoning.get('summary')}'" + assert ( + reasoning["summary"] == "concise" + ), f"Expected summary='concise', got summary='{reasoning.get('summary')}'" def test_responses_adapter_preserves_summary(self): """translate_thinking_to_reasoning should include summary when user provides it.""" @@ -436,25 +457,31 @@ class TestThinkingSummaryPreservation: ) thinking = {"type": "enabled", "budget_tokens": 5000, "summary": "concise"} - result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning(thinking) + result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning( + thinking + ) assert result == {"effort": "medium", "summary": "concise"} - def test_responses_adapter_no_summary_when_disabled(self): - """translate_thinking_to_reasoning should not include summary when flag is set and no user summary.""" + def test_responses_adapter_no_summary_by_default(self): + """translate_thinking_to_reasoning should not include summary by default (opt-in).""" import litellm from litellm.llms.anthropic.experimental_pass_through.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) - original = litellm.disable_default_reasoning_summary + original = litellm.reasoning_auto_summary try: - litellm.disable_default_reasoning_summary = True + litellm.reasoning_auto_summary = False thinking = {"type": "enabled", "budget_tokens": 5000} - result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning(thinking) + result = ( + LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning( + thinking + ) + ) assert result == {"effort": "medium"} assert "summary" not in result finally: - litellm.disable_default_reasoning_summary = original + litellm.reasoning_auto_summary = original def test_translate_thinking_for_model_preserves_summary(self): """translate_thinking_for_model should include summary in reasoning_effort dict when user provides it.""" @@ -467,4 +494,6 @@ class TestThinkingSummaryPreservation: thinking=thinking, model="openai/gpt-5.2", ) - assert result == {"reasoning_effort": {"effort": "medium", "summary": "concise"}} + assert result == { + "reasoning_effort": {"effort": "medium", "summary": "concise"} + } diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 7f77394552..6224652d9d 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -170,6 +170,7 @@ class TestOutputConfigStructuredOutput: # translate_messages_to_responses_input # --------------------------------------------------------------------------- + # Helper: cast plain dicts to the expected type so call sites stay clean. def _translate_messages(messages: List[Any]) -> List[Dict[str, Any]]: return _ADAPTER.translate_messages_to_responses_input(messages) # type: ignore[arg-type] @@ -274,7 +275,11 @@ class TestTranslateMessagesToResponsesInput: "content": [ { "type": "image", - "source": {"type": "base64", "media_type": "image/jpeg", "data": ""}, + "source": { + "type": "base64", + "media_type": "image/jpeg", + "data": "", + }, } ], } @@ -462,7 +467,10 @@ class TestTranslateMessagesToResponsesInput: ] result = _translate_messages(messages) assert len(result) == 1 - assert result[0]["content"][0] == {"type": "input_text", "text": "Describe this image:"} + assert result[0]["content"][0] == { + "type": "input_text", + "text": "Describe this image:", + } assert result[0]["content"][1] == { "type": "input_image", "image_url": "https://example.com/cat.jpg", @@ -606,7 +614,7 @@ class TestTranslateThinkingToReasoning: result = _ADAPTER.translate_thinking_to_reasoning( {"type": "enabled", "budget_tokens": 10000} ) - assert result == {"effort": "high", "summary": "detailed"} + assert result == {"effort": "high"} def test_budget_above_threshold_high_effort(self): result = _ADAPTER.translate_thinking_to_reasoning( @@ -619,19 +627,19 @@ class TestTranslateThinkingToReasoning: result = _ADAPTER.translate_thinking_to_reasoning( {"type": "enabled", "budget_tokens": 7500} ) - assert result == {"effort": "medium", "summary": "detailed"} + assert result == {"effort": "medium"} def test_budget_low_effort(self): result = _ADAPTER.translate_thinking_to_reasoning( {"type": "enabled", "budget_tokens": 3000} ) - assert result == {"effort": "low", "summary": "detailed"} + assert result == {"effort": "low"} def test_budget_minimal_effort(self): result = _ADAPTER.translate_thinking_to_reasoning( {"type": "enabled", "budget_tokens": 500} ) - assert result == {"effort": "minimal", "summary": "detailed"} + assert result == {"effort": "minimal"} def test_budget_at_exact_thresholds(self): result_medium = _ADAPTER.translate_thinking_to_reasoning( @@ -656,39 +664,37 @@ class TestTranslateThinkingToReasoning: def test_missing_budget_defaults_to_minimal(self): """Missing budget_tokens defaults to 0, which is < 2000 -> minimal.""" result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled"}) - assert result == {"effort": "minimal", "summary": "detailed"} + assert result == {"effort": "minimal"} - def test_summary_excluded_when_disable_flag_set(self): - """When disable_default_reasoning_summary is True, summary is not included.""" + def test_summary_added_when_auto_summary_enabled(self): + """When reasoning_auto_summary is True, summary='detailed' is included.""" import litellm - original = litellm.disable_default_reasoning_summary + original = litellm.reasoning_auto_summary try: - litellm.disable_default_reasoning_summary = True + litellm.reasoning_auto_summary = True result = _ADAPTER.translate_thinking_to_reasoning( {"type": "enabled", "budget_tokens": 10000} ) - assert result == {"effort": "high"} - assert "summary" not in result + assert result == {"effort": "high", "summary": "detailed"} finally: - litellm.disable_default_reasoning_summary = original + litellm.reasoning_auto_summary = original - def test_summary_excluded_when_env_var_set(self): - """When LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY env var is true, summary is not included.""" + def test_summary_added_when_env_var_set(self): + """When LITELLM_REASONING_AUTO_SUMMARY env var is true, summary is included.""" import litellm - original = litellm.disable_default_reasoning_summary + original = litellm.reasoning_auto_summary try: - litellm.disable_default_reasoning_summary = False - os.environ["LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY"] = "true" + litellm.reasoning_auto_summary = False + os.environ["LITELLM_REASONING_AUTO_SUMMARY"] = "true" result = _ADAPTER.translate_thinking_to_reasoning( {"type": "enabled", "budget_tokens": 5000} ) - assert result == {"effort": "medium"} - assert "summary" not in result + assert result == {"effort": "medium", "summary": "detailed"} finally: - litellm.disable_default_reasoning_summary = original - os.environ.pop("LITELLM_DISABLE_DEFAULT_REASONING_SUMMARY", None) + litellm.reasoning_auto_summary = original + os.environ.pop("LITELLM_REASONING_AUTO_SUMMARY", None) # --------------------------------------------------------------------------- @@ -747,7 +753,9 @@ class TestTranslateRequestBroaderCoverage: def test_tools_translated(self): req = _make_request( - tools=[{"name": "calculator", "description": "Does math.", "input_schema": {}}] + tools=[ + {"name": "calculator", "description": "Does math.", "input_schema": {}} + ] ) kwargs = _ADAPTER.translate_request(req) assert len(kwargs["tools"]) == 1 @@ -764,7 +772,8 @@ class TestTranslateRequestBroaderCoverage: def test_thinking_translated_to_reasoning(self): req = _make_request(thinking={"type": "enabled", "budget_tokens": 12000}) kwargs = _ADAPTER.translate_request(req) - assert kwargs["reasoning"] == {"effort": "high", "summary": "detailed"} + # reasoning_auto_summary is False by default, so no summary key + assert kwargs["reasoning"] == {"effort": "high"} def test_disabled_thinking_not_included_in_kwargs(self): req = _make_request(thinking={"type": "disabled"}) @@ -785,8 +794,17 @@ class TestTranslateRequestBroaderCoverage: def test_no_optional_fields_does_not_add_spurious_keys(self): req = _make_request() kwargs = _ADAPTER.translate_request(req) - for key in ("instructions", "temperature", "top_p", "tools", "tool_choice", - "reasoning", "text", "context_management", "user"): + for key in ( + "instructions", + "temperature", + "top_p", + "tools", + "tool_choice", + "reasoning", + "text", + "context_management", + "user", + ): assert key not in kwargs, f"unexpected key: {key}" @@ -833,9 +851,7 @@ def _make_output_message(texts: List[str]) -> MagicMock: return msg -def _make_function_call_item( - call_id: str, name: str, arguments: str -) -> MagicMock: +def _make_function_call_item(call_id: str, name: str, arguments: str) -> MagicMock: """Build a mock ResponseFunctionToolCall.""" from openai.types.responses import ResponseFunctionToolCall # type: ignore[import] From 35316e115f68f65772ec97159e3c7022ec841f85 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 21 Mar 2026 14:51:15 -0700 Subject: [PATCH 13/15] fix: apply Black formatting to 7 files Co-Authored-By: Claude Opus 4.6 (1M context) --- litellm/litellm_core_utils/litellm_logging.py | 10 ++++---- .../gemini/image_generation/transformation.py | 1 - litellm/llms/moonshot/chat/transformation.py | 4 +++- litellm/llms/ovhcloud/chat/transformation.py | 4 +--- .../llms/vertex_ai/gemini/transformation.py | 12 ++++++---- litellm/proxy/auth/user_api_key_auth.py | 12 +++++----- litellm/proxy/common_request_processing.py | 24 ++++++++++++------- 7 files changed, 38 insertions(+), 29 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 56f7f305dc..5323f692b8 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1686,7 +1686,9 @@ class Logging(LiteLLMLoggingBaseClass): ) return logging_result - def _merge_hidden_params_from_response_into_metadata(self, logging_result: Any) -> None: + def _merge_hidden_params_from_response_into_metadata( + self, logging_result: Any + ) -> None: """ Copy response._hidden_params into litellm_params.metadata['hidden_params']. @@ -1704,9 +1706,9 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["litellm_params"].setdefault("metadata", {}) if self.model_call_details["litellm_params"]["metadata"] is None: self.model_call_details["litellm_params"]["metadata"] = {} - self.model_call_details["litellm_params"]["metadata"]["hidden_params"] = getattr( - logging_result, "_hidden_params", {} - ) + self.model_call_details["litellm_params"]["metadata"][ + "hidden_params" + ] = getattr(logging_result, "_hidden_params", {}) def _process_hidden_params_and_response_cost( self, diff --git a/litellm/llms/gemini/image_generation/transformation.py b/litellm/llms/gemini/image_generation/transformation.py index 795ad275a6..b094fc133d 100644 --- a/litellm/llms/gemini/image_generation/transformation.py +++ b/litellm/llms/gemini/image_generation/transformation.py @@ -98,7 +98,6 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): elif modality == "IMAGE": input_tokens_details.image_tokens += token_count - return ImageUsage( input_tokens=usage_metadata.get("promptTokenCount", 0), input_tokens_details=input_tokens_details, diff --git a/litellm/llms/moonshot/chat/transformation.py b/litellm/llms/moonshot/chat/transformation.py index c97bd6c4e1..e4d7b5f033 100644 --- a/litellm/llms/moonshot/chat/transformation.py +++ b/litellm/llms/moonshot/chat/transformation.py @@ -168,7 +168,9 @@ class MoonshotChatConfig(OpenAIGPTConfig): if ( msg.get("role") == "assistant" and msg.get("tool_calls") - and not msg.get("reasoning_content") # Check using .get() which works for both dicts and Pydantic models + and not msg.get( + "reasoning_content" + ) # Check using .get() which works for both dicts and Pydantic models ): patched = dict(cast(dict, msg)) provider_fields = patched.get("provider_specific_fields") or {} diff --git a/litellm/llms/ovhcloud/chat/transformation.py b/litellm/llms/ovhcloud/chat/transformation.py index 84090fafd3..1416b782f1 100644 --- a/litellm/llms/ovhcloud/chat/transformation.py +++ b/litellm/llms/ovhcloud/chat/transformation.py @@ -28,9 +28,7 @@ class OVHCloudChatConfig(OpenAIGPTConfig): """ supports_function_calling: Optional[bool] = None try: - model_info = _get_model_info_helper( - model, custom_llm_provider="ovhcloud" - ) + model_info = _get_model_info_helper(model, custom_llm_provider="ovhcloud") supports_function_calling = model_info.get( "supports_function_calling", None ) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index f6310778c7..7945c44d44 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -555,7 +555,9 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 } } if "thought_signature" in invocation: - tc_part["thoughtSignature"] = invocation["thought_signature"] + tc_part["thoughtSignature"] = invocation[ + "thought_signature" + ] assistant_content.append(tc_part) # type: ignore # Re-inject toolResponse part if response is present @@ -566,11 +568,11 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 } if invocation.get("tool_type"): tr_dict["toolType"] = invocation["tool_type"] - tr_part: Dict[str, Any] = { - "toolResponse": tr_dict - } + tr_part: Dict[str, Any] = {"toolResponse": tr_dict} if "thought_signature" in invocation: - tr_part["thoughtSignature"] = invocation["thought_signature"] + tr_part["thoughtSignature"] = invocation[ + "thought_signature" + ] assistant_content.append(tr_part) # type: ignore msg_i += 1 diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index e2f06abc52..800cca21db 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -920,9 +920,9 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 route=route, ) if _end_user_object is not None: - end_user_params["allowed_model_region"] = ( - _end_user_object.allowed_model_region - ) + end_user_params[ + "allowed_model_region" + ] = _end_user_object.allowed_model_region if _end_user_object.litellm_budget_table is not None: _apply_budget_limits_to_end_user_params( end_user_params=end_user_params, @@ -1499,9 +1499,9 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 if _end_user_object is not None: valid_token_dict.update(end_user_params) - valid_token_dict["end_user_object_permission"] = ( - _end_user_object.object_permission - ) + valid_token_dict[ + "end_user_object_permission" + ] = _end_user_object.object_permission # check if token is from litellm-ui, litellm ui makes keys to allow users to login with sso. These keys can only be used for LiteLLM UI functions # sso/login, ui/login, /key functions and /user functions diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index bad70d30da..8147d17c10 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1004,7 +1004,8 @@ class ProxyBaseLLMRequestProcessing: self.data["deployment"] = llm_router.get_deployment(model_id=model_id) asyncio.create_task( proxy_logging_obj.update_request_status( - litellm_call_id=self.data.get("litellm_call_id", ""), status="success" + litellm_call_id=self.data.get("litellm_call_id", ""), + status="success", ) ) if self._is_streaming_request( @@ -1029,11 +1030,13 @@ class ProxyBaseLLMRequestProcessing: ) # Call response headers hook for streaming success - callback_headers = await proxy_logging_obj.post_call_response_headers_hook( - data=self.data, - user_api_key_dict=user_api_key_dict, - response=response, - request_headers=dict(request.headers), + callback_headers = ( + await proxy_logging_obj.post_call_response_headers_hook( + data=self.data, + user_api_key_dict=user_api_key_dict, + response=response, + request_headers=dict(request.headers), + ) ) if callback_headers: custom_headers.update(callback_headers) @@ -1439,7 +1442,8 @@ class ProxyBaseLLMRequestProcessing: )["guardrail_blocked"] = True except Exception as e: verbose_proxy_logger.exception( - "Error in deferred streaming guardrail initialization: %s", e, + "Error in deferred streaming guardrail initialization: %s", + e, ) finally: try: @@ -1453,7 +1457,8 @@ class ProxyBaseLLMRequestProcessing: ) except Exception as e: verbose_proxy_logger.exception( - "Error in deferred streaming async logging: %s", e, + "Error in deferred streaming async logging: %s", + e, ) try: @@ -1466,7 +1471,8 @@ class ProxyBaseLLMRequestProcessing: ) except Exception as e: verbose_proxy_logger.exception( - "Error in deferred streaming sync logging: %s", e, + "Error in deferred streaming sync logging: %s", + e, ) async def _handle_llm_api_exception( From cb4027531b1c715228db5f29dad5d61967f81dbf Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 21 Mar 2026 14:53:16 -0700 Subject: [PATCH 14/15] fix: add explicit "summary" not in result guards to opt-out test paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses Greptile feedback that test assertions were weakened when removing summary: "detailed" expectations — now every default-behavior test explicitly asserts that "summary" is absent from the result. Co-Authored-By: Claude Opus 4.6 --- ...ropic_experimental_pass_through_messages_handler.py | 4 +++- .../test_responses_adapters_transformation.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 24080ca120..33628e1d19 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -190,6 +190,7 @@ def test_openai_model_with_thinking_converts_to_reasoning(): f"reasoning should be {expected_reasoning} for budget_tokens=1024, " f"got {call_kwargs.get('reasoning')}" ) + assert "summary" not in call_kwargs["reasoning"] # Verify thinking is NOT passed directly to the Responses API assert ( @@ -230,6 +231,7 @@ class TestThinkingParameterTransformation: # reasoning_auto_summary is False by default, so no summary key assert result == {"reasoning_effort": "minimal"} assert "thinking" not in result + assert "summary" not in str(result["reasoning_effort"]) def test_translate_thinking_for_model_summary_when_enabled(self): """When reasoning_auto_summary is True, summary='detailed' is injected.""" @@ -479,7 +481,7 @@ class TestThinkingSummaryPreservation: ) ) assert result == {"effort": "medium"} - assert "summary" not in result + assert result is not None and "summary" not in result finally: litellm.reasoning_auto_summary = original diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 6224652d9d..02b817cd33 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -614,7 +614,9 @@ class TestTranslateThinkingToReasoning: result = _ADAPTER.translate_thinking_to_reasoning( {"type": "enabled", "budget_tokens": 10000} ) + # Default (reasoning_auto_summary=False): only effort, no summary assert result == {"effort": "high"} + assert result is not None and "summary" not in result def test_budget_above_threshold_high_effort(self): result = _ADAPTER.translate_thinking_to_reasoning( @@ -622,24 +624,28 @@ class TestTranslateThinkingToReasoning: ) assert result is not None assert result["effort"] == "high" + assert "summary" not in result def test_budget_medium_effort(self): result = _ADAPTER.translate_thinking_to_reasoning( {"type": "enabled", "budget_tokens": 7500} ) assert result == {"effort": "medium"} + assert result is not None and "summary" not in result def test_budget_low_effort(self): result = _ADAPTER.translate_thinking_to_reasoning( {"type": "enabled", "budget_tokens": 3000} ) assert result == {"effort": "low"} + assert result is not None and "summary" not in result def test_budget_minimal_effort(self): result = _ADAPTER.translate_thinking_to_reasoning( {"type": "enabled", "budget_tokens": 500} ) assert result == {"effort": "minimal"} + assert result is not None and "summary" not in result def test_budget_at_exact_thresholds(self): result_medium = _ADAPTER.translate_thinking_to_reasoning( @@ -647,11 +653,13 @@ class TestTranslateThinkingToReasoning: ) assert result_medium is not None assert result_medium["effort"] == "medium" + assert "summary" not in result_medium result_low = _ADAPTER.translate_thinking_to_reasoning( {"type": "enabled", "budget_tokens": 2000} ) assert result_low is not None assert result_low["effort"] == "low" + assert "summary" not in result_low def test_disabled_type_returns_none(self): result = _ADAPTER.translate_thinking_to_reasoning({"type": "disabled"}) @@ -665,6 +673,7 @@ class TestTranslateThinkingToReasoning: """Missing budget_tokens defaults to 0, which is < 2000 -> minimal.""" result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled"}) assert result == {"effort": "minimal"} + assert result is not None and "summary" not in result def test_summary_added_when_auto_summary_enabled(self): """When reasoning_auto_summary is True, summary='detailed' is included.""" @@ -774,6 +783,7 @@ class TestTranslateRequestBroaderCoverage: kwargs = _ADAPTER.translate_request(req) # reasoning_auto_summary is False by default, so no summary key assert kwargs["reasoning"] == {"effort": "high"} + assert "summary" not in kwargs["reasoning"] def test_disabled_thinking_not_included_in_kwargs(self): req = _make_request(thinking={"type": "disabled"}) From e3b62c091541e7a3df5ef38bf0094269fd6be414 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 21 Mar 2026 15:03:06 -0700 Subject: [PATCH 15/15] fix: apply Black formatting to 6 files after main merge Co-Authored-By: Claude Opus 4.6 (1M context) --- .../transformation.py | 20 +++--- litellm/integrations/langsmith.py | 4 +- .../count_tokens/handler.py | 8 ++- .../guardrail_hooks/akto/__init__.py | 4 +- .../guardrails/guardrail_hooks/akto/akto.py | 68 ++++++++++++++----- litellm/proxy/proxy_server.py | 62 +++++++++-------- 6 files changed, 105 insertions(+), 61 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index f5856ab1f4..ee4cdbcdf3 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -240,10 +240,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if key in ("max_tokens", "max_completion_tokens"): responses_api_request["max_output_tokens"] = value elif key == "tools" and value is not None: - responses_api_request["tools"] = ( - self._convert_tools_to_responses_format( - cast(List[Dict[str, Any]], value) - ) + responses_api_request[ + "tools" + ] = self._convert_tools_to_responses_format( + cast(List[Dict[str, Any]], value) ) elif key == "response_format": text_format = self._transform_response_format_to_text_format(value) @@ -1072,9 +1072,9 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): ) if provider_specific_fields: - function_chunk["provider_specific_fields"] = ( - provider_specific_fields - ) + function_chunk[ + "provider_specific_fields" + ] = provider_specific_fields tool_call_index = parsed_chunk.get("output_index", 0) tool_call_chunk = ChatCompletionToolCallChunk( @@ -1147,9 +1147,9 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): # Add provider_specific_fields to function if present if provider_specific_fields: - function_chunk["provider_specific_fields"] = ( - provider_specific_fields - ) + function_chunk[ + "provider_specific_fields" + ] = provider_specific_fields tool_call_index = parsed_chunk.get("output_index", 0) tool_call_chunk = ChatCompletionToolCallChunk( diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index ad4bee0834..b931d7ecfe 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -83,7 +83,9 @@ class LangsmithLogger(CustomBatchLogger): if _batch_size: self.batch_size = int(_batch_size) self.log_queue: List[LangsmithQueueObject] = [] - self._flush_task: Optional[asyncio.Task[Any]] = self._start_periodic_flush_task() + self._flush_task: Optional[ + asyncio.Task[Any] + ] = self._start_periodic_flush_task() def _start_periodic_flush_task(self) -> Optional[asyncio.Task[Any]]: """Start the periodic flush task only when an event loop is already running.""" diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index 82076ff360..5d94cd4212 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -105,11 +105,13 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): # Extract Vertex AI credentials and settings vertex_credentials = self.get_vertex_ai_credentials(litellm_params) vertex_project = self.get_vertex_ai_project(litellm_params) - + # Check for count_tokens specific location override - vertex_count_tokens_location = litellm_params.get("vertex_count_tokens_location") + vertex_count_tokens_location = litellm_params.get( + "vertex_count_tokens_location" + ) vertex_location_raw = self.get_vertex_ai_location(litellm_params) - + # Determine final location with precedence: # 1. vertex_count_tokens_location (if provided) # 2. vertex_location (if provided) diff --git a/litellm/proxy/guardrails/guardrail_hooks/akto/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/akto/__init__.py index 4ae2675540..c4aaea709b 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/akto/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/akto/__init__.py @@ -17,7 +17,9 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" akto_api_key=getattr(litellm_params, "akto_api_key", None), akto_account_id=getattr(litellm_params, "akto_account_id", None), akto_vxlan_id=getattr(litellm_params, "akto_vxlan_id", None), - unreachable_fallback=getattr(litellm_params, "unreachable_fallback", "fail_closed"), + unreachable_fallback=getattr( + litellm_params, "unreachable_fallback", "fail_closed" + ), guardrail_timeout=getattr(litellm_params, "guardrail_timeout", None), guardrail_name=guardrail.get("guardrail_name", ""), event_hook=litellm_params.mode, diff --git a/litellm/proxy/guardrails/guardrail_hooks/akto/akto.py b/litellm/proxy/guardrails/guardrail_hooks/akto/akto.py index be9c9cb1be..5058ee348d 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/akto/akto.py +++ b/litellm/proxy/guardrails/guardrail_hooks/akto/akto.py @@ -77,17 +77,27 @@ class AktoGuardrail(CustomGuardrail): ) self.background_tasks: set = set() - self.akto_base_url = (akto_base_url or os.environ.get("AKTO_GUARDRAIL_API_BASE", "")).rstrip("/") + self.akto_base_url = ( + akto_base_url or os.environ.get("AKTO_GUARDRAIL_API_BASE", "") + ).rstrip("/") if not self.akto_base_url: - raise ValueError("akto_base_url is required. Set AKTO_GUARDRAIL_API_BASE or pass it in litellm_params.") + raise ValueError( + "akto_base_url is required. Set AKTO_GUARDRAIL_API_BASE or pass it in litellm_params." + ) self.akto_api_key = akto_api_key or os.environ.get("AKTO_API_KEY", "") if not self.akto_api_key: - raise ValueError("akto_api_key is required. Set AKTO_API_KEY or pass it in litellm_params.") + raise ValueError( + "akto_api_key is required. Set AKTO_API_KEY or pass it in litellm_params." + ) - self.unreachable_fallback: Literal["fail_closed", "fail_open"] = unreachable_fallback + self.unreachable_fallback: Literal[ + "fail_closed", "fail_open" + ] = unreachable_fallback self.guardrail_timeout = guardrail_timeout or DEFAULT_GUARDRAIL_TIMEOUT - self.akto_account_id = akto_account_id or os.environ.get("AKTO_ACCOUNT_ID", "1000000") + self.akto_account_id = akto_account_id or os.environ.get( + "AKTO_ACCOUNT_ID", "1000000" + ) self.akto_vxlan_id = akto_vxlan_id or os.environ.get("AKTO_VXLAN_ID", "0") kwargs["supported_event_hooks"] = [ @@ -173,7 +183,9 @@ class AktoGuardrail(CustomGuardrail): body["model"] = request_data["model"] else: texts = inputs.get("texts", []) - body["messages"] = [{"role": "user", "content": t} for t in texts] if texts else [] + body["messages"] = ( + [{"role": "user", "content": t} for t in texts] if texts else [] + ) tools = inputs.get("tools") if tools: @@ -199,15 +211,23 @@ class AktoGuardrail(CustomGuardrail): texts = inputs.get("texts", []) if texts: - return {"choices": [{"message": {"content": t, "role": "assistant"}} for t in texts]} + return { + "choices": [ + {"message": {"content": t, "role": "assistant"}} for t in texts + ] + } return {} @staticmethod def build_tag_metadata(request_data: dict) -> Dict[str, str]: """Build tag/metadata dict with user_id and team_id for Akto tracking.""" tag: Dict[str, str] = {"gen-ai": "Gen AI"} - user_id = AktoGuardrail.resolve_metadata_value(request_data, "user_api_key_user_id") - team_id = AktoGuardrail.resolve_metadata_value(request_data, "user_api_key_team_id") + user_id = AktoGuardrail.resolve_metadata_value( + request_data, "user_api_key_user_id" + ) + team_id = AktoGuardrail.resolve_metadata_value( + request_data, "user_api_key_team_id" + ) if user_id: tag["user_id"] = user_id if team_id: @@ -236,15 +256,23 @@ class AktoGuardrail(CustomGuardrail): response_headers: Dict[str, str] = {} if include_response: response_body = self.build_response_body(inputs, request_data) - response_payload = json.dumps({"body": json.dumps(response_body)}) # Double-encoded + response_payload = json.dumps( + {"body": json.dumps(response_body)} + ) # Double-encoded response_headers = {"content-type": "application/json"} # Extract client IP from proxy headers ip = "" proxy_req = request_data.get("proxy_server_request", {}) - proxy_headers = proxy_req.get("headers", {}) if isinstance(proxy_req, dict) else {} + proxy_headers = ( + proxy_req.get("headers", {}) if isinstance(proxy_req, dict) else {} + ) if isinstance(proxy_headers, dict): - ip = proxy_headers.get("x-forwarded-for") or proxy_headers.get("x-real-ip") or "" + ip = ( + proxy_headers.get("x-forwarded-for") + or proxy_headers.get("x-real-ip") + or "" + ) if "," in ip: ip = ip.split(",")[0].strip() @@ -253,7 +281,9 @@ class AktoGuardrail(CustomGuardrail): "requestHeaders": json.dumps(request_headers), "responseHeaders": json.dumps(response_headers), "method": "POST", - "requestPayload": json.dumps({"body": json.dumps(request_body)}), # Double-encoded + "requestPayload": json.dumps( + {"body": json.dumps(request_body)} + ), # Double-encoded "responsePayload": response_payload, "ip": ip, "destIp": "127.0.0.1", @@ -393,7 +423,9 @@ class AktoGuardrail(CustomGuardrail): if input_type == "request": # Pre_call: awaited guardrail check (no ingestion) - payload = self.build_akto_payload(inputs, request_data, include_response=False) + payload = self.build_akto_payload( + inputs, request_data, include_response=False + ) try: response = await self.send_request( guardrails=True, @@ -419,7 +451,9 @@ class AktoGuardrail(CustomGuardrail): ) blocked_payload["responsePayload"] = json.dumps( { - "body": json.dumps({"x-blocked-by": "Akto Proxy", "reason": reason}), + "body": json.dumps( + {"x-blocked-by": "Akto Proxy", "reason": reason} + ), } ) blocked_payload["responseHeaders"] = json.dumps( @@ -442,7 +476,9 @@ class AktoGuardrail(CustomGuardrail): elif input_type == "response": # Post_call: fire-and-forget combined guardrail + ingest - payload = self.build_akto_payload(inputs, request_data, include_response=True) + payload = self.build_akto_payload( + inputs, request_data, include_response=True + ) task = asyncio.create_task( self.fire_and_forget_request( guardrails=True, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ecf25d73cb..5aa0b9ea4a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -639,9 +639,9 @@ except ImportError: server_root_path = get_server_root_path() _license_check = LicenseCheck() premium_user: bool = _license_check.is_premium() -premium_user_data: Optional["EnterpriseLicenseData"] = ( - _license_check.airgapped_license_data -) +premium_user_data: Optional[ + "EnterpriseLicenseData" +] = _license_check.airgapped_license_data global_max_parallel_request_retries_env: Optional[str] = os.getenv( "LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES" ) @@ -1524,9 +1524,9 @@ master_key: Optional[str] = None config_agents: Optional[List[AgentConfig]] = None otel_logging = False prisma_client: Optional[PrismaClient] = None -shared_aiohttp_session: Optional["ClientSession"] = ( - None # Global shared session for connection reuse -) +shared_aiohttp_session: Optional[ + "ClientSession" +] = None # Global shared session for connection reuse user_api_key_cache = DualCache( default_in_memory_ttl=UserAPIKeyCacheTTLEnum.in_memory_cache_ttl.value ) @@ -1534,13 +1534,13 @@ model_max_budget_limiter = _PROXY_VirtualKeyModelMaxBudgetLimiter( dual_cache=user_api_key_cache ) litellm.logging_callback_manager.add_litellm_callback(model_max_budget_limiter) -redis_usage_cache: Optional[RedisCache] = ( - None # redis cache used for tracking spend, tpm/rpm limits -) +redis_usage_cache: Optional[ + RedisCache +] = None # redis cache used for tracking spend, tpm/rpm limits polling_via_cache_enabled: Union[Literal["all"], List[str], bool] = False -native_background_mode: List[str] = ( - [] -) # Models that should use native provider background mode instead of polling +native_background_mode: List[ + str +] = [] # Models that should use native provider background mode instead of polling polling_cache_ttl: int = 3600 # Default 1 hour TTL for polling cache user_custom_auth = None user_custom_key_generate = None @@ -1900,9 +1900,9 @@ async def update_cache( # noqa: PLR0915 _id = "team_id:{}".format(team_id) try: # Fetch the existing cost for the given user - existing_spend_obj: Optional[LiteLLM_TeamTable] = ( - await user_api_key_cache.async_get_cache(key=_id) - ) + existing_spend_obj: Optional[ + LiteLLM_TeamTable + ] = await user_api_key_cache.async_get_cache(key=_id) if existing_spend_obj is None: # do nothing if team not in api key cache return @@ -2023,9 +2023,11 @@ def run_ollama_serve(): with open(os.devnull, "w") as devnull: subprocess.Popen(command, stdout=devnull, stderr=devnull) except Exception as e: - verbose_proxy_logger.debug(f""" + verbose_proxy_logger.debug( + f""" LiteLLM Warning: proxy started with `ollama` model\n`ollama serve` failed with Exception{e}. \nEnsure you run `ollama serve` - """) + """ + ) def _get_process_rss_mb() -> Optional[float]: @@ -4976,10 +4978,10 @@ class ProxyConfig: ) try: - guardrails_in_db: List[Guardrail] = ( - await GuardrailRegistry.get_all_guardrails_from_db( - prisma_client=prisma_client - ) + guardrails_in_db: List[ + Guardrail + ] = await GuardrailRegistry.get_all_guardrails_from_db( + prisma_client=prisma_client ) verbose_proxy_logger.debug( "guardrails from the DB %s", str(guardrails_in_db) @@ -5361,9 +5363,9 @@ async def initialize( # noqa: PLR0915 user_api_base = api_base dynamic_config[user_model]["api_base"] = api_base if api_version: - os.environ["AZURE_API_VERSION"] = ( - api_version # set this for azure - litellm can read this from the env - ) + os.environ[ + "AZURE_API_VERSION" + ] = api_version # set this for azure - litellm can read this from the env if max_tokens: # model-specific param dynamic_config[user_model]["max_tokens"] = max_tokens if temperature: # model-specific param @@ -5700,9 +5702,9 @@ class ProxyStartupEvent: """ from litellm.secret_managers.main import str_to_bool - _use_redis_transaction_buffer: Optional[Union[bool, str]] = ( - general_settings.get("use_redis_transaction_buffer", False) - ) + _use_redis_transaction_buffer: Optional[ + Union[bool, str] + ] = general_settings.get("use_redis_transaction_buffer", False) if isinstance(_use_redis_transaction_buffer, str): _use_redis_transaction_buffer = str_to_bool(_use_redis_transaction_buffer) @@ -12297,9 +12299,9 @@ async def get_config_list( hasattr(sub_field_info, "description") and sub_field_info.description is not None ): - nested_fields[idx].field_description = ( - sub_field_info.description - ) + nested_fields[ + idx + ].field_description = sub_field_info.description idx += 1 _stored_in_db = None