Merge pull request #25673 from michelligabriele/fix/responses-api-cache-key

fix(caching): add Responses API params to cache key allow-list

litellm/litellm_core_utils/model_param_helper.py

@@ -11,6 +11,10 @@ from openai.types.completion_create_params import (
     CompletionCreateParamsStreaming as TextCompletionCreateParamsStreaming,
 )
 from openai.types.embedding_create_params import EmbeddingCreateParams
+from openai.types.responses.response_create_params import (
+    ResponseCreateParamsNonStreaming,
+    ResponseCreateParamsStreaming,
+)
 
 from litellm._logging import verbose_logger
 from litellm.types.rerank import RerankRequest
@@ -65,6 +69,9 @@ class ModelParamHelper:
             ModelParamHelper._get_litellm_supported_transcription_kwargs()
         )
         rerank_kwargs = ModelParamHelper._get_litellm_supported_rerank_kwargs()
+        responses_api_kwargs = (
+            ModelParamHelper._get_litellm_supported_responses_api_kwargs()
+        )
         exclude_kwargs = ModelParamHelper._get_exclude_kwargs()
 
         combined_kwargs = chat_completion_kwargs.union(
@@ -72,6 +79,7 @@ class ModelParamHelper:
             embedding_kwargs,
             transcription_kwargs,
             rerank_kwargs,
+            responses_api_kwargs,
         )
         combined_kwargs = combined_kwargs.difference(exclude_kwargs)
         return combined_kwargs
@@ -167,6 +175,21 @@ class ModelParamHelper:
             verbose_logger.debug("Error getting transcription kwargs %s", str(e))
             return set()
 
+    @staticmethod
+    def _get_litellm_supported_responses_api_kwargs() -> Set[str]:
+        """
+        Get the litellm supported responses API kwargs
+
+        This follows the OpenAI API Spec
+        """
+        non_streaming_params: Set[str] = set(
+            getattr(ResponseCreateParamsNonStreaming, "__annotations__", {}).keys()
+        )
+        streaming_params: Set[str] = set(
+            getattr(ResponseCreateParamsStreaming, "__annotations__", {}).keys()
+        )
+        return non_streaming_params.union(streaming_params)
+
     @staticmethod
     def _get_exclude_kwargs() -> Set[str]:
         """

tests/local_testing/test_unit_test_caching.py

@@ -131,6 +131,57 @@ def test_get_cache_key_text_completion():
     assert cache_key_2 == cache_key_3
 
 
+def test_get_cache_key_responses_api():
+    """
+    Regression test: two /v1/responses calls that differ only in
+    `instructions` (or any Responses-API-only param) must produce
+    different cache keys. Mirrors the chat / embedding / text-completion
+    cache-key tests above.
+    """
+    cache = Cache()
+
+    base_kwargs = {
+        "model": "openai/gpt-4.1",
+        "input": [{"role": "user", "content": "what is the weather"}],
+        "temperature": 0.3,
+    }
+
+    kwargs_a = {
+        **base_kwargs,
+        "instructions": "summarize the weather on 10th May",
+    }
+    kwargs_b = {
+        **base_kwargs,
+        "instructions": "summarize the weather on 7th May",
+    }
+
+    key_a = cache.get_cache_key(**kwargs_a)
+    key_b = cache.get_cache_key(**kwargs_b)
+
+    assert isinstance(key_a, str) and len(key_a) > 0
+    assert key_a != key_b, (
+        "instructions must be part of the Responses API cache key"
+    )
+
+    # Sanity: identical payloads must still collide (cache hits still work)
+    key_a_again = cache.get_cache_key(**kwargs_a)
+    assert key_a == key_a_again
+
+    # Spot-check a handful of other Responses-only params individually.
+    for param, value_x, value_y in [
+        ("previous_response_id", "resp_aaa", "resp_bbb"),
+        ("reasoning", {"effort": "low"}, {"effort": "high"}),
+        ("include", ["reasoning.encrypted_content"], []),
+        ("max_output_tokens", 100, 500),
+        ("background", True, False),
+    ]:
+        kx = {**base_kwargs, param: value_x}
+        ky = {**base_kwargs, param: value_y}
+        assert cache.get_cache_key(**kx) != cache.get_cache_key(**ky), (
+            f"Responses-API param `{param}` is not part of the cache key"
+        )
+
+
 def test_get_hashed_cache_key():
     cache = Cache()
     cache_key = "model:gpt-3.5-turbo,messages:Hello world"

tests/test_litellm/test_model_param_helper.py

@@ -31,3 +31,32 @@ def test_get_standard_logging_model_parameters_excludes_prompt_content():
     assert "prompt" not in result
     assert "input" not in result
     assert result == {"temperature": 0.5}
+
+
+def test_get_all_llm_api_params_includes_responses_api():
+    """
+    Regression guard for the Responses API cache-key bug:
+    Responses-API-only kwargs must be present in the cache-key allow-list,
+    otherwise Cache.get_cache_key() silently drops them and two requests
+    that differ only in (e.g.) `instructions` collide on the same key.
+    """
+    all_params = ModelParamHelper._get_all_llm_api_params()
+    responses_only_params = {
+        "instructions",
+        "previous_response_id",
+        "reasoning",
+        "include",
+        "store",
+        "background",
+        "max_output_tokens",
+        "max_tool_calls",
+        "prompt_cache_key",
+        "prompt_cache_retention",
+        "context_management",
+        "conversation",
+        "safety_identifier",
+    }
+    missing = responses_only_params - all_params
+    assert missing == set(), (
+        f"Responses-API kwargs missing from cache-key allow-list: {sorted(missing)}"
+    )