From 94b319c57747d9ce0a6afb82c433775bb625952a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 30 Apr 2026 00:45:50 +0000 Subject: [PATCH 01/30] tests(llm_translation): add VCR cassette infrastructure for offline replay Live LLM e2e tests have been draining provider billing accounts and going flaky on outages (LIT-2683). This change introduces vcrpy-backed cassette replay so CI can exercise the same end-to-end LiteLLM transformation paths without hitting the live provider: - Add 'vcrpy==8.1.1' to the dev dependency group. - New 'tests/llm_translation/vcr_config.py' centralises the VCR config: filters auth/secret headers and per-request response headers, matches on method+URI+body, and exposes 'LITELLM_VCR_RECORD_MODE' for re-recording. - New 'tests/llm_translation/test_anthropic_completion_vcr.py' demonstrates the pattern with one non-streaming and one streaming Anthropic test that replay from cassettes shipped under 'cassettes/'. - New 'tests/llm_translation/cassettes/_record_anthropic_fixtures.py' lets contributors regenerate the canned Anthropic cassettes against a local in-process mock (no API key required), and 'cassettes/README.md' documents the full record/replay/refresh workflow. - New 'make test-llm-translation-record FILE=...' Makefile target to refresh cassettes against the live API. Co-authored-by: Mateo Wang --- Makefile | 15 ++ pyproject.toml | 1 + tests/llm_translation/Readme.md | 21 +- tests/llm_translation/cassettes/README.md | 80 ++++++ .../cassettes/_record_anthropic_fixtures.py | 228 ++++++++++++++++++ .../cassettes/anthropic_basic_completion.yaml | 127 ++++++++++ .../anthropic_streaming_completion.yaml | 168 +++++++++++++ .../test_anthropic_completion_vcr.py | 100 ++++++++ tests/llm_translation/vcr_config.py | 123 ++++++++++ uv.lock | 17 +- 10 files changed, 877 insertions(+), 3 deletions(-) create mode 100644 tests/llm_translation/cassettes/README.md create mode 100644 tests/llm_translation/cassettes/_record_anthropic_fixtures.py create mode 100644 tests/llm_translation/cassettes/anthropic_basic_completion.yaml create mode 100644 tests/llm_translation/cassettes/anthropic_streaming_completion.yaml create mode 100644 tests/llm_translation/test_anthropic_completion_vcr.py create mode 100644 tests/llm_translation/vcr_config.py diff --git a/Makefile b/Makefile index b6b674ff3b..0e5af3e17c 100644 --- a/Makefile +++ b/Makefile @@ -185,3 +185,18 @@ test-llm-translation-single: install-test-deps $(UV_RUN) pytest tests/llm_translation/$(FILE) \ --junitxml=test-results/junit.xml \ -v --tb=short --maxfail=100 --timeout=300 + +# VCR cassette helpers -------------------------------------------------------- +# Re-record a single VCR-backed translation test file against the live API. +# Provider credentials must be exported (e.g. ANTHROPIC_API_KEY). +# +# Example: +# ANTHROPIC_API_KEY=sk-ant-... make test-llm-translation-record \ +# FILE=test_anthropic_completion_vcr.py +test-llm-translation-record: install-test-deps + @if [ -z "$(FILE)" ]; then \ + echo "Usage: make test-llm-translation-record FILE=test_filename.py"; \ + exit 1; \ + fi + LITELLM_VCR_RECORD_MODE=once \ + $(UV_RUN) pytest tests/llm_translation/$(FILE) -v --tb=short diff --git a/pyproject.toml b/pyproject.toml index 657632d69e..7db6c6d9b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -149,6 +149,7 @@ dev = [ "parameterized==0.9.0", "openapi-core==0.22.0; python_version < '3.14'", "pytest-timeout==2.4.0", + "vcrpy==8.1.1", ] proxy-dev = [ "prisma==0.11.0", diff --git a/tests/llm_translation/Readme.md b/tests/llm_translation/Readme.md index db84e7c33c..b5a76e48ea 100644 --- a/tests/llm_translation/Readme.md +++ b/tests/llm_translation/Readme.md @@ -1,3 +1,20 @@ -Unit tests for individual LLM providers. +Unit tests for individual LLM providers. -Name of the test file is the name of the LLM provider - e.g. `test_openai.py` is for OpenAI. \ No newline at end of file +Name of the test file is the name of the LLM provider - e.g. `test_openai.py` is for OpenAI. + +## VCR-backed tests + +Files matching `*_vcr.py` (e.g. `test_anthropic_completion_vcr.py`) replay +recorded HTTP traffic from `cassettes/` instead of calling the real provider. +They run offline by default — no API keys required, no per-PR cost. + +To re-record against the live API: + +```bash +ANTHROPIC_API_KEY=sk-ant-... \ + make test-llm-translation-record FILE=test_anthropic_completion_vcr.py +``` + +See [`cassettes/README.md`](./cassettes/README.md) for the full workflow, +including how to add a new cassette-backed test and what to scrub from +recordings before committing. diff --git a/tests/llm_translation/cassettes/README.md b/tests/llm_translation/cassettes/README.md new file mode 100644 index 0000000000..0820d01ebd --- /dev/null +++ b/tests/llm_translation/cassettes/README.md @@ -0,0 +1,80 @@ +# VCR cassettes for LLM translation tests + +This directory holds [vcrpy](https://vcrpy.readthedocs.io/) cassettes used by +`tests/llm_translation/` to replay real provider HTTP traffic without hitting +the live API. + +Why this exists is tracked in +[LIT-2683](https://linear.app/litellm-ai/issue/LIT-2683) and discussed in +`#sdlc` on Slack: e2e tests were repeatedly draining provider billing accounts +and producing flaky CI on outages. Recording the HTTP exchange once and +replaying it on subsequent runs gives us realistic provider responses +(streaming, headers, edge-case payloads) at zero per-PR cost. + +## How to add a new cassette-backed test + +1. Pick a small, deterministic call. Avoid prompts whose output depends on + wall-clock time, randomness, or live web data. +2. Add a test in a `*_vcr.py` file under `tests/llm_translation/`. Wrap it + with `@litellm_vcr.use_cassette(".yaml")` from + `tests/llm_translation/vcr_config.py`. +3. Record the cassette once: + + ```bash + LITELLM_VCR_RECORD_MODE=once \ + ANTHROPIC_API_KEY=sk-ant-... \ + uv run pytest tests/llm_translation/test_my_provider_vcr.py::test_my_case -v + ``` + + or, equivalently: + + ```bash + ANTHROPIC_API_KEY=sk-ant-... \ + make test-llm-translation-record FILE=test_my_provider_vcr.py + ``` + +4. Inspect the resulting YAML file: + - **Strip any secrets** that survived `vcr_config.py`'s header filter. + `vcr_config.py` already removes the common ones (`Authorization`, + `x-api-key`, `cookie`, AWS sigv4 headers, etc.) — but a request *body* + might contain a token if your test passed one inline. + - Trim very large response bodies if they aren't load-bearing for the + assertion. +5. Commit the cassette alongside the test. + +## Re-recording + +Run the same `make test-llm-translation-record` command. vcrpy's `once` mode +will *not* overwrite an existing cassette — delete the file first if you're +intentionally refreshing it: + +```bash +rm tests/llm_translation/cassettes/anthropic_basic_completion.yaml +ANTHROPIC_API_KEY=sk-ant-... make test-llm-translation-record \ + FILE=test_anthropic_completion_vcr.py +``` + +## Refreshing the canned Anthropic fixtures + +The two Anthropic cassettes in this directory +(`anthropic_basic_completion.yaml` and `anthropic_streaming_completion.yaml`) +are recorded against an in-process mock so contributors can regenerate them +without an `ANTHROPIC_API_KEY`: + +```bash +uv run python tests/llm_translation/cassettes/_record_anthropic_fixtures.py +``` + +For a full refresh against the real API, delete the cassettes first and use +the `LITELLM_VCR_RECORD_MODE=once` path with a real key. + +## Don't + +- Don't commit cassettes containing real API keys, OAuth tokens, or PII. + When in doubt, `grep -i 'sk-\|bearer\|api-key' cassettes/*.yaml` after + recording. +- Don't rely on cassettes for tests of *non-deterministic* behavior + (rate-limit retries, timeouts, the model itself making a creative choice). + Mock those at the LiteLLM layer instead. +- Don't record both real and mock host names into the same cassette without + rewriting the URL — vcrpy matches on host/port by default. diff --git a/tests/llm_translation/cassettes/_record_anthropic_fixtures.py b/tests/llm_translation/cassettes/_record_anthropic_fixtures.py new file mode 100644 index 0000000000..0e75fd9650 --- /dev/null +++ b/tests/llm_translation/cassettes/_record_anthropic_fixtures.py @@ -0,0 +1,228 @@ +"""Helper script that records Anthropic-shaped cassettes against a local mock. + +This is a *one-shot* utility, not a test. It exists so we can deterministically +regenerate the canned Anthropic cassettes shipped under +``tests/llm_translation/cassettes/`` without spending real provider credits and +without needing an ``ANTHROPIC_API_KEY``. + +Run it with:: + + uv run python tests/llm_translation/cassettes/_record_anthropic_fixtures.py + +The script: + +1. Spins up a tiny in-process HTTP server that returns canned Anthropic + ``/v1/messages`` payloads (one non-streaming, one SSE streaming). +2. Records LiteLLM's real outbound HTTP through vcrpy. +3. Rewrites the cassette URL/Host so replay matches genuine + ``https://api.anthropic.com/v1/messages`` traffic. + +If you want to refresh against the *real* Anthropic API instead, use the +``LITELLM_VCR_RECORD_MODE=once`` workflow described in +``tests/llm_translation/vcr_config.py`` — that path needs a real API key. +""" + +from __future__ import annotations + +import json +import os +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Iterable + +import vcr # type: ignore[import-not-found] + +REPO_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(REPO_ROOT)) + +import litellm # noqa: E402 + +CASSETTE_DIR = Path(__file__).parent +MOCK_HOST = "127.0.0.1" +NON_STREAM_PORT = 18765 +STREAM_PORT = 18766 +REAL_ANTHROPIC_HOST = "api.anthropic.com" + +NON_STREAM_RESPONSE: dict[str, Any] = { + "id": "msg_01ABCDEFGHIJKLMNOPQRSTUV", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5-20250929", + "content": [{"type": "text", "text": "Hello! How can I help you today?"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": { + "input_tokens": 12, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 11, + }, +} + +STREAM_EVENTS: list[tuple[str, dict[str, Any]]] = [ + ( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_01STREAMABCDEFGH", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5-20250929", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 14, "output_tokens": 1}, + }, + }, + ), + ( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + ), + ( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "Hello"}, + }, + ), + ( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": " from"}, + }, + ), + ( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": " LiteLLM!"}, + }, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 5}, + }, + ), + ("message_stop", {"type": "message_stop"}), +] + + +def _make_handler(mode: str) -> type[BaseHTTPRequestHandler]: + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args: Any, **kwargs: Any) -> None: # silence + return + + def do_POST(self) -> None: # noqa: N802 + length = int(self.headers.get("Content-Length", "0")) + self.rfile.read(length) + if mode == "json": + body = json.dumps(NON_STREAM_RESPONSE).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("anthropic-ratelimit-requests-limit", "4000") + self.send_header("anthropic-ratelimit-requests-remaining", "3999") + self.end_headers() + self.wfile.write(body) + else: + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.end_headers() + for event_name, data in STREAM_EVENTS: + chunk = ( + f"event: {event_name}\n" f"data: {json.dumps(data)}\n\n" + ).encode("utf-8") + self.wfile.write(chunk) + self.wfile.flush() + + return Handler + + +def _serve(port: int, mode: str) -> ThreadingHTTPServer: + srv = ThreadingHTTPServer((MOCK_HOST, port), _make_handler(mode)) + threading.Thread(target=srv.serve_forever, daemon=True).start() + return srv + + +def _rewrite_cassette_to_real_host(path: Path, mock_host_port: str) -> None: + """Replace mock host/port in the cassette with the real Anthropic host.""" + text = path.read_text() + text = text.replace(f"http://{mock_host_port}", f"https://{REAL_ANTHROPIC_HOST}") + text = text.replace(mock_host_port, REAL_ANTHROPIC_HOST) + path.write_text(text) + + +def _consume(iterable: Iterable[Any]) -> None: + for _ in iterable: + pass + + +def record_non_streaming() -> None: + cassette = CASSETTE_DIR / "anthropic_basic_completion.yaml" + server = _serve(NON_STREAM_PORT, "json") + try: + my_vcr = vcr.VCR( + record_mode="all", + filter_headers=["authorization", "x-api-key", "anthropic-version"], + ) + with my_vcr.use_cassette(str(cassette)): + response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Hello!"}], + api_base=f"http://{MOCK_HOST}:{NON_STREAM_PORT}", + api_key="sk-ant-recording", + ) + assert response.choices[0].message.content + finally: + server.shutdown() + _rewrite_cassette_to_real_host(cassette, f"{MOCK_HOST}:{NON_STREAM_PORT}") + + +def record_streaming() -> None: + cassette = CASSETTE_DIR / "anthropic_streaming_completion.yaml" + server = _serve(STREAM_PORT, "stream") + try: + my_vcr = vcr.VCR( + record_mode="all", + filter_headers=["authorization", "x-api-key", "anthropic-version"], + ) + with my_vcr.use_cassette(str(cassette)): + stream = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Hello!"}], + api_base=f"http://{MOCK_HOST}:{STREAM_PORT}", + api_key="sk-ant-recording", + stream=True, + ) + _consume(stream) + finally: + server.shutdown() + _rewrite_cassette_to_real_host(cassette, f"{MOCK_HOST}:{STREAM_PORT}") + + +def main() -> None: + os.environ.setdefault("LITELLM_LOG", "WARNING") + record_non_streaming() + record_streaming() + print(f"Wrote cassettes to {CASSETTE_DIR}") + + +if __name__ == "__main__": + main() diff --git a/tests/llm_translation/cassettes/anthropic_basic_completion.yaml b/tests/llm_translation/cassettes/anthropic_basic_completion.yaml new file mode 100644 index 0000000000..458d41f69e --- /dev/null +++ b/tests/llm_translation/cassettes/anthropic_basic_completion.yaml @@ -0,0 +1,127 @@ +interactions: +- request: + body: '{"model": "claude-sonnet-4-5-20250929", "messages": [{"role": "user", "content": + [{"type": "text", "text": "Hello!"}]}], "max_tokens": 64000}' + headers: + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.anthropic.com + User-Agent: + - litellm/1.84.0 + accept: + - application/json + content-type: + - application/json + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"id": "msg_01ABCDEFGHIJKLMNOPQRSTUV", "type": "message", "role": "assistant", + "model": "claude-sonnet-4-5-20250929", "content": [{"type": "text", "text": + "Hello! How can I help you today?"}], "stop_reason": "end_turn", "stop_sequence": + null, "usage": {"input_tokens": 12, "cache_creation_input_tokens": 0, "cache_read_input_tokens": + 0, "output_tokens": 11}}' + headers: + Content-Type: + - application/json + anthropic-ratelimit-requests-limit: + - '4000' + anthropic-ratelimit-requests-remaining: + - '3999' + anthropic-ratelimit-tokens-limit: + - '400000' + anthropic-ratelimit-tokens-remaining: + - '399988' + status: + code: 200 + message: OK +- request: + body: '{"model": "claude-sonnet-4-5-20250929", "messages": [{"role": "user", "content": + [{"type": "text", "text": "Hello!"}]}], "max_tokens": 64000}' + headers: + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '141' + Host: + - api.anthropic.com + User-Agent: + - litellm/1.84.0 + accept: + - application/json + content-type: + - application/json + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"id": "msg_01ABCDEFGHIJKLMNOPQRSTUV", "type": "message", "role": "assistant", + "model": "claude-sonnet-4-5-20250929", "content": [{"type": "text", "text": + "Hello! How can I help you today?"}], "stop_reason": "end_turn", "stop_sequence": + null, "usage": {"input_tokens": 12, "cache_creation_input_tokens": 0, "cache_read_input_tokens": + 0, "output_tokens": 11}}' + headers: + Content-Length: + - '358' + Content-Type: + - application/json + Date: + - Thu, 30 Apr 2026 00:43:16 GMT + Server: + - BaseHTTP/0.6 Python/3.12.3 + anthropic-ratelimit-requests-limit: + - '4000' + anthropic-ratelimit-requests-remaining: + - '3999' + status: + code: 200 + message: OK +- request: + body: '{"model": "claude-sonnet-4-5-20250929", "messages": [{"role": "user", "content": + [{"type": "text", "text": "Hello!"}]}], "max_tokens": 64000}' + headers: + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '141' + Host: + - api.anthropic.com + User-Agent: + - litellm/1.84.0 + accept: + - application/json + content-type: + - application/json + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"id": "msg_01ABCDEFGHIJKLMNOPQRSTUV", "type": "message", "role": "assistant", + "model": "claude-sonnet-4-5-20250929", "content": [{"type": "text", "text": + "Hello! How can I help you today?"}], "stop_reason": "end_turn", "stop_sequence": + null, "usage": {"input_tokens": 12, "cache_creation_input_tokens": 0, "cache_read_input_tokens": + 0, "output_tokens": 11}}' + headers: + Content-Length: + - '358' + Content-Type: + - application/json + Date: + - Thu, 30 Apr 2026 00:45:24 GMT + Server: + - BaseHTTP/0.6 Python/3.12.3 + anthropic-ratelimit-requests-limit: + - '4000' + anthropic-ratelimit-requests-remaining: + - '3999' + status: + code: 200 + message: OK +version: 1 diff --git a/tests/llm_translation/cassettes/anthropic_streaming_completion.yaml b/tests/llm_translation/cassettes/anthropic_streaming_completion.yaml new file mode 100644 index 0000000000..3f80a85ec5 --- /dev/null +++ b/tests/llm_translation/cassettes/anthropic_streaming_completion.yaml @@ -0,0 +1,168 @@ +interactions: +- request: + body: '{"model": "claude-sonnet-4-5-20250929", "messages": [{"role": "user", "content": + [{"type": "text", "text": "Hello!"}]}], "max_tokens": 64000, "stream": true}' + headers: + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '157' + Host: + - api.anthropic.com + User-Agent: + - litellm/1.84.0 + accept: + - application/json + content-type: + - application/json + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: 'event: message_start + + data: {"type": "message_start", "message": {"id": "msg_01STREAMABCDEFGH", + "type": "message", "role": "assistant", "model": "claude-sonnet-4-5-20250929", + "content": [], "stop_reason": null, "stop_sequence": null, "usage": {"input_tokens": + 14, "output_tokens": 1}}} + + + event: content_block_start + + data: {"type": "content_block_start", "index": 0, "content_block": {"type": + "text", "text": ""}} + + + event: content_block_delta + + data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", + "text": "Hello"}} + + + event: content_block_delta + + data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", + "text": " from"}} + + + event: content_block_delta + + data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", + "text": " LiteLLM!"}} + + + event: content_block_stop + + data: {"type": "content_block_stop", "index": 0} + + + event: message_delta + + data: {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": + null}, "usage": {"output_tokens": 5}} + + + event: message_stop + + data: {"type": "message_stop"} + + + ' + headers: + Cache-Control: + - no-cache + Content-Type: + - text/event-stream + Date: + - Thu, 30 Apr 2026 00:43:17 GMT + Server: + - BaseHTTP/0.6 Python/3.12.3 + status: + code: 200 + message: OK +- request: + body: '{"model": "claude-sonnet-4-5-20250929", "messages": [{"role": "user", "content": + [{"type": "text", "text": "Hello!"}]}], "max_tokens": 64000, "stream": true}' + headers: + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '157' + Host: + - api.anthropic.com + User-Agent: + - litellm/1.84.0 + accept: + - application/json + content-type: + - application/json + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: 'event: message_start + + data: {"type": "message_start", "message": {"id": "msg_01STREAMABCDEFGH", + "type": "message", "role": "assistant", "model": "claude-sonnet-4-5-20250929", + "content": [], "stop_reason": null, "stop_sequence": null, "usage": {"input_tokens": + 14, "output_tokens": 1}}} + + + event: content_block_start + + data: {"type": "content_block_start", "index": 0, "content_block": {"type": + "text", "text": ""}} + + + event: content_block_delta + + data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", + "text": "Hello"}} + + + event: content_block_delta + + data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", + "text": " from"}} + + + event: content_block_delta + + data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", + "text": " LiteLLM!"}} + + + event: content_block_stop + + data: {"type": "content_block_stop", "index": 0} + + + event: message_delta + + data: {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": + null}, "usage": {"output_tokens": 5}} + + + event: message_stop + + data: {"type": "message_stop"} + + + ' + headers: + Cache-Control: + - no-cache + Content-Type: + - text/event-stream + Date: + - Thu, 30 Apr 2026 00:45:25 GMT + Server: + - BaseHTTP/0.6 Python/3.12.3 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/llm_translation/test_anthropic_completion_vcr.py b/tests/llm_translation/test_anthropic_completion_vcr.py new file mode 100644 index 0000000000..6367e65e01 --- /dev/null +++ b/tests/llm_translation/test_anthropic_completion_vcr.py @@ -0,0 +1,100 @@ +""" +VCR-backed Anthropic completion tests. + +These tests exercise the same end-to-end ``litellm.completion`` code paths +as ``test_anthropic_completion.py`` but replay HTTP traffic from cassettes +under ``cassettes/`` instead of calling ``api.anthropic.com``. CI can run +them with no API key and zero cost. + +To re-record after a deliberate change to request shape (or to refresh +against the live API), set ``LITELLM_VCR_RECORD_MODE=once`` and provide a +real ``ANTHROPIC_API_KEY``:: + + LITELLM_VCR_RECORD_MODE=once \\ + ANTHROPIC_API_KEY=sk-ant-... \\ + uv run pytest tests/llm_translation/test_anthropic_completion_vcr.py -v + +See ``tests/llm_translation/vcr_config.py`` and ``tests/llm_translation/cassettes/README.md`` +for the full workflow. +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) +sys.path.insert(0, os.path.dirname(__file__)) + +import litellm # noqa: E402 + +from vcr_config import litellm_vcr # noqa: E402 + + +# A non-secret placeholder API key. We never want a real key written to a +# cassette, and ``vcr_config`` filters Authorization / x-api-key headers +# anyway. Using a deterministic placeholder also stops the SDK from raising +# when ``ANTHROPIC_API_KEY`` is unset (the common CI case). +PLACEHOLDER_ANTHROPIC_API_KEY = "sk-ant-vcr-placeholder" + + +@pytest.fixture(autouse=True) +def _placeholder_anthropic_key(monkeypatch): + """Provide a placeholder key when none is set so replay works offline. + + If a real key is present in the environment (e.g. when re-recording), + we leave it untouched. + """ + if not os.environ.get("ANTHROPIC_API_KEY"): + monkeypatch.setenv("ANTHROPIC_API_KEY", PLACEHOLDER_ANTHROPIC_API_KEY) + + +@litellm_vcr.use_cassette("anthropic_basic_completion.yaml") +def test_anthropic_basic_completion_replay(): + """Smoke-test that a vanilla Anthropic completion replays from a cassette. + + This is the canonical example for the cassette-based testing pattern: + no API key required at runtime, deterministic output, and the full + LiteLLM transformation pipeline (request shaping + response parsing) + runs against a real-shape Anthropic payload. + """ + response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Hello!"}], + ) + + assert response is not None + assert response.choices[0].message.content == ("Hello! How can I help you today?") + assert response.usage.prompt_tokens == 12 + assert response.usage.completion_tokens == 11 + # Anthropic sets stop_reason="end_turn" → litellm normalises to "stop" + assert response.choices[0].finish_reason == "stop" + + +@litellm_vcr.use_cassette("anthropic_streaming_completion.yaml") +def test_anthropic_streaming_completion_replay(): + """Replay a streaming Anthropic completion from a cassette. + + Exercises the SSE chunk parser and the public streaming surface. The + underlying cassette captures every ``content_block_delta`` event Anthropic + emits, so any regression in the streaming transformation will surface here. + """ + stream = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Hello!"}], + stream=True, + ) + + collected_text = "" + finish_reason = None + for chunk in stream: + if not chunk.choices: + continue + delta = chunk.choices[0].delta + if delta and delta.content: + collected_text += delta.content + if chunk.choices[0].finish_reason: + finish_reason = chunk.choices[0].finish_reason + + assert collected_text == "Hello from LiteLLM!" + assert finish_reason == "stop" diff --git a/tests/llm_translation/vcr_config.py b/tests/llm_translation/vcr_config.py new file mode 100644 index 0000000000..7e641c9cad --- /dev/null +++ b/tests/llm_translation/vcr_config.py @@ -0,0 +1,123 @@ +""" +Shared VCR configuration for ``tests/llm_translation``. + +This module centralises the cassette setup used by tests that would otherwise +hit a real LLM provider over the network. The goal is to let CI replay +recorded HTTP traffic by default — no API keys required — and to provide a +single switch for re-recording cassettes against the live provider. + +Usage in a test:: + + from .vcr_config import litellm_vcr # noqa: E402 + + @litellm_vcr.use_cassette("anthropic_basic_completion.yaml") + def test_basic_completion(): + resp = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Hello!"}], + ) + assert resp.choices[0].message.content + +Recording mode +-------------- +By default the cassette is replayed (``record_mode='none'``). To re-record: + + LITELLM_VCR_RECORD_MODE=once \\ + ANTHROPIC_API_KEY=sk-ant-... \\ + uv run pytest tests/llm_translation/test_anthropic_completion_vcr.py + +Valid values for ``LITELLM_VCR_RECORD_MODE`` mirror vcrpy's record modes: +``none`` (replay only — fail on missing cassette), ``once`` (record if the +cassette doesn't exist), ``new_episodes`` (append new interactions), and +``all`` (always re-record). See the vcrpy docs for details. + +Why this exists +--------------- +Per the discussion that produced LIT-2683, our e2e tests repeatedly drained +provider billing accounts and produced flaky CI on outages. Recording the +HTTP exchange once and replaying it on subsequent runs gives us realistic +provider responses (including streaming, headers, and edge-case payloads) +without per-PR cost or rate-limit risk. Re-record periodically to catch +real provider drift. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +import vcr + +CASSETTE_DIR: Path = Path(__file__).parent / "cassettes" + +# Headers that must never be persisted to a cassette. These are matched +# case-insensitively by vcrpy. +_FILTERED_REQUEST_HEADERS = ( + "authorization", + "x-api-key", + "anthropic-api-key", + "openai-api-key", + "azure-api-key", + "api-key", + "cookie", + "x-amz-security-token", + "x-amz-date", + "x-amz-content-sha256", + "amz-sdk-invocation-id", + "amz-sdk-request", +) + +_FILTERED_RESPONSE_HEADERS = ( + "set-cookie", + "x-request-id", + "cf-ray", + "anthropic-organization-id", + "openai-organization", + "request-id", +) + + +def _record_mode() -> str: + """Resolve the active vcrpy record mode from the environment. + + Defaults to ``"none"`` so CI never accidentally hits the live provider. + """ + mode = os.environ.get("LITELLM_VCR_RECORD_MODE", "none").strip().lower() + if mode not in {"none", "once", "new_episodes", "all"}: + raise ValueError( + f"LITELLM_VCR_RECORD_MODE={mode!r} is not a valid vcrpy record mode." + ) + return mode + + +def _build_vcr() -> vcr.VCR: + """Construct the shared ``VCR`` instance used by translation tests.""" + return vcr.VCR( + cassette_library_dir=str(CASSETTE_DIR), + record_mode=_record_mode(), + # Match on method + URI + body so streaming vs non-streaming and + # different prompts get distinct cassettes. + match_on=("method", "scheme", "host", "port", "path", "query", "body"), + filter_headers=list(_FILTERED_REQUEST_HEADERS), + decode_compressed_response=True, + ) + + +def _scrub_response(response: Any) -> Any: + """Strip per-request response headers we don't want in the cassette.""" + if not isinstance(response, dict): + return response + headers = response.get("headers") or {} + if isinstance(headers, dict): + for header in list(headers): + if header.lower() in _FILTERED_RESPONSE_HEADERS: + headers.pop(header, None) + return response + + +litellm_vcr: vcr.VCR = _build_vcr() +litellm_vcr.before_record_response = _scrub_response + + +__all__ = ["litellm_vcr", "CASSETTE_DIR"] diff --git a/uv.lock b/uv.lock index f837e2b5ef..b2e8aef8a2 100644 --- a/uv.lock +++ b/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ ] [options] -exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. +exclude-newer = "2026-04-27T00:38:13.673780212Z" exclude-newer-span = "P3D" [manifest] @@ -3242,6 +3242,7 @@ dev = [ { name = "types-redis" }, { name = "types-requests" }, { name = "types-setuptools" }, + { name = "vcrpy" }, ] healthcheck = [ { name = "httpx" }, @@ -3395,6 +3396,7 @@ dev = [ { name = "types-redis", specifier = "==4.6.0.20241004" }, { name = "types-requests", specifier = "==2.32.4.20260107" }, { name = "types-setuptools", specifier = "==75.8.0.20250225" }, + { name = "vcrpy", specifier = "==8.1.1" }, ] healthcheck = [ { name = "httpx", specifier = "==0.28.1" }, @@ -7541,6 +7543,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/63/9a/0962b05b308494e3202d3f794a6e85abe471fe3cafdbcf95c2e8c713aabd/uvloop-0.21.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a5c39f217ab3c663dc699c04cbd50c13813e31d917642d459fdcec07555cc553", size = 4660018, upload-time = "2024-10-14T23:38:10.888Z" }, ] +[[package]] +name = "vcrpy" +version = "8.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/07/bcfd5ebd7cb308026ab78a353e091bd699593358be49197d39d004e5ad83/vcrpy-8.1.1.tar.gz", hash = "sha256:58e3053e33b423f3594031cb758c3f4d1df931307f1e67928e30cf352df7709f", size = 85770, upload-time = "2026-01-04T19:22:03.886Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/d7/f79b05a5d728f8786876a7d75dfb0c5cae27e428081b2d60152fb52f155f/vcrpy-8.1.1-py3-none-any.whl", hash = "sha256:2d16f31ad56493efb6165182dd99767207031b0da3f68b18f975545ede8ac4b9", size = 42445, upload-time = "2026-01-04T19:22:02.532Z" }, +] + [[package]] name = "waitress" version = "3.0.2" From 72c92920b25a2e575df54a4d661e06ffdb6c327c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 30 Apr 2026 01:34:55 +0000 Subject: [PATCH 02/30] fix(tests): make Anthropic VCR fixture script reproducible Delete existing cassettes before recording (record_mode='all' with vcrpy appends rather than overwriting), and strip non-deterministic response headers (Date, Server) so re-running the helper produces a byte-stable diff. Regenerate the committed cassettes with the fixed script so they match what contributors get when following the README. --- .../cassettes/_record_anthropic_fixtures.py | 30 +++++++ .../cassettes/anthropic_basic_completion.yaml | 86 ------------------ .../anthropic_streaming_completion.yaml | 87 ------------------- 3 files changed, 30 insertions(+), 173 deletions(-) diff --git a/tests/llm_translation/cassettes/_record_anthropic_fixtures.py b/tests/llm_translation/cassettes/_record_anthropic_fixtures.py index 0e75fd9650..c17c56169e 100644 --- a/tests/llm_translation/cassettes/_record_anthropic_fixtures.py +++ b/tests/llm_translation/cassettes/_record_anthropic_fixtures.py @@ -26,6 +26,7 @@ from __future__ import annotations import json import os +import re import sys import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -161,12 +162,37 @@ def _serve(port: int, mode: str) -> ThreadingHTTPServer: return srv +# Headers that vary every run (timestamps, server build) and must be stripped +# so the cassette is byte-stable across regenerations. Replay does not depend +# on them. +_NON_DETERMINISTIC_HEADERS = ("Date", "Server") + + +def _strip_nondeterministic_headers(path: Path) -> None: + """Remove headers whose values change every run from the cassette.""" + text = path.read_text() + for header in _NON_DETERMINISTIC_HEADERS: + # Matches a YAML block like:: + # + # Date: + # - Thu, 30 Apr 2026 00:43:16 GMT + # + # under the response ``headers:`` mapping. Indentation is fixed by vcrpy. + pattern = re.compile( + rf"^ {re.escape(header)}:\n - .*\n", + re.MULTILINE, + ) + text = pattern.sub("", text) + path.write_text(text) + + def _rewrite_cassette_to_real_host(path: Path, mock_host_port: str) -> None: """Replace mock host/port in the cassette with the real Anthropic host.""" text = path.read_text() text = text.replace(f"http://{mock_host_port}", f"https://{REAL_ANTHROPIC_HOST}") text = text.replace(mock_host_port, REAL_ANTHROPIC_HOST) path.write_text(text) + _strip_nondeterministic_headers(path) def _consume(iterable: Iterable[Any]) -> None: @@ -176,6 +202,8 @@ def _consume(iterable: Iterable[Any]) -> None: def record_non_streaming() -> None: cassette = CASSETTE_DIR / "anthropic_basic_completion.yaml" + if cassette.exists(): + cassette.unlink() server = _serve(NON_STREAM_PORT, "json") try: my_vcr = vcr.VCR( @@ -197,6 +225,8 @@ def record_non_streaming() -> None: def record_streaming() -> None: cassette = CASSETTE_DIR / "anthropic_streaming_completion.yaml" + if cassette.exists(): + cassette.unlink() server = _serve(STREAM_PORT, "stream") try: my_vcr = vcr.VCR( diff --git a/tests/llm_translation/cassettes/anthropic_basic_completion.yaml b/tests/llm_translation/cassettes/anthropic_basic_completion.yaml index 458d41f69e..f593c5ab96 100644 --- a/tests/llm_translation/cassettes/anthropic_basic_completion.yaml +++ b/tests/llm_translation/cassettes/anthropic_basic_completion.yaml @@ -1,43 +1,4 @@ interactions: -- request: - body: '{"model": "claude-sonnet-4-5-20250929", "messages": [{"role": "user", "content": - [{"type": "text", "text": "Hello!"}]}], "max_tokens": 64000}' - headers: - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Host: - - api.anthropic.com - User-Agent: - - litellm/1.84.0 - accept: - - application/json - content-type: - - application/json - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"id": "msg_01ABCDEFGHIJKLMNOPQRSTUV", "type": "message", "role": "assistant", - "model": "claude-sonnet-4-5-20250929", "content": [{"type": "text", "text": - "Hello! How can I help you today?"}], "stop_reason": "end_turn", "stop_sequence": - null, "usage": {"input_tokens": 12, "cache_creation_input_tokens": 0, "cache_read_input_tokens": - 0, "output_tokens": 11}}' - headers: - Content-Type: - - application/json - anthropic-ratelimit-requests-limit: - - '4000' - anthropic-ratelimit-requests-remaining: - - '3999' - anthropic-ratelimit-tokens-limit: - - '400000' - anthropic-ratelimit-tokens-remaining: - - '399988' - status: - code: 200 - message: OK - request: body: '{"model": "claude-sonnet-4-5-20250929", "messages": [{"role": "user", "content": [{"type": "text", "text": "Hello!"}]}], "max_tokens": 64000}' @@ -70,53 +31,6 @@ interactions: - '358' Content-Type: - application/json - Date: - - Thu, 30 Apr 2026 00:43:16 GMT - Server: - - BaseHTTP/0.6 Python/3.12.3 - anthropic-ratelimit-requests-limit: - - '4000' - anthropic-ratelimit-requests-remaining: - - '3999' - status: - code: 200 - message: OK -- request: - body: '{"model": "claude-sonnet-4-5-20250929", "messages": [{"role": "user", "content": - [{"type": "text", "text": "Hello!"}]}], "max_tokens": 64000}' - headers: - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '141' - Host: - - api.anthropic.com - User-Agent: - - litellm/1.84.0 - accept: - - application/json - content-type: - - application/json - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"id": "msg_01ABCDEFGHIJKLMNOPQRSTUV", "type": "message", "role": "assistant", - "model": "claude-sonnet-4-5-20250929", "content": [{"type": "text", "text": - "Hello! How can I help you today?"}], "stop_reason": "end_turn", "stop_sequence": - null, "usage": {"input_tokens": 12, "cache_creation_input_tokens": 0, "cache_read_input_tokens": - 0, "output_tokens": 11}}' - headers: - Content-Length: - - '358' - Content-Type: - - application/json - Date: - - Thu, 30 Apr 2026 00:45:24 GMT - Server: - - BaseHTTP/0.6 Python/3.12.3 anthropic-ratelimit-requests-limit: - '4000' anthropic-ratelimit-requests-remaining: diff --git a/tests/llm_translation/cassettes/anthropic_streaming_completion.yaml b/tests/llm_translation/cassettes/anthropic_streaming_completion.yaml index 3f80a85ec5..25c0c6c3d2 100644 --- a/tests/llm_translation/cassettes/anthropic_streaming_completion.yaml +++ b/tests/llm_translation/cassettes/anthropic_streaming_completion.yaml @@ -75,93 +75,6 @@ interactions: - no-cache Content-Type: - text/event-stream - Date: - - Thu, 30 Apr 2026 00:43:17 GMT - Server: - - BaseHTTP/0.6 Python/3.12.3 - status: - code: 200 - message: OK -- request: - body: '{"model": "claude-sonnet-4-5-20250929", "messages": [{"role": "user", "content": - [{"type": "text", "text": "Hello!"}]}], "max_tokens": 64000, "stream": true}' - headers: - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '157' - Host: - - api.anthropic.com - User-Agent: - - litellm/1.84.0 - accept: - - application/json - content-type: - - application/json - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: 'event: message_start - - data: {"type": "message_start", "message": {"id": "msg_01STREAMABCDEFGH", - "type": "message", "role": "assistant", "model": "claude-sonnet-4-5-20250929", - "content": [], "stop_reason": null, "stop_sequence": null, "usage": {"input_tokens": - 14, "output_tokens": 1}}} - - - event: content_block_start - - data: {"type": "content_block_start", "index": 0, "content_block": {"type": - "text", "text": ""}} - - - event: content_block_delta - - data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", - "text": "Hello"}} - - - event: content_block_delta - - data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", - "text": " from"}} - - - event: content_block_delta - - data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", - "text": " LiteLLM!"}} - - - event: content_block_stop - - data: {"type": "content_block_stop", "index": 0} - - - event: message_delta - - data: {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": - null}, "usage": {"output_tokens": 5}} - - - event: message_stop - - data: {"type": "message_stop"} - - - ' - headers: - Cache-Control: - - no-cache - Content-Type: - - text/event-stream - Date: - - Thu, 30 Apr 2026 00:45:25 GMT - Server: - - BaseHTTP/0.6 Python/3.12.3 status: code: 200 message: OK From 05333e42ba8a3265f69110b34fa17d49ac86b833 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 30 Apr 2026 18:07:43 +0000 Subject: [PATCH 03/30] tests(llm_translation): switch to pytest-recording for marker-based bulk capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per Yuneng's feedback, use a single @pytest.mark.vcr marker so one record sweep populates cassettes for every marked test across all providers, instead of forcing each test to bind to a hard-coded cassette path. Changes vs. the initial scaffolding: - Add 'pytest-recording==0.13.4' on top of vcrpy. Adopt its layout: cassettes live at 'cassettes//.yaml', resolved automatically. New tests just decorate with '@pytest.mark.vcr' — no imports or path bookkeeping. - Move the shared filter/match config into a 'vcr_config' fixture in 'tests/llm_translation/conftest.py' (consumed by pytest-recording for every marked test in the dir). Drop the standalone 'vcr_config.py'. - Bulk record / replay via the standard '--record-mode' CLI flag: 'make test-llm-translation-record' now sweeps every '@pytest.mark.vcr' test under tests/llm_translation in one shot. Optional 'TARGET=' var scopes to a single file. - Move existing cassettes to the per-test paths and update the local in-process Anthropic regenerator to write to the same paths. - Refresh README + Makefile target docs to match the sweep workflow. Co-authored-by: Mateo Wang --- Makefile | 19 ++- pyproject.toml | 1 + tests/llm_translation/Readme.md | 19 ++- tests/llm_translation/cassettes/README.md | 129 +++++++++++------- .../cassettes/_record_anthropic_fixtures.py | 37 +++-- ...st_anthropic_basic_completion_replay.yaml} | 0 ...nthropic_streaming_completion_replay.yaml} | 0 tests/llm_translation/conftest.py | 88 ++++++++++++ .../test_anthropic_completion_vcr.py | 59 ++++---- tests/llm_translation/vcr_config.py | 123 ----------------- uv.lock | 17 ++- 11 files changed, 263 insertions(+), 229 deletions(-) rename tests/llm_translation/cassettes/{anthropic_basic_completion.yaml => test_anthropic_completion_vcr/test_anthropic_basic_completion_replay.yaml} (100%) rename tests/llm_translation/cassettes/{anthropic_streaming_completion.yaml => test_anthropic_completion_vcr/test_anthropic_streaming_completion_replay.yaml} (100%) delete mode 100644 tests/llm_translation/vcr_config.py diff --git a/Makefile b/Makefile index 0e5af3e17c..8b9e10fa63 100644 --- a/Makefile +++ b/Makefile @@ -187,16 +187,15 @@ test-llm-translation-single: install-test-deps -v --tb=short --maxfail=100 --timeout=300 # VCR cassette helpers -------------------------------------------------------- -# Re-record a single VCR-backed translation test file against the live API. -# Provider credentials must be exported (e.g. ANTHROPIC_API_KEY). +# Sweep-record every @pytest.mark.vcr test under tests/llm_translation in one +# shot. Provider credentials must be exported for the providers exercised +# (e.g. ANTHROPIC_API_KEY, OPENAI_API_KEY, AWS_*). # -# Example: +# Examples: +# ANTHROPIC_API_KEY=sk-ant-... make test-llm-translation-record # ANTHROPIC_API_KEY=sk-ant-... make test-llm-translation-record \ -# FILE=test_anthropic_completion_vcr.py +# TARGET=test_anthropic_completion_vcr.py +TARGET ?= . test-llm-translation-record: install-test-deps - @if [ -z "$(FILE)" ]; then \ - echo "Usage: make test-llm-translation-record FILE=test_filename.py"; \ - exit 1; \ - fi - LITELLM_VCR_RECORD_MODE=once \ - $(UV_RUN) pytest tests/llm_translation/$(FILE) -v --tb=short + $(UV_RUN) pytest tests/llm_translation/$(TARGET) \ + -m vcr --record-mode=once -v --tb=short diff --git a/pyproject.toml b/pyproject.toml index 7db6c6d9b1..34003a19a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -150,6 +150,7 @@ dev = [ "openapi-core==0.22.0; python_version < '3.14'", "pytest-timeout==2.4.0", "vcrpy==8.1.1", + "pytest-recording==0.13.4", ] proxy-dev = [ "prisma==0.11.0", diff --git a/tests/llm_translation/Readme.md b/tests/llm_translation/Readme.md index b5a76e48ea..a7fcfe072a 100644 --- a/tests/llm_translation/Readme.md +++ b/tests/llm_translation/Readme.md @@ -4,15 +4,24 @@ Name of the test file is the name of the LLM provider - e.g. `test_openai.py` is ## VCR-backed tests -Files matching `*_vcr.py` (e.g. `test_anthropic_completion_vcr.py`) replay -recorded HTTP traffic from `cassettes/` instead of calling the real provider. -They run offline by default — no API keys required, no per-PR cost. +Tests decorated with `@pytest.mark.vcr` (typically in `*_vcr.py` files, +e.g. `test_anthropic_completion_vcr.py`) replay recorded HTTP traffic from +`cassettes/` via [`pytest-recording`](https://github.com/kiwicom/pytest-recording) +instead of calling the real provider. They run offline by default — no API +keys required, no per-PR cost. -To re-record against the live API: +To re-record every marked test in one sweep: + +```bash +ANTHROPIC_API_KEY=sk-ant-... OPENAI_API_KEY=sk-... \ + make test-llm-translation-record +``` + +To scope to a single file: ```bash ANTHROPIC_API_KEY=sk-ant-... \ - make test-llm-translation-record FILE=test_anthropic_completion_vcr.py + make test-llm-translation-record TARGET=test_anthropic_completion_vcr.py ``` See [`cassettes/README.md`](./cassettes/README.md) for the full workflow, diff --git a/tests/llm_translation/cassettes/README.md b/tests/llm_translation/cassettes/README.md index 0820d01ebd..eff373a768 100644 --- a/tests/llm_translation/cassettes/README.md +++ b/tests/llm_translation/cassettes/README.md @@ -11,70 +11,105 @@ and producing flaky CI on outages. Recording the HTTP exchange once and replaying it on subsequent runs gives us realistic provider responses (streaming, headers, edge-case payloads) at zero per-PR cost. -## How to add a new cassette-backed test +## Layout + +We use [`pytest-recording`](https://github.com/kiwicom/pytest-recording), +which auto-resolves the cassette path from the test location: + +``` +tests/llm_translation/ + cassettes/ + / + .yaml + test__completion_vcr.py + conftest.py # provides the shared vcr_config fixture +``` + +For example, a test +`tests/llm_translation/test_anthropic_completion_vcr.py::test_basic` is backed by +`tests/llm_translation/cassettes/test_anthropic_completion_vcr/test_basic.yaml`. + +## Adding a new cassette-backed test 1. Pick a small, deterministic call. Avoid prompts whose output depends on wall-clock time, randomness, or live web data. -2. Add a test in a `*_vcr.py` file under `tests/llm_translation/`. Wrap it - with `@litellm_vcr.use_cassette(".yaml")` from - `tests/llm_translation/vcr_config.py`. -3. Record the cassette once: +2. Write the test as you normally would and decorate it with + `@pytest.mark.vcr`. No imports beyond `pytest` are needed — the + `vcr_config` fixture in `conftest.py` is applied automatically. +3. Run the sweep recorder once with the credentials you need. Recording is + strictly opt-in via `--record-mode=once`; the default replay mode never + touches the network. - ```bash - LITELLM_VCR_RECORD_MODE=once \ - ANTHROPIC_API_KEY=sk-ant-... \ - uv run pytest tests/llm_translation/test_my_provider_vcr.py::test_my_case -v - ``` +## Bulk re-record (the common path) - or, equivalently: - - ```bash - ANTHROPIC_API_KEY=sk-ant-... \ - make test-llm-translation-record FILE=test_my_provider_vcr.py - ``` - -4. Inspect the resulting YAML file: - - **Strip any secrets** that survived `vcr_config.py`'s header filter. - `vcr_config.py` already removes the common ones (`Authorization`, - `x-api-key`, `cookie`, AWS sigv4 headers, etc.) — but a request *body* - might contain a token if your test passed one inline. - - Trim very large response bodies if they aren't load-bearing for the - assertion. -5. Commit the cassette alongside the test. - -## Re-recording - -Run the same `make test-llm-translation-record` command. vcrpy's `once` mode -will *not* overwrite an existing cassette — delete the file first if you're -intentionally refreshing it: +A single sweep replays every `@pytest.mark.vcr` test under +`tests/llm_translation`, hitting the live provider only for tests that don't +yet have a cassette: ```bash -rm tests/llm_translation/cassettes/anthropic_basic_completion.yaml -ANTHROPIC_API_KEY=sk-ant-... make test-llm-translation-record \ - FILE=test_anthropic_completion_vcr.py +ANTHROPIC_API_KEY=sk-ant-... \ +OPENAI_API_KEY=sk-... \ +AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... \ + make test-llm-translation-record ``` -## Refreshing the canned Anthropic fixtures +Or scope it to a single file: -The two Anthropic cassettes in this directory -(`anthropic_basic_completion.yaml` and `anthropic_streaming_completion.yaml`) -are recorded against an in-process mock so contributors can regenerate them -without an `ANTHROPIC_API_KEY`: +```bash +ANTHROPIC_API_KEY=sk-ant-... \ + make test-llm-translation-record TARGET=test_anthropic_completion_vcr.py +``` + +vcrpy's `once` record mode does **not** overwrite an existing cassette — +delete the file first if you're intentionally refreshing it: + +```bash +rm tests/llm_translation/cassettes/test_anthropic_completion_vcr/test_basic.yaml +ANTHROPIC_API_KEY=sk-ant-... \ + make test-llm-translation-record TARGET=test_anthropic_completion_vcr.py +``` + +To force a full refresh of every cassette in one shot: + +```bash +rm -rf tests/llm_translation/cassettes/test_* +ANTHROPIC_API_KEY=... OPENAI_API_KEY=... AWS_* \ + uv run pytest tests/llm_translation -m vcr --record-mode=all -v +``` + +## Refreshing the canned Anthropic fixtures (no API key) + +The two Anthropic cassettes shipped with this directory are recorded against +an in-process mock so contributors can regenerate them without an +`ANTHROPIC_API_KEY`: ```bash uv run python tests/llm_translation/cassettes/_record_anthropic_fixtures.py ``` For a full refresh against the real API, delete the cassettes first and use -the `LITELLM_VCR_RECORD_MODE=once` path with a real key. +the bulk-record sweep above. + +## Cassette hygiene + +After recording, **always inspect the YAML before committing**: + +- The `vcr_config` fixture in `conftest.py` already filters the common + request headers (`Authorization`, `x-api-key`, `anthropic-api-key`, AWS + sigv4 headers, cookies, GCP keys, …) and per-request response headers + (`set-cookie`, `cf-ray`, request IDs, org IDs, dates). +- A request *body* might still contain a token if your test passed one + inline — scrub it manually. +- Quick sanity check: `grep -i 'sk-\|bearer\|api-key' cassettes//*.yaml` + should be clean. +- Trim unhelpful response bodies if they're megabytes large but the + assertion only needs a few fields. ## Don't -- Don't commit cassettes containing real API keys, OAuth tokens, or PII. - When in doubt, `grep -i 'sk-\|bearer\|api-key' cassettes/*.yaml` after - recording. +- Don't commit cassettes with real API keys, OAuth tokens, or PII. - Don't rely on cassettes for tests of *non-deterministic* behavior - (rate-limit retries, timeouts, the model itself making a creative choice). - Mock those at the LiteLLM layer instead. -- Don't record both real and mock host names into the same cassette without - rewriting the URL — vcrpy matches on host/port by default. + (rate-limit retries, timeouts, model creativity). Mock those at the + LiteLLM layer instead. +- Don't manually edit cassette YAML beyond scrubbing — the format is + byte-sensitive (e.g. content-length headers must match the body). diff --git a/tests/llm_translation/cassettes/_record_anthropic_fixtures.py b/tests/llm_translation/cassettes/_record_anthropic_fixtures.py index c17c56169e..91472fbbd1 100644 --- a/tests/llm_translation/cassettes/_record_anthropic_fixtures.py +++ b/tests/llm_translation/cassettes/_record_anthropic_fixtures.py @@ -1,9 +1,8 @@ """Helper script that records Anthropic-shaped cassettes against a local mock. -This is a *one-shot* utility, not a test. It exists so we can deterministically -regenerate the canned Anthropic cassettes shipped under -``tests/llm_translation/cassettes/`` without spending real provider credits and -without needing an ``ANTHROPIC_API_KEY``. +This is a *one-shot* utility, not a test. It exists so contributors can +regenerate the canned Anthropic cassettes shipped with this PR without +spending real provider credits and without needing an ``ANTHROPIC_API_KEY``. Run it with:: @@ -17,9 +16,14 @@ The script: 3. Rewrites the cassette URL/Host so replay matches genuine ``https://api.anthropic.com/v1/messages`` traffic. -If you want to refresh against the *real* Anthropic API instead, use the -``LITELLM_VCR_RECORD_MODE=once`` workflow described in -``tests/llm_translation/vcr_config.py`` — that path needs a real API key. +The cassettes are written to the per-test paths that ``pytest-recording`` +expects (``cassettes//.yaml``) so the existing tests +in ``test_anthropic_completion_vcr.py`` pick them up unchanged. + +For a refresh against the *real* Anthropic API, use the +``--record-mode=once`` sweep described in +``tests/llm_translation/cassettes/README.md`` — that path needs a real +``ANTHROPIC_API_KEY``. """ from __future__ import annotations @@ -40,7 +44,7 @@ sys.path.insert(0, str(REPO_ROOT)) import litellm # noqa: E402 -CASSETTE_DIR = Path(__file__).parent +CASSETTE_DIR = Path(__file__).parent / "test_anthropic_completion_vcr" MOCK_HOST = "127.0.0.1" NON_STREAM_PORT = 18765 STREAM_PORT = 18766 @@ -201,14 +205,18 @@ def _consume(iterable: Iterable[Any]) -> None: def record_non_streaming() -> None: - cassette = CASSETTE_DIR / "anthropic_basic_completion.yaml" + cassette = CASSETTE_DIR / "test_anthropic_basic_completion_replay.yaml" if cassette.exists(): cassette.unlink() server = _serve(NON_STREAM_PORT, "json") try: my_vcr = vcr.VCR( record_mode="all", - filter_headers=["authorization", "x-api-key", "anthropic-version"], + filter_headers=[ + "authorization", + "x-api-key", + "anthropic-version", + ], ) with my_vcr.use_cassette(str(cassette)): response = litellm.completion( @@ -224,14 +232,18 @@ def record_non_streaming() -> None: def record_streaming() -> None: - cassette = CASSETTE_DIR / "anthropic_streaming_completion.yaml" + cassette = CASSETTE_DIR / "test_anthropic_streaming_completion_replay.yaml" if cassette.exists(): cassette.unlink() server = _serve(STREAM_PORT, "stream") try: my_vcr = vcr.VCR( record_mode="all", - filter_headers=["authorization", "x-api-key", "anthropic-version"], + filter_headers=[ + "authorization", + "x-api-key", + "anthropic-version", + ], ) with my_vcr.use_cassette(str(cassette)): stream = litellm.completion( @@ -249,6 +261,7 @@ def record_streaming() -> None: def main() -> None: os.environ.setdefault("LITELLM_LOG", "WARNING") + CASSETTE_DIR.mkdir(parents=True, exist_ok=True) record_non_streaming() record_streaming() print(f"Wrote cassettes to {CASSETTE_DIR}") diff --git a/tests/llm_translation/cassettes/anthropic_basic_completion.yaml b/tests/llm_translation/cassettes/test_anthropic_completion_vcr/test_anthropic_basic_completion_replay.yaml similarity index 100% rename from tests/llm_translation/cassettes/anthropic_basic_completion.yaml rename to tests/llm_translation/cassettes/test_anthropic_completion_vcr/test_anthropic_basic_completion_replay.yaml diff --git a/tests/llm_translation/cassettes/anthropic_streaming_completion.yaml b/tests/llm_translation/cassettes/test_anthropic_completion_vcr/test_anthropic_streaming_completion_replay.yaml similarity index 100% rename from tests/llm_translation/cassettes/anthropic_streaming_completion.yaml rename to tests/llm_translation/cassettes/test_anthropic_completion_vcr/test_anthropic_streaming_completion_replay.yaml diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index d315dc63bc..831e62d67e 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -18,6 +18,94 @@ import litellm import asyncio + +# --------------------------------------------------------------------------- +# VCR cassette infrastructure (pytest-recording) +# --------------------------------------------------------------------------- +# Tests marked with ``@pytest.mark.vcr`` replay HTTP traffic from a cassette +# under ``cassettes//.yaml`` instead of hitting the +# live provider. Default record mode is ``none`` (replay only) so CI never +# accidentally calls a real LLM. To re-record every marked test in one sweep:: +# +# ANTHROPIC_API_KEY=sk-ant-... \ +# uv run pytest tests/llm_translation -m vcr --record-mode=once +# +# See ``tests/llm_translation/cassettes/README.md`` for the full workflow. + +# Headers that must never be persisted to a cassette. Matched +# case-insensitively by vcrpy. +_FILTERED_REQUEST_HEADERS = ( + "authorization", + "x-api-key", + "anthropic-api-key", + "openai-api-key", + "azure-api-key", + "api-key", + "cookie", + "x-amz-security-token", + "x-amz-date", + "x-amz-content-sha256", + "amz-sdk-invocation-id", + "amz-sdk-request", + "x-goog-api-key", + "x-goog-user-project", +) + +# Per-request response headers we strip so cassettes diff cleanly across +# re-records. +_FILTERED_RESPONSE_HEADERS = ( + "set-cookie", + "x-request-id", + "request-id", + "cf-ray", + "anthropic-organization-id", + "openai-organization", + "x-amzn-requestid", + "x-amzn-trace-id", + "date", +) + + +def _scrub_response(response): + """Strip per-request response headers we don't want in the cassette.""" + if not isinstance(response, dict): + return response + headers = response.get("headers") or {} + if isinstance(headers, dict): + for header in list(headers): + if header.lower() in _FILTERED_RESPONSE_HEADERS: + headers.pop(header, None) + return response + + +@pytest.fixture(scope="module") +def vcr_config(): + """Shared VCR config consumed by ``pytest-recording``. + + Applied to every ``@pytest.mark.vcr`` test in this directory. + """ + return { + "filter_headers": list(_FILTERED_REQUEST_HEADERS), + "decode_compressed_response": True, + # Match on full request shape so streaming vs non-streaming and + # different prompts produce distinct cassettes. + "match_on": ( + "method", + "scheme", + "host", + "port", + "path", + "query", + "body", + ), + "before_record_response": _scrub_response, + } + + +# pytest-recording's default cassette dir is +# ``/cassettes/``. Keep that — it gives every test its +# own file and avoids name collisions across modules. + # --------------------------------------------------------------------------- # Capture TRUE defaults at conftest import time (before test modules pollute). # --------------------------------------------------------------------------- diff --git a/tests/llm_translation/test_anthropic_completion_vcr.py b/tests/llm_translation/test_anthropic_completion_vcr.py index 6367e65e01..04f2914deb 100644 --- a/tests/llm_translation/test_anthropic_completion_vcr.py +++ b/tests/llm_translation/test_anthropic_completion_vcr.py @@ -1,21 +1,21 @@ """ -VCR-backed Anthropic completion tests. +Cassette-replayed Anthropic completion tests. -These tests exercise the same end-to-end ``litellm.completion`` code paths -as ``test_anthropic_completion.py`` but replay HTTP traffic from cassettes -under ``cassettes/`` instead of calling ``api.anthropic.com``. CI can run -them with no API key and zero cost. +These tests exercise the same end-to-end ``litellm.completion`` code paths as +``test_anthropic_completion.py`` but replay HTTP traffic from cassettes under +``cassettes/test_anthropic_completion_vcr/`` instead of calling +``api.anthropic.com``. CI runs them with no API key and zero cost. -To re-record after a deliberate change to request shape (or to refresh -against the live API), set ``LITELLM_VCR_RECORD_MODE=once`` and provide a -real ``ANTHROPIC_API_KEY``:: +Add a new test by writing it normally and decorating with ``@pytest.mark.vcr``. +The cassette path is resolved automatically from the test module + test name +by ``pytest-recording`` (see ``conftest.py``). - LITELLM_VCR_RECORD_MODE=once \\ - ANTHROPIC_API_KEY=sk-ant-... \\ - uv run pytest tests/llm_translation/test_anthropic_completion_vcr.py -v +To re-record every marked test in one sweep:: -See ``tests/llm_translation/vcr_config.py`` and ``tests/llm_translation/cassettes/README.md`` -for the full workflow. + ANTHROPIC_API_KEY=sk-ant-... \\ + uv run pytest tests/llm_translation -m vcr --record-mode=once + +See ``tests/llm_translation/cassettes/README.md`` for the full workflow. """ import os @@ -24,39 +24,35 @@ import sys import pytest sys.path.insert(0, os.path.abspath("../..")) -sys.path.insert(0, os.path.dirname(__file__)) import litellm # noqa: E402 -from vcr_config import litellm_vcr # noqa: E402 - - -# A non-secret placeholder API key. We never want a real key written to a -# cassette, and ``vcr_config`` filters Authorization / x-api-key headers -# anyway. Using a deterministic placeholder also stops the SDK from raising -# when ``ANTHROPIC_API_KEY`` is unset (the common CI case). +# A non-secret placeholder API key. The vcr_config fixture in conftest.py +# filters Authorization / x-api-key headers from cassettes, so this value +# never lands on disk; it only stops the SDK from raising when +# ``ANTHROPIC_API_KEY`` is unset (the common CI case). PLACEHOLDER_ANTHROPIC_API_KEY = "sk-ant-vcr-placeholder" @pytest.fixture(autouse=True) def _placeholder_anthropic_key(monkeypatch): - """Provide a placeholder key when none is set so replay works offline. + """Ensure an API key is set so replay works offline. - If a real key is present in the environment (e.g. when re-recording), - we leave it untouched. + If a real key is present (e.g. when re-recording with + ``--record-mode=once``), we leave it untouched. """ if not os.environ.get("ANTHROPIC_API_KEY"): monkeypatch.setenv("ANTHROPIC_API_KEY", PLACEHOLDER_ANTHROPIC_API_KEY) -@litellm_vcr.use_cassette("anthropic_basic_completion.yaml") +@pytest.mark.vcr def test_anthropic_basic_completion_replay(): """Smoke-test that a vanilla Anthropic completion replays from a cassette. - This is the canonical example for the cassette-based testing pattern: - no API key required at runtime, deterministic output, and the full - LiteLLM transformation pipeline (request shaping + response parsing) - runs against a real-shape Anthropic payload. + This is the canonical example for the cassette-based testing pattern: no + API key required at runtime, deterministic output, and the full LiteLLM + transformation pipeline (request shaping + response parsing) runs against + a real-shape Anthropic payload. """ response = litellm.completion( model="anthropic/claude-sonnet-4-5-20250929", @@ -71,13 +67,14 @@ def test_anthropic_basic_completion_replay(): assert response.choices[0].finish_reason == "stop" -@litellm_vcr.use_cassette("anthropic_streaming_completion.yaml") +@pytest.mark.vcr def test_anthropic_streaming_completion_replay(): """Replay a streaming Anthropic completion from a cassette. Exercises the SSE chunk parser and the public streaming surface. The underlying cassette captures every ``content_block_delta`` event Anthropic - emits, so any regression in the streaming transformation will surface here. + emits, so any regression in the streaming transformation will surface + here. """ stream = litellm.completion( model="anthropic/claude-sonnet-4-5-20250929", diff --git a/tests/llm_translation/vcr_config.py b/tests/llm_translation/vcr_config.py deleted file mode 100644 index 7e641c9cad..0000000000 --- a/tests/llm_translation/vcr_config.py +++ /dev/null @@ -1,123 +0,0 @@ -""" -Shared VCR configuration for ``tests/llm_translation``. - -This module centralises the cassette setup used by tests that would otherwise -hit a real LLM provider over the network. The goal is to let CI replay -recorded HTTP traffic by default — no API keys required — and to provide a -single switch for re-recording cassettes against the live provider. - -Usage in a test:: - - from .vcr_config import litellm_vcr # noqa: E402 - - @litellm_vcr.use_cassette("anthropic_basic_completion.yaml") - def test_basic_completion(): - resp = litellm.completion( - model="anthropic/claude-sonnet-4-5-20250929", - messages=[{"role": "user", "content": "Hello!"}], - ) - assert resp.choices[0].message.content - -Recording mode --------------- -By default the cassette is replayed (``record_mode='none'``). To re-record: - - LITELLM_VCR_RECORD_MODE=once \\ - ANTHROPIC_API_KEY=sk-ant-... \\ - uv run pytest tests/llm_translation/test_anthropic_completion_vcr.py - -Valid values for ``LITELLM_VCR_RECORD_MODE`` mirror vcrpy's record modes: -``none`` (replay only — fail on missing cassette), ``once`` (record if the -cassette doesn't exist), ``new_episodes`` (append new interactions), and -``all`` (always re-record). See the vcrpy docs for details. - -Why this exists ---------------- -Per the discussion that produced LIT-2683, our e2e tests repeatedly drained -provider billing accounts and produced flaky CI on outages. Recording the -HTTP exchange once and replaying it on subsequent runs gives us realistic -provider responses (including streaming, headers, and edge-case payloads) -without per-PR cost or rate-limit risk. Re-record periodically to catch -real provider drift. -""" - -from __future__ import annotations - -import os -from pathlib import Path -from typing import Any - -import vcr - -CASSETTE_DIR: Path = Path(__file__).parent / "cassettes" - -# Headers that must never be persisted to a cassette. These are matched -# case-insensitively by vcrpy. -_FILTERED_REQUEST_HEADERS = ( - "authorization", - "x-api-key", - "anthropic-api-key", - "openai-api-key", - "azure-api-key", - "api-key", - "cookie", - "x-amz-security-token", - "x-amz-date", - "x-amz-content-sha256", - "amz-sdk-invocation-id", - "amz-sdk-request", -) - -_FILTERED_RESPONSE_HEADERS = ( - "set-cookie", - "x-request-id", - "cf-ray", - "anthropic-organization-id", - "openai-organization", - "request-id", -) - - -def _record_mode() -> str: - """Resolve the active vcrpy record mode from the environment. - - Defaults to ``"none"`` so CI never accidentally hits the live provider. - """ - mode = os.environ.get("LITELLM_VCR_RECORD_MODE", "none").strip().lower() - if mode not in {"none", "once", "new_episodes", "all"}: - raise ValueError( - f"LITELLM_VCR_RECORD_MODE={mode!r} is not a valid vcrpy record mode." - ) - return mode - - -def _build_vcr() -> vcr.VCR: - """Construct the shared ``VCR`` instance used by translation tests.""" - return vcr.VCR( - cassette_library_dir=str(CASSETTE_DIR), - record_mode=_record_mode(), - # Match on method + URI + body so streaming vs non-streaming and - # different prompts get distinct cassettes. - match_on=("method", "scheme", "host", "port", "path", "query", "body"), - filter_headers=list(_FILTERED_REQUEST_HEADERS), - decode_compressed_response=True, - ) - - -def _scrub_response(response: Any) -> Any: - """Strip per-request response headers we don't want in the cassette.""" - if not isinstance(response, dict): - return response - headers = response.get("headers") or {} - if isinstance(headers, dict): - for header in list(headers): - if header.lower() in _FILTERED_RESPONSE_HEADERS: - headers.pop(header, None) - return response - - -litellm_vcr: vcr.VCR = _build_vcr() -litellm_vcr.before_record_response = _scrub_response - - -__all__ = ["litellm_vcr", "CASSETTE_DIR"] diff --git a/uv.lock b/uv.lock index b2e8aef8a2..7308122e42 100644 --- a/uv.lock +++ b/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-04-27T00:38:13.673780212Z" +exclude-newer = "2026-04-27T18:03:18.987479976Z" exclude-newer-span = "P3D" [manifest] @@ -3231,6 +3231,7 @@ dev = [ { name = "pytest-cov" }, { name = "pytest-mock" }, { name = "pytest-postgresql" }, + { name = "pytest-recording" }, { name = "pytest-rerunfailures" }, { name = "pytest-timeout" }, { name = "pytest-xdist" }, @@ -3385,6 +3386,7 @@ dev = [ { name = "pytest-cov", specifier = "==5.0.0" }, { name = "pytest-mock", specifier = "==3.15.1" }, { name = "pytest-postgresql", specifier = "==7.0.2" }, + { name = "pytest-recording", specifier = "==0.13.4" }, { name = "pytest-rerunfailures", specifier = "==15.1" }, { name = "pytest-timeout", specifier = "==2.4.0" }, { name = "pytest-xdist", specifier = "==3.8.0" }, @@ -5922,6 +5924,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/57/f2db5a80b10c3ac48ce41786cb9b14172f997509ee1b1055ab7db4238e5e/pytest_postgresql-7.0.2-py3-none-any.whl", hash = "sha256:0b0d31c51620a9c1d6be93286af354256bc58a47c379f56f4147b22da6e81fb5", size = 41447, upload-time = "2025-05-17T20:17:58.011Z" }, ] +[[package]] +name = "pytest-recording" +version = "0.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "vcrpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/32/9c/f4027c5f1693847b06d11caf4b4f6bb09f22c1581ada4663877ec166b8c6/pytest_recording-0.13.4.tar.gz", hash = "sha256:568d64b2a85992eec4ae0a419c855d5fd96782c5fb016784d86f18053792768c", size = 26576, upload-time = "2025-05-08T10:41:11.231Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/c2/ce34735972cc42d912173e79f200fe66530225190c06655c5632a9d88f1e/pytest_recording-0.13.4-py3-none-any.whl", hash = "sha256:ad49a434b51b1c4f78e85b1e6b74fdcc2a0a581ca16e52c798c6ace971f7f439", size = 13723, upload-time = "2025-05-08T10:41:09.684Z" }, +] + [[package]] name = "pytest-rerunfailures" version = "15.1" From 0e880dc83628e1ab7c3cf4a72b330d534d001e2a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 30 Apr 2026 18:11:44 +0000 Subject: [PATCH 04/30] tests(llm_translation): add pytest-recording to license allowlist + greptile fixes CI's license check fails on the new dev dep because liccheck cannot read the PEP 639 'License-Expression' field that pytest-recording uses. Add the package to the manually-verified allowlist (MIT, confirmed via PyPI classifier). Also addresses greptile P2 review comments: - Add 'anthropic-version' to the request-header filter list so live and mock recordings produce structurally identical cassettes. - Replace the indentation-sensitive regex in '_strip_nondeterministic_headers' with a YAML parse-and-rewrite so the helper keeps working if vcrpy ever changes its serialization style. Co-authored-by: Mateo Wang --- tests/code_coverage_tests/liccheck.ini | 1 + .../cassettes/_record_anthropic_fixtures.py | 29 +++++++++---------- tests/llm_translation/conftest.py | 3 ++ 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/tests/code_coverage_tests/liccheck.ini b/tests/code_coverage_tests/liccheck.ini index 4ecc0d0bc9..344e38da83 100644 --- a/tests/code_coverage_tests/liccheck.ini +++ b/tests/code_coverage_tests/liccheck.ini @@ -169,3 +169,4 @@ langchain-mcp-adapters: >=0.2.1 # MIT License langgraph: >=1.0.10 # MIT License langgraph-prebuilt: >=1.0.8 # MIT License - https://github.com/langchain-ai/langgraph/blob/main/LICENSE pytest-rerunfailures: >=15.1 # MPL 2.0 license +pytest-recording: >=0.13.4 # MIT license diff --git a/tests/llm_translation/cassettes/_record_anthropic_fixtures.py b/tests/llm_translation/cassettes/_record_anthropic_fixtures.py index 91472fbbd1..81321b992e 100644 --- a/tests/llm_translation/cassettes/_record_anthropic_fixtures.py +++ b/tests/llm_translation/cassettes/_record_anthropic_fixtures.py @@ -30,7 +30,6 @@ from __future__ import annotations import json import os -import re import sys import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -38,6 +37,7 @@ from pathlib import Path from typing import Any, Iterable import vcr # type: ignore[import-not-found] +import yaml # type: ignore[import-not-found] REPO_ROOT = Path(__file__).resolve().parents[3] sys.path.insert(0, str(REPO_ROOT)) @@ -173,21 +173,18 @@ _NON_DETERMINISTIC_HEADERS = ("Date", "Server") def _strip_nondeterministic_headers(path: Path) -> None: - """Remove headers whose values change every run from the cassette.""" - text = path.read_text() - for header in _NON_DETERMINISTIC_HEADERS: - # Matches a YAML block like:: - # - # Date: - # - Thu, 30 Apr 2026 00:43:16 GMT - # - # under the response ``headers:`` mapping. Indentation is fixed by vcrpy. - pattern = re.compile( - rf"^ {re.escape(header)}:\n - .*\n", - re.MULTILINE, - ) - text = pattern.sub("", text) - path.write_text(text) + """Remove headers whose values change every run from the cassette. + + Parses + rewrites the YAML rather than regex-substituting against a + fixed indentation, so this stays correct if vcrpy ever changes its + serialization style. + """ + cassette = yaml.safe_load(path.read_text()) + for interaction in cassette.get("interactions") or []: + headers = (interaction.get("response") or {}).get("headers") or {} + for header in _NON_DETERMINISTIC_HEADERS: + headers.pop(header, None) + path.write_text(yaml.safe_dump(cassette, default_flow_style=False, sort_keys=False)) def _rewrite_cassette_to_real_host(path: Path, mock_host_port: str) -> None: diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 831e62d67e..d944c9046c 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -38,6 +38,9 @@ _FILTERED_REQUEST_HEADERS = ( "authorization", "x-api-key", "anthropic-api-key", + # Strip ``anthropic-version`` so live-recorded and mock-recorded cassettes + # have the same shape (the local mock helper drops it too). + "anthropic-version", "openai-api-key", "azure-api-key", "api-key", From 33a051636d2f0b075dc09ae0ff3ff048f7848864 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Apr 2026 20:28:11 +0000 Subject: [PATCH 05/30] tests(llm_translation): add Redis cassette persister with 24h TTL Stores VCR cassettes in Redis under litellm:vcr:cassette: with a 24h expiry instead of YAML on disk. The TTL means each daily CI run starts with an aged-out cache, naturally re-records against live providers, and surfaces upstream API drift within a day without a manual `make` re-record sweep. Opt-in via LITELLM_VCR_REDIS=1; default behaviour is unchanged so local dev keeps the on-disk cassettes. before_record_response now drops non-2xx responses so a transient 5xx or 429 from a provider can't poison the cache for the rest of the TTL window. Vcr-marked tests bump litellm.num_retries to 3 during recording so provider-SDK exponential backoff kicks in on the cache-miss path. Tests cover the three surfaces we depend on in CI: serialize/deserialize roundtrip via the real vcrpy serializer, TTL is actually applied to saved keys, cache miss raises CassetteNotFoundError so vcrpy falls through to record mode, and 2xx-only filtering across the status-code matrix (2xx kept, 3xx/4xx/5xx dropped, with 429 and 503 explicitly pinned). --- tests/llm_translation/_vcr_redis_persister.py | 111 ++++++++++++++ tests/llm_translation/conftest.py | 53 ++++++- .../test_vcr_redis_persister.py | 137 ++++++++++++++++++ 3 files changed, 300 insertions(+), 1 deletion(-) create mode 100644 tests/llm_translation/_vcr_redis_persister.py create mode 100644 tests/llm_translation/test_vcr_redis_persister.py diff --git a/tests/llm_translation/_vcr_redis_persister.py b/tests/llm_translation/_vcr_redis_persister.py new file mode 100644 index 0000000000..88423393c5 --- /dev/null +++ b/tests/llm_translation/_vcr_redis_persister.py @@ -0,0 +1,111 @@ +"""Redis-backed cassette persister for vcrpy. + +Stores the same serialized cassette payload that ``FilesystemPersister`` +would write to disk, but under a Redis key with a 24h TTL. Cassettes +auto-expire so the next CI run after the rollover re-records against the +live provider, surfacing API drift within a day instead of waiting for a +human to refresh ``cassettes/*.yaml`` by hand. + +On a cache miss we raise ``CassetteNotFoundError``; vcrpy's record-mode +machinery catches that and falls through to a live HTTP call, which then +gets persisted via ``save_cassette``. Non-2xx responses are filtered out +upstream by ``conftest.before_record_response`` so a transient provider +failure can't poison the cache for 24h. +""" + +from __future__ import annotations + +import os +from typing import Any, Optional + +from vcr.persisters.filesystem import CassetteNotFoundError +from vcr.serialize import deserialize, serialize + +CASSETTE_TTL_SECONDS = 24 * 60 * 60 +REDIS_KEY_PREFIX = "litellm:vcr:cassette:" + + +def redis_key_for(cassette_path: str) -> str: + """Map a cassette file path to a stable Redis key. + + Uses the path relative to CWD so keys are stable across machines. + """ + rel = os.path.relpath(str(cassette_path)) + return f"{REDIS_KEY_PREFIX}{rel}" + + +def _build_default_client(): + import redis + + host = os.environ.get("REDIS_HOST") + if not host: + raise RuntimeError( + "REDIS_HOST is not set; cannot build Redis cassette persister" + ) + return redis.Redis( + host=host, + port=int(os.environ.get("REDIS_PORT", 6379)), + password=os.environ.get("REDIS_PASSWORD") or None, + socket_timeout=5, + socket_connect_timeout=5, + decode_responses=False, + ) + + +def make_redis_persister( + client: Optional[Any] = None, + ttl_seconds: int = CASSETTE_TTL_SECONDS, +): + """Build a vcrpy-compatible persister bound to a Redis client. + + The returned object exposes ``load_cassette`` / ``save_cassette`` and is + a drop-in replacement for ``vcr.persisters.filesystem.FilesystemPersister``. + Pass an explicit ``client`` in tests; production callers omit it and let + the persister build a client from ``REDIS_HOST`` / ``REDIS_PORT`` / + ``REDIS_PASSWORD``. + """ + redis_client = client if client is not None else _build_default_client() + + class _RedisPersister: + @staticmethod + def load_cassette(cassette_path, serializer): + data = redis_client.get(redis_key_for(cassette_path)) + if data is None: + raise CassetteNotFoundError() + if isinstance(data, bytes): + data = data.decode("utf-8") + return deserialize(data, serializer) + + @staticmethod + def save_cassette(cassette_path, cassette_dict, serializer): + data = serialize(cassette_dict, serializer) + payload = data.encode("utf-8") if isinstance(data, str) else data + redis_client.set( + redis_key_for(cassette_path), + payload, + ex=ttl_seconds, + ) + + return _RedisPersister + + +def filter_non_2xx_response(response): + """vcrpy ``before_record_response`` hook that drops non-2xx responses. + + Returning ``None`` tells vcrpy to skip persisting the response (see + ``vcr.cassette.Cassette.append``). This prevents transient 5xx/429 + failures from being baked into the cache for the rest of the TTL window. + """ + if not isinstance(response, dict): + return response + status = response.get("status") + code = None + if isinstance(status, dict): + code = status.get("code") + elif isinstance(status, int): + code = status + if code is None: + return response + if 200 <= int(code) < 300: + return response + return None diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index d944c9046c..0d9148a61a 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -14,10 +14,19 @@ import pytest sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path +# Make sibling helper modules under tests/llm_translation/ importable regardless +# of the directory pytest is invoked from. +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + import litellm import asyncio +from _vcr_redis_persister import ( # noqa: E402 (sibling module under conftest dir) + filter_non_2xx_response, + make_redis_persister, +) + # --------------------------------------------------------------------------- # VCR cassette infrastructure (pytest-recording) @@ -81,6 +90,16 @@ def _scrub_response(response): return response +def _before_record_response(response): + """Compose per-request scrubbing with the 2xx-only cache policy. + + Order matters: we scrub headers first so we don't leak request IDs even + on responses we end up dropping from the cassette mid-development. + """ + response = _scrub_response(response) + return filter_non_2xx_response(response) + + @pytest.fixture(scope="module") def vcr_config(): """Shared VCR config consumed by ``pytest-recording``. @@ -101,10 +120,22 @@ def vcr_config(): "query", "body", ), - "before_record_response": _scrub_response, + "before_record_response": _before_record_response, } +def pytest_recording_configure(config, vcr): + """Swap vcrpy's default filesystem persister for a Redis-backed one. + + Opt-in via ``LITELLM_VCR_REDIS=1`` so local dev keeps the YAML-on-disk + behaviour and CI (which sets the flag) gets a 24h-TTL cache that + auto-refreshes against live providers without manual ``make`` runs. + """ + if os.environ.get("LITELLM_VCR_REDIS") != "1": + return + vcr.register_persister(make_redis_persister()) + + # pytest-recording's default cassette dir is # ``/cassettes/``. Keep that — it gives every test its # own file and avoids name collisions across modules. @@ -187,6 +218,26 @@ def setup_and_teardown(event_loop): # Add event_loop as a dependency event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) +# Number of attempts a vcr-marked test gets when recording against a live +# provider. Replay-only runs never reach the network so this only matters on +# cache miss / record mode. Tenacity-style exponential backoff is provided by +# the underlying provider SDKs (openai, anthropic) when they see 429/5xx, so +# bumping num_retries propagates retry-with-backoff for free. +_VCR_RECORD_RETRIES = 3 + + +@pytest.fixture(autouse=True) +def _vcr_record_retries(setup_and_teardown, request): + """Configure record-time retries for ``@pytest.mark.vcr`` tests. + + Depends on ``setup_and_teardown`` so this runs *after* the per-test + ``importlib.reload(litellm)`` resets ``num_retries`` back to None. + """ + if request.node.get_closest_marker("vcr") is None: + return + litellm.num_retries = _VCR_RECORD_RETRIES + + def pytest_collection_modifyitems(config, items): # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests custom_logger_tests = [ diff --git a/tests/llm_translation/test_vcr_redis_persister.py b/tests/llm_translation/test_vcr_redis_persister.py new file mode 100644 index 0000000000..e9ba57ac73 --- /dev/null +++ b/tests/llm_translation/test_vcr_redis_persister.py @@ -0,0 +1,137 @@ +"""Tests for the Redis-backed vcrpy cassette persister. + +These cover the three behaviours we actually rely on in CI: + +1. ``save_cassette`` followed by ``load_cassette`` returns the same + request/response pairs (roundtrip via the real vcrpy serializer). +2. Saved keys expire after ~24h so the cache auto-refreshes against live + providers without manual ``make`` runs. +3. ``load_cassette`` raises ``CassetteNotFoundError`` on a miss, so vcrpy's + record-mode machinery falls through to a live HTTP call instead of + silently matching against an empty cassette. + +We also pin the 2xx-only filter so a transient 5xx/429 from the provider +can't be baked into the cache for the rest of the TTL window. +""" + +from __future__ import annotations + +import os +import sys + +import fakeredis +import pytest +from vcr.persisters.filesystem import CassetteNotFoundError +from vcr.request import Request +from vcr.serializers import yamlserializer + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from _vcr_redis_persister import ( # noqa: E402 + CASSETTE_TTL_SECONDS, + filter_non_2xx_response, + make_redis_persister, + redis_key_for, +) + + +def _sample_cassette_dict(): + """Build a minimal cassette payload that exercises serialize/deserialize.""" + request = Request( + method="POST", + uri="https://api.anthropic.com/v1/messages", + body=b'{"model":"claude","messages":[{"role":"user","content":"hi"}]}', + headers={"content-type": "application/json"}, + ) + response = { + "status": {"code": 200, "message": "OK"}, + "headers": {"content-type": ["application/json"]}, + # vcrpy stores response bodies as bytes; mirror that so the + # roundtrip assertion exercises real-world serialization shapes. + "body": {"string": b'{"id":"msg_1","type":"message"}'}, + } + return {"requests": [request], "responses": [response]} + + +def _persister_with_fake_redis(): + fake = fakeredis.FakeStrictRedis() + return fake, make_redis_persister(client=fake) + + +def test_save_then_load_roundtrips_cassette_content(): + """A saved cassette must come back from ``load_cassette`` identical to + what was put in. If serialize/deserialize ever drift (e.g. encoding bug) + every replay-mode test in the suite breaks; this catches it cheaply.""" + fake, persister = _persister_with_fake_redis() + cassette_path = "tests/llm_translation/cassettes/test_x/test_y.yaml" + + persister.save_cassette(cassette_path, _sample_cassette_dict(), yamlserializer) + requests, responses = persister.load_cassette(cassette_path, yamlserializer) + + assert len(requests) == 1 + assert len(responses) == 1 + assert requests[0].method == "POST" + assert requests[0].uri == "https://api.anthropic.com/v1/messages" + assert responses[0]["status"]["code"] == 200 + assert responses[0]["body"]["string"] == b'{"id":"msg_1","type":"message"}' + + +def test_saved_key_has_24h_ttl(): + """The whole point of the Redis backend is that entries auto-expire after + 24h so each daily CI run re-records against live providers. If the TTL + isn't being applied, the cache never refreshes and we silently mask + upstream API drift.""" + fake, persister = _persister_with_fake_redis() + cassette_path = "tests/llm_translation/cassettes/test_x/test_ttl.yaml" + + persister.save_cassette(cassette_path, _sample_cassette_dict(), yamlserializer) + + ttl = fake.ttl(redis_key_for(cassette_path)) + assert ttl > 0, "key was saved without an expiry — would never refresh" + assert ttl <= CASSETTE_TTL_SECONDS + assert ttl >= CASSETTE_TTL_SECONDS - 5 # allow tiny clock slack + + +def test_load_missing_key_raises_cassette_not_found(): + """Cache miss must surface as ``CassetteNotFoundError``. vcrpy's record + machinery catches that exception and falls through to the live HTTP + call; if we returned empty/None instead, vcrpy would treat it as a + cassette with zero matching requests and the test would fail with a + confusing ``CannotOverwriteExistingCassetteException``.""" + _, persister = _persister_with_fake_redis() + with pytest.raises(CassetteNotFoundError): + persister.load_cassette("never/recorded.yaml", yamlserializer) + + +@pytest.mark.parametrize( + ("status_code", "expect_dropped"), + [ + (200, False), + (201, False), + (204, False), + (299, False), + (300, True), + (400, True), + (401, True), + (404, True), + (429, True), # rate limit — must never be cached + (500, True), # transient 5xx — must never be cached + (502, True), + (503, True), + ], +) +def test_only_2xx_responses_are_cached(status_code, expect_dropped): + """Pin the cache-poisoning protection: a non-2xx must be dropped from + the cassette (returned as ``None`` from the hook) so a transient 429 + or 503 doesn't get pinned for the rest of the TTL window. 2xx + responses must pass through untouched.""" + response = { + "status": {"code": status_code, "message": "X"}, + "headers": {}, + "body": {"string": ""}, + } + result = filter_non_2xx_response(response) + if expect_dropped: + assert result is None + else: + assert result is response From c7d647b567b3349e282fa9b109b0119642b161c2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Apr 2026 21:40:58 +0000 Subject: [PATCH 06/30] tests: drop YAML cassettes, make Redis-backed VCR the default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes the YAML cassette feature entirely and replaces it with a Redis-only flow. Every test in tests/llm_translation/ and tests/llm_responses_api_testing/ is auto-marked @pytest.mark.vcr via conftest.pytest_collection_modifyitems, so any provider call lands in the Redis cache (litellm:vcr:cassette:, 24h TTL). First run records, runs within the day replay, day rollover re-records and surfaces upstream API drift within 24h. VCR is on by default. Set LITELLM_VCR_DISABLE=1, or simply leave REDIS_HOST unset, to opt out — both bypass the auto-marker entirely so nothing about cassettes runs. record_mode is "once" so cache-miss records and cache-hit replays. The 8 existing respx-using files in tests/llm_translation are excluded from the auto-marker (vcrpy and respx both patch the httpx transport; applying both makes one silently win). The persister's own unit-test file is also excluded so it doesn't recursively run inside a cassette. The persister moved from tests/llm_translation/_vcr_redis_persister.py to tests/_vcr_redis_persister.py so both conftests share it. The two demo tests in test_anthropic_completion_vcr.py were ported into test_anthropic_completion.py and the demo file was deleted. Adds tests/_flush_vcr_cache.py + a Make target (test-llm-translation-flush-vcr-cache) that scans litellm:vcr:cassette:* and pipelines DELETEs, for the "I want the next CI run to re-record now" workflow. Drops the now-dead test-llm-translation-record target. Provider keys are still required on cache-miss (which happens on first run and once a day after that). Replay-mode runs need only Redis. --- Makefile | 21 +- tests/_flush_vcr_cache.py | 53 ++++ .../_vcr_redis_persister.py | 0 tests/llm_responses_api_testing/conftest.py | 109 ++++++- tests/llm_translation/Readme.md | 44 +-- tests/llm_translation/cassettes/README.md | 115 -------- .../cassettes/_record_anthropic_fixtures.py | 268 ------------------ ...est_anthropic_basic_completion_replay.yaml | 41 --- ...anthropic_streaming_completion_replay.yaml | 81 ------ tests/llm_translation/conftest.py | 111 +++++--- .../test_anthropic_completion.py | 48 ++++ .../test_anthropic_completion_vcr.py | 97 ------- .../test_vcr_redis_persister.py | 6 +- 13 files changed, 322 insertions(+), 672 deletions(-) create mode 100644 tests/_flush_vcr_cache.py rename tests/{llm_translation => }/_vcr_redis_persister.py (100%) delete mode 100644 tests/llm_translation/cassettes/README.md delete mode 100644 tests/llm_translation/cassettes/_record_anthropic_fixtures.py delete mode 100644 tests/llm_translation/cassettes/test_anthropic_completion_vcr/test_anthropic_basic_completion_replay.yaml delete mode 100644 tests/llm_translation/cassettes/test_anthropic_completion_vcr/test_anthropic_streaming_completion_replay.yaml delete mode 100644 tests/llm_translation/test_anthropic_completion_vcr.py diff --git a/Makefile b/Makefile index 8b9e10fa63..125b0e749d 100644 --- a/Makefile +++ b/Makefile @@ -186,16 +186,11 @@ test-llm-translation-single: install-test-deps --junitxml=test-results/junit.xml \ -v --tb=short --maxfail=100 --timeout=300 -# VCR cassette helpers -------------------------------------------------------- -# Sweep-record every @pytest.mark.vcr test under tests/llm_translation in one -# shot. Provider credentials must be exported for the providers exercised -# (e.g. ANTHROPIC_API_KEY, OPENAI_API_KEY, AWS_*). -# -# Examples: -# ANTHROPIC_API_KEY=sk-ant-... make test-llm-translation-record -# ANTHROPIC_API_KEY=sk-ant-... make test-llm-translation-record \ -# TARGET=test_anthropic_completion_vcr.py -TARGET ?= . -test-llm-translation-record: install-test-deps - $(UV_RUN) pytest tests/llm_translation/$(TARGET) \ - -m vcr --record-mode=once -v --tb=short +# VCR cache helpers ----------------------------------------------------------- +# Drop every Redis key under the ``litellm:vcr:cassette:*`` prefix. Use this +# when you want the next CI run (or local run) to re-record against live +# providers immediately instead of waiting for the 24h TTL to roll over. +# Reads REDIS_HOST / REDIS_PORT / REDIS_PASSWORD from the environment, the +# same vars CircleCI uses for its other Redis-backed jobs. +test-llm-translation-flush-vcr-cache: + $(UV_RUN) python tests/_flush_vcr_cache.py diff --git a/tests/_flush_vcr_cache.py b/tests/_flush_vcr_cache.py new file mode 100644 index 0000000000..4bb3fd7ba8 --- /dev/null +++ b/tests/_flush_vcr_cache.py @@ -0,0 +1,53 @@ +"""Flush every VCR cassette stored in Redis. + +Run via ``make test-llm-translation-flush-vcr-cache``. Use when you want the +next test run to re-record against live providers right now instead of +waiting for the 24h TTL to expire. + +Reads ``REDIS_HOST``, ``REDIS_PORT``, ``REDIS_PASSWORD`` from the environment. +""" + +from __future__ import annotations + +import os +import sys + +import redis + +PREFIX = "litellm:vcr:cassette:" +SCAN_BATCH = 500 + + +def _client() -> redis.Redis: + host = os.environ.get("REDIS_HOST") + if not host: + sys.exit("REDIS_HOST is not set; cannot flush VCR cache") + return redis.Redis( + host=host, + port=int(os.environ.get("REDIS_PORT", 6379)), + password=os.environ.get("REDIS_PASSWORD") or None, + socket_timeout=5, + socket_connect_timeout=5, + decode_responses=False, + ) + + +def main() -> None: + client = _client() + deleted = 0 + pipeline = client.pipeline(transaction=False) + pending = 0 + for key in client.scan_iter(match=f"{PREFIX}*", count=SCAN_BATCH): + pipeline.delete(key) + pending += 1 + if pending >= SCAN_BATCH: + deleted += sum(pipeline.execute()) + pipeline = client.pipeline(transaction=False) + pending = 0 + if pending: + deleted += sum(pipeline.execute()) + print(f"Deleted {deleted} VCR cassette key(s) under {PREFIX!r}") + + +if __name__ == "__main__": + main() diff --git a/tests/llm_translation/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py similarity index 100% rename from tests/llm_translation/_vcr_redis_persister.py rename to tests/_vcr_redis_persister.py diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index 0b03348190..adda3f5188 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -1,5 +1,12 @@ # conftest.py +# +# Auto-applies ``@pytest.mark.vcr`` to every collected test (see +# ``pytest_collection_modifyitems``) so live provider calls land in the +# Redis-backed VCR cache. The persister, header scrubbing and 2xx-only +# filtering live in ``tests/_vcr_redis_persister.py``; the cache key and +# 24h TTL match the llm_translation conftest. +import asyncio import importlib import os import sys @@ -9,9 +16,93 @@ import pytest sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path -import litellm -import asyncio +import litellm # noqa: E402 + +from tests._vcr_redis_persister import ( # noqa: E402 + filter_non_2xx_response, + make_redis_persister, +) + + +# Headers that must never be persisted to a cassette. +_FILTERED_REQUEST_HEADERS = ( + "authorization", + "x-api-key", + "anthropic-api-key", + "anthropic-version", + "openai-api-key", + "azure-api-key", + "api-key", + "cookie", + "x-amz-security-token", + "x-amz-date", + "x-amz-content-sha256", + "amz-sdk-invocation-id", + "amz-sdk-request", + "x-goog-api-key", + "x-goog-user-project", +) + +# Per-request response headers we strip so cassettes diff cleanly. +_FILTERED_RESPONSE_HEADERS = ( + "set-cookie", + "x-request-id", + "request-id", + "cf-ray", + "anthropic-organization-id", + "openai-organization", + "x-amzn-requestid", + "x-amzn-trace-id", + "date", +) + + +def _scrub_response(response): + if not isinstance(response, dict): + return response + headers = response.get("headers") or {} + if isinstance(headers, dict): + for header in list(headers): + if header.lower() in _FILTERED_RESPONSE_HEADERS: + headers.pop(header, None) + return response + + +def _before_record_response(response): + response = _scrub_response(response) + return filter_non_2xx_response(response) + + +@pytest.fixture(scope="module") +def vcr_config(): + return { + "filter_headers": list(_FILTERED_REQUEST_HEADERS), + "decode_compressed_response": True, + "record_mode": "once", + "match_on": ( + "method", + "scheme", + "host", + "port", + "path", + "query", + "body", + ), + "before_record_response": _before_record_response, + } + + +def _vcr_disabled() -> bool: + if os.environ.get("LITELLM_VCR_DISABLE") == "1": + return True + return not os.environ.get("REDIS_HOST") + + +def pytest_recording_configure(config, vcr): + if _vcr_disabled(): + return + vcr.register_persister(make_redis_persister()) @pytest.fixture(scope="session") @@ -61,15 +152,23 @@ def setup_and_teardown(): def pytest_collection_modifyitems(config, items): - # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests + # Auto-apply ``@pytest.mark.vcr`` so any provider call lands in the + # Redis cache. No respx files exist in this directory today; if any are + # added later, exclude them by filename here. Skip entirely when VCR + # is disabled (no REDIS_HOST or LITELLM_VCR_DISABLE=1). + if not _vcr_disabled(): + for item in items: + if item.get_closest_marker("vcr") is not None: + continue + item.add_marker(pytest.mark.vcr) + + # Preserve historical custom_logger ordering. custom_logger_tests = [ item for item in items if "custom_logger" in item.parent.name ] other_tests = [item for item in items if "custom_logger" not in item.parent.name] - # Sort tests based on their names custom_logger_tests.sort(key=lambda x: x.name) other_tests.sort(key=lambda x: x.name) - # Reorder the items list items[:] = custom_logger_tests + other_tests diff --git a/tests/llm_translation/Readme.md b/tests/llm_translation/Readme.md index a7fcfe072a..e684c73202 100644 --- a/tests/llm_translation/Readme.md +++ b/tests/llm_translation/Readme.md @@ -2,28 +2,40 @@ Unit tests for individual LLM providers. Name of the test file is the name of the LLM provider - e.g. `test_openai.py` is for OpenAI. -## VCR-backed tests +## Redis-backed VCR cache -Tests decorated with `@pytest.mark.vcr` (typically in `*_vcr.py` files, -e.g. `test_anthropic_completion_vcr.py`) replay recorded HTTP traffic from -`cassettes/` via [`pytest-recording`](https://github.com/kiwicom/pytest-recording) -instead of calling the real provider. They run offline by default — no API -keys required, no per-PR cost. +Every test in this directory is auto-decorated with `@pytest.mark.vcr` (via +`conftest.py`). The first time a test runs we hit the live provider and +record the HTTP exchange into Redis under +`litellm:vcr:cassette:`. Every subsequent run within 24h replays +from Redis without touching the network. The 24h TTL means each new day's +first run records again, so upstream API drift surfaces within a day. -To re-record every marked test in one sweep: +The persister, header scrubbing, and 2xx-only filtering are defined in +`tests/_vcr_redis_persister.py`. Files that already use `respx` (which +patches the same httpx transport vcrpy does) are excluded from the +auto-marker — see `_RESPX_CONFLICTING_FILES` in `conftest.py`. + +### Required environment + +`REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD` — same vars CircleCI uses for +its other Redis-backed jobs. Provider credentials +(`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `AWS_*`, etc.) are needed only on +cache-miss (the daily re-record), not on replay. + +### Flushing the cache + +When you want the next run to re-record immediately instead of waiting +for the 24h TTL: ```bash -ANTHROPIC_API_KEY=sk-ant-... OPENAI_API_KEY=sk-... \ - make test-llm-translation-record +make test-llm-translation-flush-vcr-cache ``` -To scope to a single file: +### Disabling VCR + +Skip the cache entirely (every call goes live, no recording): ```bash -ANTHROPIC_API_KEY=sk-ant-... \ - make test-llm-translation-record TARGET=test_anthropic_completion_vcr.py +LITELLM_VCR_DISABLE=1 uv run pytest tests/llm_translation/test_.py ``` - -See [`cassettes/README.md`](./cassettes/README.md) for the full workflow, -including how to add a new cassette-backed test and what to scrub from -recordings before committing. diff --git a/tests/llm_translation/cassettes/README.md b/tests/llm_translation/cassettes/README.md deleted file mode 100644 index eff373a768..0000000000 --- a/tests/llm_translation/cassettes/README.md +++ /dev/null @@ -1,115 +0,0 @@ -# VCR cassettes for LLM translation tests - -This directory holds [vcrpy](https://vcrpy.readthedocs.io/) cassettes used by -`tests/llm_translation/` to replay real provider HTTP traffic without hitting -the live API. - -Why this exists is tracked in -[LIT-2683](https://linear.app/litellm-ai/issue/LIT-2683) and discussed in -`#sdlc` on Slack: e2e tests were repeatedly draining provider billing accounts -and producing flaky CI on outages. Recording the HTTP exchange once and -replaying it on subsequent runs gives us realistic provider responses -(streaming, headers, edge-case payloads) at zero per-PR cost. - -## Layout - -We use [`pytest-recording`](https://github.com/kiwicom/pytest-recording), -which auto-resolves the cassette path from the test location: - -``` -tests/llm_translation/ - cassettes/ - / - .yaml - test__completion_vcr.py - conftest.py # provides the shared vcr_config fixture -``` - -For example, a test -`tests/llm_translation/test_anthropic_completion_vcr.py::test_basic` is backed by -`tests/llm_translation/cassettes/test_anthropic_completion_vcr/test_basic.yaml`. - -## Adding a new cassette-backed test - -1. Pick a small, deterministic call. Avoid prompts whose output depends on - wall-clock time, randomness, or live web data. -2. Write the test as you normally would and decorate it with - `@pytest.mark.vcr`. No imports beyond `pytest` are needed — the - `vcr_config` fixture in `conftest.py` is applied automatically. -3. Run the sweep recorder once with the credentials you need. Recording is - strictly opt-in via `--record-mode=once`; the default replay mode never - touches the network. - -## Bulk re-record (the common path) - -A single sweep replays every `@pytest.mark.vcr` test under -`tests/llm_translation`, hitting the live provider only for tests that don't -yet have a cassette: - -```bash -ANTHROPIC_API_KEY=sk-ant-... \ -OPENAI_API_KEY=sk-... \ -AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... \ - make test-llm-translation-record -``` - -Or scope it to a single file: - -```bash -ANTHROPIC_API_KEY=sk-ant-... \ - make test-llm-translation-record TARGET=test_anthropic_completion_vcr.py -``` - -vcrpy's `once` record mode does **not** overwrite an existing cassette — -delete the file first if you're intentionally refreshing it: - -```bash -rm tests/llm_translation/cassettes/test_anthropic_completion_vcr/test_basic.yaml -ANTHROPIC_API_KEY=sk-ant-... \ - make test-llm-translation-record TARGET=test_anthropic_completion_vcr.py -``` - -To force a full refresh of every cassette in one shot: - -```bash -rm -rf tests/llm_translation/cassettes/test_* -ANTHROPIC_API_KEY=... OPENAI_API_KEY=... AWS_* \ - uv run pytest tests/llm_translation -m vcr --record-mode=all -v -``` - -## Refreshing the canned Anthropic fixtures (no API key) - -The two Anthropic cassettes shipped with this directory are recorded against -an in-process mock so contributors can regenerate them without an -`ANTHROPIC_API_KEY`: - -```bash -uv run python tests/llm_translation/cassettes/_record_anthropic_fixtures.py -``` - -For a full refresh against the real API, delete the cassettes first and use -the bulk-record sweep above. - -## Cassette hygiene - -After recording, **always inspect the YAML before committing**: - -- The `vcr_config` fixture in `conftest.py` already filters the common - request headers (`Authorization`, `x-api-key`, `anthropic-api-key`, AWS - sigv4 headers, cookies, GCP keys, …) and per-request response headers - (`set-cookie`, `cf-ray`, request IDs, org IDs, dates). -- A request *body* might still contain a token if your test passed one - inline — scrub it manually. -- Quick sanity check: `grep -i 'sk-\|bearer\|api-key' cassettes//*.yaml` - should be clean. -- Trim unhelpful response bodies if they're megabytes large but the - assertion only needs a few fields. - -## Don't - -- Don't commit cassettes with real API keys, OAuth tokens, or PII. -- Don't rely on cassettes for tests of *non-deterministic* behavior - (rate-limit retries, timeouts, model creativity). Mock those at the - LiteLLM layer instead. -- Don't manually edit cassette YAML beyond scrubbing — the format is - byte-sensitive (e.g. content-length headers must match the body). diff --git a/tests/llm_translation/cassettes/_record_anthropic_fixtures.py b/tests/llm_translation/cassettes/_record_anthropic_fixtures.py deleted file mode 100644 index 81321b992e..0000000000 --- a/tests/llm_translation/cassettes/_record_anthropic_fixtures.py +++ /dev/null @@ -1,268 +0,0 @@ -"""Helper script that records Anthropic-shaped cassettes against a local mock. - -This is a *one-shot* utility, not a test. It exists so contributors can -regenerate the canned Anthropic cassettes shipped with this PR without -spending real provider credits and without needing an ``ANTHROPIC_API_KEY``. - -Run it with:: - - uv run python tests/llm_translation/cassettes/_record_anthropic_fixtures.py - -The script: - -1. Spins up a tiny in-process HTTP server that returns canned Anthropic - ``/v1/messages`` payloads (one non-streaming, one SSE streaming). -2. Records LiteLLM's real outbound HTTP through vcrpy. -3. Rewrites the cassette URL/Host so replay matches genuine - ``https://api.anthropic.com/v1/messages`` traffic. - -The cassettes are written to the per-test paths that ``pytest-recording`` -expects (``cassettes//.yaml``) so the existing tests -in ``test_anthropic_completion_vcr.py`` pick them up unchanged. - -For a refresh against the *real* Anthropic API, use the -``--record-mode=once`` sweep described in -``tests/llm_translation/cassettes/README.md`` — that path needs a real -``ANTHROPIC_API_KEY``. -""" - -from __future__ import annotations - -import json -import os -import sys -import threading -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from typing import Any, Iterable - -import vcr # type: ignore[import-not-found] -import yaml # type: ignore[import-not-found] - -REPO_ROOT = Path(__file__).resolve().parents[3] -sys.path.insert(0, str(REPO_ROOT)) - -import litellm # noqa: E402 - -CASSETTE_DIR = Path(__file__).parent / "test_anthropic_completion_vcr" -MOCK_HOST = "127.0.0.1" -NON_STREAM_PORT = 18765 -STREAM_PORT = 18766 -REAL_ANTHROPIC_HOST = "api.anthropic.com" - -NON_STREAM_RESPONSE: dict[str, Any] = { - "id": "msg_01ABCDEFGHIJKLMNOPQRSTUV", - "type": "message", - "role": "assistant", - "model": "claude-sonnet-4-5-20250929", - "content": [{"type": "text", "text": "Hello! How can I help you today?"}], - "stop_reason": "end_turn", - "stop_sequence": None, - "usage": { - "input_tokens": 12, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "output_tokens": 11, - }, -} - -STREAM_EVENTS: list[tuple[str, dict[str, Any]]] = [ - ( - "message_start", - { - "type": "message_start", - "message": { - "id": "msg_01STREAMABCDEFGH", - "type": "message", - "role": "assistant", - "model": "claude-sonnet-4-5-20250929", - "content": [], - "stop_reason": None, - "stop_sequence": None, - "usage": {"input_tokens": 14, "output_tokens": 1}, - }, - }, - ), - ( - "content_block_start", - { - "type": "content_block_start", - "index": 0, - "content_block": {"type": "text", "text": ""}, - }, - ), - ( - "content_block_delta", - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "text_delta", "text": "Hello"}, - }, - ), - ( - "content_block_delta", - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "text_delta", "text": " from"}, - }, - ), - ( - "content_block_delta", - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "text_delta", "text": " LiteLLM!"}, - }, - ), - ("content_block_stop", {"type": "content_block_stop", "index": 0}), - ( - "message_delta", - { - "type": "message_delta", - "delta": {"stop_reason": "end_turn", "stop_sequence": None}, - "usage": {"output_tokens": 5}, - }, - ), - ("message_stop", {"type": "message_stop"}), -] - - -def _make_handler(mode: str) -> type[BaseHTTPRequestHandler]: - class Handler(BaseHTTPRequestHandler): - def log_message(self, *args: Any, **kwargs: Any) -> None: # silence - return - - def do_POST(self) -> None: # noqa: N802 - length = int(self.headers.get("Content-Length", "0")) - self.rfile.read(length) - if mode == "json": - body = json.dumps(NON_STREAM_RESPONSE).encode("utf-8") - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.send_header("anthropic-ratelimit-requests-limit", "4000") - self.send_header("anthropic-ratelimit-requests-remaining", "3999") - self.end_headers() - self.wfile.write(body) - else: - self.send_response(200) - self.send_header("Content-Type", "text/event-stream") - self.send_header("Cache-Control", "no-cache") - self.end_headers() - for event_name, data in STREAM_EVENTS: - chunk = ( - f"event: {event_name}\n" f"data: {json.dumps(data)}\n\n" - ).encode("utf-8") - self.wfile.write(chunk) - self.wfile.flush() - - return Handler - - -def _serve(port: int, mode: str) -> ThreadingHTTPServer: - srv = ThreadingHTTPServer((MOCK_HOST, port), _make_handler(mode)) - threading.Thread(target=srv.serve_forever, daemon=True).start() - return srv - - -# Headers that vary every run (timestamps, server build) and must be stripped -# so the cassette is byte-stable across regenerations. Replay does not depend -# on them. -_NON_DETERMINISTIC_HEADERS = ("Date", "Server") - - -def _strip_nondeterministic_headers(path: Path) -> None: - """Remove headers whose values change every run from the cassette. - - Parses + rewrites the YAML rather than regex-substituting against a - fixed indentation, so this stays correct if vcrpy ever changes its - serialization style. - """ - cassette = yaml.safe_load(path.read_text()) - for interaction in cassette.get("interactions") or []: - headers = (interaction.get("response") or {}).get("headers") or {} - for header in _NON_DETERMINISTIC_HEADERS: - headers.pop(header, None) - path.write_text(yaml.safe_dump(cassette, default_flow_style=False, sort_keys=False)) - - -def _rewrite_cassette_to_real_host(path: Path, mock_host_port: str) -> None: - """Replace mock host/port in the cassette with the real Anthropic host.""" - text = path.read_text() - text = text.replace(f"http://{mock_host_port}", f"https://{REAL_ANTHROPIC_HOST}") - text = text.replace(mock_host_port, REAL_ANTHROPIC_HOST) - path.write_text(text) - _strip_nondeterministic_headers(path) - - -def _consume(iterable: Iterable[Any]) -> None: - for _ in iterable: - pass - - -def record_non_streaming() -> None: - cassette = CASSETTE_DIR / "test_anthropic_basic_completion_replay.yaml" - if cassette.exists(): - cassette.unlink() - server = _serve(NON_STREAM_PORT, "json") - try: - my_vcr = vcr.VCR( - record_mode="all", - filter_headers=[ - "authorization", - "x-api-key", - "anthropic-version", - ], - ) - with my_vcr.use_cassette(str(cassette)): - response = litellm.completion( - model="anthropic/claude-sonnet-4-5-20250929", - messages=[{"role": "user", "content": "Hello!"}], - api_base=f"http://{MOCK_HOST}:{NON_STREAM_PORT}", - api_key="sk-ant-recording", - ) - assert response.choices[0].message.content - finally: - server.shutdown() - _rewrite_cassette_to_real_host(cassette, f"{MOCK_HOST}:{NON_STREAM_PORT}") - - -def record_streaming() -> None: - cassette = CASSETTE_DIR / "test_anthropic_streaming_completion_replay.yaml" - if cassette.exists(): - cassette.unlink() - server = _serve(STREAM_PORT, "stream") - try: - my_vcr = vcr.VCR( - record_mode="all", - filter_headers=[ - "authorization", - "x-api-key", - "anthropic-version", - ], - ) - with my_vcr.use_cassette(str(cassette)): - stream = litellm.completion( - model="anthropic/claude-sonnet-4-5-20250929", - messages=[{"role": "user", "content": "Hello!"}], - api_base=f"http://{MOCK_HOST}:{STREAM_PORT}", - api_key="sk-ant-recording", - stream=True, - ) - _consume(stream) - finally: - server.shutdown() - _rewrite_cassette_to_real_host(cassette, f"{MOCK_HOST}:{STREAM_PORT}") - - -def main() -> None: - os.environ.setdefault("LITELLM_LOG", "WARNING") - CASSETTE_DIR.mkdir(parents=True, exist_ok=True) - record_non_streaming() - record_streaming() - print(f"Wrote cassettes to {CASSETTE_DIR}") - - -if __name__ == "__main__": - main() diff --git a/tests/llm_translation/cassettes/test_anthropic_completion_vcr/test_anthropic_basic_completion_replay.yaml b/tests/llm_translation/cassettes/test_anthropic_completion_vcr/test_anthropic_basic_completion_replay.yaml deleted file mode 100644 index f593c5ab96..0000000000 --- a/tests/llm_translation/cassettes/test_anthropic_completion_vcr/test_anthropic_basic_completion_replay.yaml +++ /dev/null @@ -1,41 +0,0 @@ -interactions: -- request: - body: '{"model": "claude-sonnet-4-5-20250929", "messages": [{"role": "user", "content": - [{"type": "text", "text": "Hello!"}]}], "max_tokens": 64000}' - headers: - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '141' - Host: - - api.anthropic.com - User-Agent: - - litellm/1.84.0 - accept: - - application/json - content-type: - - application/json - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"id": "msg_01ABCDEFGHIJKLMNOPQRSTUV", "type": "message", "role": "assistant", - "model": "claude-sonnet-4-5-20250929", "content": [{"type": "text", "text": - "Hello! How can I help you today?"}], "stop_reason": "end_turn", "stop_sequence": - null, "usage": {"input_tokens": 12, "cache_creation_input_tokens": 0, "cache_read_input_tokens": - 0, "output_tokens": 11}}' - headers: - Content-Length: - - '358' - Content-Type: - - application/json - anthropic-ratelimit-requests-limit: - - '4000' - anthropic-ratelimit-requests-remaining: - - '3999' - status: - code: 200 - message: OK -version: 1 diff --git a/tests/llm_translation/cassettes/test_anthropic_completion_vcr/test_anthropic_streaming_completion_replay.yaml b/tests/llm_translation/cassettes/test_anthropic_completion_vcr/test_anthropic_streaming_completion_replay.yaml deleted file mode 100644 index 25c0c6c3d2..0000000000 --- a/tests/llm_translation/cassettes/test_anthropic_completion_vcr/test_anthropic_streaming_completion_replay.yaml +++ /dev/null @@ -1,81 +0,0 @@ -interactions: -- request: - body: '{"model": "claude-sonnet-4-5-20250929", "messages": [{"role": "user", "content": - [{"type": "text", "text": "Hello!"}]}], "max_tokens": 64000, "stream": true}' - headers: - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '157' - Host: - - api.anthropic.com - User-Agent: - - litellm/1.84.0 - accept: - - application/json - content-type: - - application/json - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: 'event: message_start - - data: {"type": "message_start", "message": {"id": "msg_01STREAMABCDEFGH", - "type": "message", "role": "assistant", "model": "claude-sonnet-4-5-20250929", - "content": [], "stop_reason": null, "stop_sequence": null, "usage": {"input_tokens": - 14, "output_tokens": 1}}} - - - event: content_block_start - - data: {"type": "content_block_start", "index": 0, "content_block": {"type": - "text", "text": ""}} - - - event: content_block_delta - - data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", - "text": "Hello"}} - - - event: content_block_delta - - data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", - "text": " from"}} - - - event: content_block_delta - - data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", - "text": " LiteLLM!"}} - - - event: content_block_stop - - data: {"type": "content_block_stop", "index": 0} - - - event: message_delta - - data: {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": - null}, "usage": {"output_tokens": 5}} - - - event: message_stop - - data: {"type": "message_stop"} - - - ' - headers: - Cache-Control: - - no-cache - Content-Type: - - text/event-stream - status: - code: 200 - message: OK -version: 1 diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 0d9148a61a..ac87d00980 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -4,7 +4,14 @@ # Mirrors the pattern in tests/local_testing/conftest.py: # - Function-scoped fixture resets litellm globals to true defaults # - Module-scoped reload only in single-process mode +# +# Also wires up the Redis-backed VCR cache. Every test in this directory is +# auto-marked with ``@pytest.mark.vcr`` (see ``pytest_collection_modifyitems``) +# unless its file appears in ``_RESPX_CONFLICTING_FILES`` — those use respx, +# which patches the same httpx transport vcrpy does. Cache key naming, TTL, +# and 2xx-only filtering live in ``tests/_vcr_redis_persister.py``. +import asyncio import importlib import os import sys @@ -14,32 +21,48 @@ import pytest sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path -# Make sibling helper modules under tests/llm_translation/ importable regardless -# of the directory pytest is invoked from. -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import litellm +import litellm # noqa: E402 -import asyncio - -from _vcr_redis_persister import ( # noqa: E402 (sibling module under conftest dir) +from tests._vcr_redis_persister import ( # noqa: E402 filter_non_2xx_response, make_redis_persister, ) # --------------------------------------------------------------------------- -# VCR cassette infrastructure (pytest-recording) +# VCR cassette infrastructure (pytest-recording + Redis) # --------------------------------------------------------------------------- -# Tests marked with ``@pytest.mark.vcr`` replay HTTP traffic from a cassette -# under ``cassettes//.yaml`` instead of hitting the -# live provider. Default record mode is ``none`` (replay only) so CI never -# accidentally calls a real LLM. To re-record every marked test in one sweep:: -# -# ANTHROPIC_API_KEY=sk-ant-... \ -# uv run pytest tests/llm_translation -m vcr --record-mode=once -# -# See ``tests/llm_translation/cassettes/README.md`` for the full workflow. +# All tests in tests/llm_translation/ are auto-marked with ``@pytest.mark.vcr`` +# (excluding the respx-using files listed below). On cache miss vcrpy records +# the live response into Redis under ``litellm:vcr:cassette:`` with +# a 24h TTL; subsequent runs within that window replay without touching the +# network. Set ``LITELLM_VCR_DISABLE=1`` to skip VCR entirely (e.g. when +# debugging an upstream API change locally). + +# Test files that use ``respx`` to patch httpx. vcrpy patches the same +# transport, so applying both to the same test will make one of them silently +# win and the other look like a no-op. Skip auto-marking these. +_RESPX_CONFLICTING_FILES = frozenset( + { + "test_azure_o_series.py", + "test_gpt4o_audio.py", + "test_nvidia_nim.py", + "test_openai.py", + "test_openai_o1.py", + "test_prompt_caching.py", + "test_text_completion_unit_tests.py", + "test_xai.py", + } +) + +# The persister's own unit tests must not run inside a VCR cassette context — +# they call ``save_cassette`` / ``load_cassette`` directly against fakeredis +# and don't make HTTP calls, but auto-marking them would still wrap each +# test in a Redis lookup we don't want. +_VCR_AUTO_MARKER_SKIP_FILES = _RESPX_CONFLICTING_FILES | frozenset( + {"test_vcr_redis_persister.py"} +) # Headers that must never be persisted to a cassette. Matched # case-insensitively by vcrpy. @@ -47,8 +70,8 @@ _FILTERED_REQUEST_HEADERS = ( "authorization", "x-api-key", "anthropic-api-key", - # Strip ``anthropic-version`` so live-recorded and mock-recorded cassettes - # have the same shape (the local mock helper drops it too). + # Strip ``anthropic-version`` so cassettes have a stable shape across + # SDK versions that bump the header. "anthropic-version", "openai-api-key", "azure-api-key", @@ -104,11 +127,17 @@ def _before_record_response(response): def vcr_config(): """Shared VCR config consumed by ``pytest-recording``. - Applied to every ``@pytest.mark.vcr`` test in this directory. + ``record_mode="once"`` is what makes this a useful daily cache: + - cassette absent (cache miss) → record the live call into Redis, + - cassette present (cache hit) → replay only. + 24h TTL on the Redis key means each new day's first run records against + live providers, surfacing API drift within a day instead of silently + serving stale responses forever. """ return { "filter_headers": list(_FILTERED_REQUEST_HEADERS), "decode_compressed_response": True, + "record_mode": "once", # Match on full request shape so streaming vs non-streaming and # different prompts produce distinct cassettes. "match_on": ( @@ -124,22 +153,24 @@ def vcr_config(): } -def pytest_recording_configure(config, vcr): - """Swap vcrpy's default filesystem persister for a Redis-backed one. +def _vcr_disabled() -> bool: + """VCR is disabled when explicitly opted out or when Redis isn't wired. - Opt-in via ``LITELLM_VCR_REDIS=1`` so local dev keeps the YAML-on-disk - behaviour and CI (which sets the flag) gets a 24h-TTL cache that - auto-refreshes against live providers without manual ``make`` runs. + No Redis means no cache to read from or write to — fall back to live + calls instead of silently writing YAML to disk (which we don't ship). """ - if os.environ.get("LITELLM_VCR_REDIS") != "1": + if os.environ.get("LITELLM_VCR_DISABLE") == "1": + return True + return not os.environ.get("REDIS_HOST") + + +def pytest_recording_configure(config, vcr): + """Register the Redis-backed cassette persister.""" + if _vcr_disabled(): return vcr.register_persister(make_redis_persister()) -# pytest-recording's default cassette dir is -# ``/cassettes/``. Keep that — it gives every test its -# own file and avoids name collisions across modules. - # --------------------------------------------------------------------------- # Capture TRUE defaults at conftest import time (before test modules pollute). # --------------------------------------------------------------------------- @@ -170,7 +201,6 @@ def event_loop(): @pytest.fixture(scope="function", autouse=True) def setup_and_teardown(event_loop): # Add event_loop as a dependency - curr_dir = os.getcwd() sys.path.insert(0, os.path.abspath("../..")) import litellm @@ -239,15 +269,28 @@ def _vcr_record_retries(setup_and_teardown, request): def pytest_collection_modifyitems(config, items): - # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests + # 1. Auto-apply ``@pytest.mark.vcr`` to every collected test in this + # directory so any provider call lands in the Redis cache. Skip files + # that use respx (it patches the same transport vcrpy does) and the + # persister's own unit tests. Skip entirely if VCR is disabled (no + # REDIS_HOST or LITELLM_VCR_DISABLE=1) so dev runs without Redis + # don't go through cassette logic at all. + if not _vcr_disabled(): + for item in items: + filename = os.path.basename(str(item.fspath)) + if filename in _VCR_AUTO_MARKER_SKIP_FILES: + continue + if item.get_closest_marker("vcr") is not None: + continue + item.add_marker(pytest.mark.vcr) + + # 2. Preserve the historical ordering of custom_logger tests vs the rest. custom_logger_tests = [ item for item in items if "custom_logger" in item.parent.name ] other_tests = [item for item in items if "custom_logger" not in item.parent.name] - # Sort tests based on their names custom_logger_tests.sort(key=lambda x: x.name) other_tests.sort(key=lambda x: x.name) - # Reorder the items list items[:] = custom_logger_tests + other_tests diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index fdf8c24ac9..ae42155642 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -1885,3 +1885,51 @@ def test_metadata_filter_applies_to_azure_anthropic(): headers={}, ) assert data.get("metadata") == {"user_id": "u2"} + + +def test_anthropic_basic_completion_replay(): + """Smoke-test that a vanilla Anthropic completion replays from a cassette. + + Exercises the full LiteLLM transformation pipeline (request shaping + + response parsing) against a real-shape Anthropic payload. The cassette + is loaded from the Redis-backed VCR cache configured in conftest.py. + """ + response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Hello!"}], + ) + + assert response is not None + assert response.choices[0].message.content == ("Hello! How can I help you today?") + assert response.usage.prompt_tokens == 12 + assert response.usage.completion_tokens == 11 + # Anthropic sets stop_reason="end_turn" → litellm normalises to "stop" + assert response.choices[0].finish_reason == "stop" + + +def test_anthropic_streaming_completion_replay(): + """Replay a streaming Anthropic completion from the VCR cache. + + Exercises the SSE chunk parser and the public streaming surface — any + regression in the streaming transformation surfaces here because the + cassette captures every ``content_block_delta`` event Anthropic emits. + """ + stream = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Hello!"}], + stream=True, + ) + + collected_text = "" + finish_reason = None + for chunk in stream: + if not chunk.choices: + continue + delta = chunk.choices[0].delta + if delta and delta.content: + collected_text += delta.content + if chunk.choices[0].finish_reason: + finish_reason = chunk.choices[0].finish_reason + + assert collected_text == "Hello from LiteLLM!" + assert finish_reason == "stop" diff --git a/tests/llm_translation/test_anthropic_completion_vcr.py b/tests/llm_translation/test_anthropic_completion_vcr.py deleted file mode 100644 index 04f2914deb..0000000000 --- a/tests/llm_translation/test_anthropic_completion_vcr.py +++ /dev/null @@ -1,97 +0,0 @@ -""" -Cassette-replayed Anthropic completion tests. - -These tests exercise the same end-to-end ``litellm.completion`` code paths as -``test_anthropic_completion.py`` but replay HTTP traffic from cassettes under -``cassettes/test_anthropic_completion_vcr/`` instead of calling -``api.anthropic.com``. CI runs them with no API key and zero cost. - -Add a new test by writing it normally and decorating with ``@pytest.mark.vcr``. -The cassette path is resolved automatically from the test module + test name -by ``pytest-recording`` (see ``conftest.py``). - -To re-record every marked test in one sweep:: - - ANTHROPIC_API_KEY=sk-ant-... \\ - uv run pytest tests/llm_translation -m vcr --record-mode=once - -See ``tests/llm_translation/cassettes/README.md`` for the full workflow. -""" - -import os -import sys - -import pytest - -sys.path.insert(0, os.path.abspath("../..")) - -import litellm # noqa: E402 - -# A non-secret placeholder API key. The vcr_config fixture in conftest.py -# filters Authorization / x-api-key headers from cassettes, so this value -# never lands on disk; it only stops the SDK from raising when -# ``ANTHROPIC_API_KEY`` is unset (the common CI case). -PLACEHOLDER_ANTHROPIC_API_KEY = "sk-ant-vcr-placeholder" - - -@pytest.fixture(autouse=True) -def _placeholder_anthropic_key(monkeypatch): - """Ensure an API key is set so replay works offline. - - If a real key is present (e.g. when re-recording with - ``--record-mode=once``), we leave it untouched. - """ - if not os.environ.get("ANTHROPIC_API_KEY"): - monkeypatch.setenv("ANTHROPIC_API_KEY", PLACEHOLDER_ANTHROPIC_API_KEY) - - -@pytest.mark.vcr -def test_anthropic_basic_completion_replay(): - """Smoke-test that a vanilla Anthropic completion replays from a cassette. - - This is the canonical example for the cassette-based testing pattern: no - API key required at runtime, deterministic output, and the full LiteLLM - transformation pipeline (request shaping + response parsing) runs against - a real-shape Anthropic payload. - """ - response = litellm.completion( - model="anthropic/claude-sonnet-4-5-20250929", - messages=[{"role": "user", "content": "Hello!"}], - ) - - assert response is not None - assert response.choices[0].message.content == ("Hello! How can I help you today?") - assert response.usage.prompt_tokens == 12 - assert response.usage.completion_tokens == 11 - # Anthropic sets stop_reason="end_turn" → litellm normalises to "stop" - assert response.choices[0].finish_reason == "stop" - - -@pytest.mark.vcr -def test_anthropic_streaming_completion_replay(): - """Replay a streaming Anthropic completion from a cassette. - - Exercises the SSE chunk parser and the public streaming surface. The - underlying cassette captures every ``content_block_delta`` event Anthropic - emits, so any regression in the streaming transformation will surface - here. - """ - stream = litellm.completion( - model="anthropic/claude-sonnet-4-5-20250929", - messages=[{"role": "user", "content": "Hello!"}], - stream=True, - ) - - collected_text = "" - finish_reason = None - for chunk in stream: - if not chunk.choices: - continue - delta = chunk.choices[0].delta - if delta and delta.content: - collected_text += delta.content - if chunk.choices[0].finish_reason: - finish_reason = chunk.choices[0].finish_reason - - assert collected_text == "Hello from LiteLLM!" - assert finish_reason == "stop" diff --git a/tests/llm_translation/test_vcr_redis_persister.py b/tests/llm_translation/test_vcr_redis_persister.py index e9ba57ac73..9ca23410bd 100644 --- a/tests/llm_translation/test_vcr_redis_persister.py +++ b/tests/llm_translation/test_vcr_redis_persister.py @@ -25,9 +25,11 @@ from vcr.persisters.filesystem import CassetteNotFoundError from vcr.request import Request from vcr.serializers import yamlserializer -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +# Make tests/ importable as a package so we can pull the shared persister +# without depending on pytest's CWD. +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) -from _vcr_redis_persister import ( # noqa: E402 +from tests._vcr_redis_persister import ( # noqa: E402 CASSETTE_TTL_SECONDS, filter_non_2xx_response, make_redis_persister, From e1f2b4b818fed51edbea2d85be4e80c3dd796173 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Apr 2026 21:48:48 +0000 Subject: [PATCH 07/30] tests(vcr): trim non-load-bearing comments and docstrings Removes commentary that restated the code, including: - module-level banners explaining what the conftest does (covered by Readme.md and the function bodies) - docstrings on _scrub_response, _before_record_response, vcr_config, _vcr_disabled, pytest_recording_configure (function names + bodies are self-evident) - inline notes about header filtering, match_on, etc. - per-test docstrings restating the test name Keeps the two non-obvious notes that aren't recoverable from the code: the vcrpy/respx httpx-transport collision rationale on _RESPX_CONFLICTING_FILES, the vcrpy "return None to skip persisting" contract on filter_non_2xx_response, and the fixture-ordering dependency on _vcr_record_retries. --- Makefile | 6 -- tests/_flush_vcr_cache.py | 9 --- tests/_vcr_redis_persister.py | 59 ++------------ tests/llm_responses_api_testing/conftest.py | 16 +--- tests/llm_translation/conftest.py | 77 ++----------------- .../test_anthropic_completion.py | 13 ---- .../test_vcr_redis_persister.py | 52 ++----------- 7 files changed, 19 insertions(+), 213 deletions(-) diff --git a/Makefile b/Makefile index 125b0e749d..5dbd308a3e 100644 --- a/Makefile +++ b/Makefile @@ -186,11 +186,5 @@ test-llm-translation-single: install-test-deps --junitxml=test-results/junit.xml \ -v --tb=short --maxfail=100 --timeout=300 -# VCR cache helpers ----------------------------------------------------------- -# Drop every Redis key under the ``litellm:vcr:cassette:*`` prefix. Use this -# when you want the next CI run (or local run) to re-record against live -# providers immediately instead of waiting for the 24h TTL to roll over. -# Reads REDIS_HOST / REDIS_PORT / REDIS_PASSWORD from the environment, the -# same vars CircleCI uses for its other Redis-backed jobs. test-llm-translation-flush-vcr-cache: $(UV_RUN) python tests/_flush_vcr_cache.py diff --git a/tests/_flush_vcr_cache.py b/tests/_flush_vcr_cache.py index 4bb3fd7ba8..dfaba2367c 100644 --- a/tests/_flush_vcr_cache.py +++ b/tests/_flush_vcr_cache.py @@ -1,12 +1,3 @@ -"""Flush every VCR cassette stored in Redis. - -Run via ``make test-llm-translation-flush-vcr-cache``. Use when you want the -next test run to re-record against live providers right now instead of -waiting for the 24h TTL to expire. - -Reads ``REDIS_HOST``, ``REDIS_PORT``, ``REDIS_PASSWORD`` from the environment. -""" - from __future__ import annotations import os diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index 88423393c5..3b1e456c0a 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -1,18 +1,3 @@ -"""Redis-backed cassette persister for vcrpy. - -Stores the same serialized cassette payload that ``FilesystemPersister`` -would write to disk, but under a Redis key with a 24h TTL. Cassettes -auto-expire so the next CI run after the rollover re-records against the -live provider, surfacing API drift within a day instead of waiting for a -human to refresh ``cassettes/*.yaml`` by hand. - -On a cache miss we raise ``CassetteNotFoundError``; vcrpy's record-mode -machinery catches that and falls through to a live HTTP call, which then -gets persisted via ``save_cassette``. Non-2xx responses are filtered out -upstream by ``conftest.before_record_response`` so a transient provider -failure can't poison the cache for 24h. -""" - from __future__ import annotations import os @@ -26,12 +11,7 @@ REDIS_KEY_PREFIX = "litellm:vcr:cassette:" def redis_key_for(cassette_path: str) -> str: - """Map a cassette file path to a stable Redis key. - - Uses the path relative to CWD so keys are stable across machines. - """ - rel = os.path.relpath(str(cassette_path)) - return f"{REDIS_KEY_PREFIX}{rel}" + return f"{REDIS_KEY_PREFIX}{os.path.relpath(str(cassette_path))}" def _build_default_client(): @@ -39,9 +19,7 @@ def _build_default_client(): host = os.environ.get("REDIS_HOST") if not host: - raise RuntimeError( - "REDIS_HOST is not set; cannot build Redis cassette persister" - ) + raise RuntimeError("REDIS_HOST is not set") return redis.Redis( host=host, port=int(os.environ.get("REDIS_PORT", 6379)), @@ -56,14 +34,6 @@ def make_redis_persister( client: Optional[Any] = None, ttl_seconds: int = CASSETTE_TTL_SECONDS, ): - """Build a vcrpy-compatible persister bound to a Redis client. - - The returned object exposes ``load_cassette`` / ``save_cassette`` and is - a drop-in replacement for ``vcr.persisters.filesystem.FilesystemPersister``. - Pass an explicit ``client`` in tests; production callers omit it and let - the persister build a client from ``REDIS_HOST`` / ``REDIS_PORT`` / - ``REDIS_PASSWORD``. - """ redis_client = client if client is not None else _build_default_client() class _RedisPersister: @@ -80,32 +50,17 @@ def make_redis_persister( def save_cassette(cassette_path, cassette_dict, serializer): data = serialize(cassette_dict, serializer) payload = data.encode("utf-8") if isinstance(data, str) else data - redis_client.set( - redis_key_for(cassette_path), - payload, - ex=ttl_seconds, - ) + redis_client.set(redis_key_for(cassette_path), payload, ex=ttl_seconds) return _RedisPersister def filter_non_2xx_response(response): - """vcrpy ``before_record_response`` hook that drops non-2xx responses. - - Returning ``None`` tells vcrpy to skip persisting the response (see - ``vcr.cassette.Cassette.append``). This prevents transient 5xx/429 - failures from being baked into the cache for the rest of the TTL window. - """ + # Returning None tells vcrpy to skip persisting; see Cassette.append. if not isinstance(response, dict): return response status = response.get("status") - code = None - if isinstance(status, dict): - code = status.get("code") - elif isinstance(status, int): - code = status - if code is None: + code = status.get("code") if isinstance(status, dict) else status + if not isinstance(code, int): return response - if 200 <= int(code) < 300: - return response - return None + return response if 200 <= code < 300 else None diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index adda3f5188..7209d2f957 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -1,10 +1,4 @@ # conftest.py -# -# Auto-applies ``@pytest.mark.vcr`` to every collected test (see -# ``pytest_collection_modifyitems``) so live provider calls land in the -# Redis-backed VCR cache. The persister, header scrubbing and 2xx-only -# filtering live in ``tests/_vcr_redis_persister.py``; the cache key and -# 24h TTL match the llm_translation conftest. import asyncio import importlib @@ -25,7 +19,6 @@ from tests._vcr_redis_persister import ( # noqa: E402 ) -# Headers that must never be persisted to a cassette. _FILTERED_REQUEST_HEADERS = ( "authorization", "x-api-key", @@ -44,7 +37,6 @@ _FILTERED_REQUEST_HEADERS = ( "x-goog-user-project", ) -# Per-request response headers we strip so cassettes diff cleanly. _FILTERED_RESPONSE_HEADERS = ( "set-cookie", "x-request-id", @@ -70,8 +62,7 @@ def _scrub_response(response): def _before_record_response(response): - response = _scrub_response(response) - return filter_non_2xx_response(response) + return filter_non_2xx_response(_scrub_response(response)) @pytest.fixture(scope="module") @@ -152,17 +143,12 @@ def setup_and_teardown(): def pytest_collection_modifyitems(config, items): - # Auto-apply ``@pytest.mark.vcr`` so any provider call lands in the - # Redis cache. No respx files exist in this directory today; if any are - # added later, exclude them by filename here. Skip entirely when VCR - # is disabled (no REDIS_HOST or LITELLM_VCR_DISABLE=1). if not _vcr_disabled(): for item in items: if item.get_closest_marker("vcr") is not None: continue item.add_marker(pytest.mark.vcr) - # Preserve historical custom_logger ordering. custom_logger_tests = [ item for item in items if "custom_logger" in item.parent.name ] diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index ac87d00980..98b7fb16bc 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -4,12 +4,6 @@ # Mirrors the pattern in tests/local_testing/conftest.py: # - Function-scoped fixture resets litellm globals to true defaults # - Module-scoped reload only in single-process mode -# -# Also wires up the Redis-backed VCR cache. Every test in this directory is -# auto-marked with ``@pytest.mark.vcr`` (see ``pytest_collection_modifyitems``) -# unless its file appears in ``_RESPX_CONFLICTING_FILES`` — those use respx, -# which patches the same httpx transport vcrpy does. Cache key naming, TTL, -# and 2xx-only filtering live in ``tests/_vcr_redis_persister.py``. import asyncio import importlib @@ -30,19 +24,9 @@ from tests._vcr_redis_persister import ( # noqa: E402 ) -# --------------------------------------------------------------------------- -# VCR cassette infrastructure (pytest-recording + Redis) -# --------------------------------------------------------------------------- -# All tests in tests/llm_translation/ are auto-marked with ``@pytest.mark.vcr`` -# (excluding the respx-using files listed below). On cache miss vcrpy records -# the live response into Redis under ``litellm:vcr:cassette:`` with -# a 24h TTL; subsequent runs within that window replay without touching the -# network. Set ``LITELLM_VCR_DISABLE=1`` to skip VCR entirely (e.g. when -# debugging an upstream API change locally). - -# Test files that use ``respx`` to patch httpx. vcrpy patches the same -# transport, so applying both to the same test will make one of them silently -# win and the other look like a no-op. Skip auto-marking these. +# vcrpy and respx both patch the httpx transport — applying both makes one +# silently win. Files in this set use respx and are skipped by the +# auto-marker below. _RESPX_CONFLICTING_FILES = frozenset( { "test_azure_o_series.py", @@ -55,23 +39,14 @@ _RESPX_CONFLICTING_FILES = frozenset( "test_xai.py", } ) - -# The persister's own unit tests must not run inside a VCR cassette context — -# they call ``save_cassette`` / ``load_cassette`` directly against fakeredis -# and don't make HTTP calls, but auto-marking them would still wrap each -# test in a Redis lookup we don't want. _VCR_AUTO_MARKER_SKIP_FILES = _RESPX_CONFLICTING_FILES | frozenset( {"test_vcr_redis_persister.py"} ) -# Headers that must never be persisted to a cassette. Matched -# case-insensitively by vcrpy. _FILTERED_REQUEST_HEADERS = ( "authorization", "x-api-key", "anthropic-api-key", - # Strip ``anthropic-version`` so cassettes have a stable shape across - # SDK versions that bump the header. "anthropic-version", "openai-api-key", "azure-api-key", @@ -86,8 +61,6 @@ _FILTERED_REQUEST_HEADERS = ( "x-goog-user-project", ) -# Per-request response headers we strip so cassettes diff cleanly across -# re-records. _FILTERED_RESPONSE_HEADERS = ( "set-cookie", "x-request-id", @@ -102,7 +75,6 @@ _FILTERED_RESPONSE_HEADERS = ( def _scrub_response(response): - """Strip per-request response headers we don't want in the cassette.""" if not isinstance(response, dict): return response headers = response.get("headers") or {} @@ -114,32 +86,15 @@ def _scrub_response(response): def _before_record_response(response): - """Compose per-request scrubbing with the 2xx-only cache policy. - - Order matters: we scrub headers first so we don't leak request IDs even - on responses we end up dropping from the cassette mid-development. - """ - response = _scrub_response(response) - return filter_non_2xx_response(response) + return filter_non_2xx_response(_scrub_response(response)) @pytest.fixture(scope="module") def vcr_config(): - """Shared VCR config consumed by ``pytest-recording``. - - ``record_mode="once"`` is what makes this a useful daily cache: - - cassette absent (cache miss) → record the live call into Redis, - - cassette present (cache hit) → replay only. - 24h TTL on the Redis key means each new day's first run records against - live providers, surfacing API drift within a day instead of silently - serving stale responses forever. - """ return { "filter_headers": list(_FILTERED_REQUEST_HEADERS), "decode_compressed_response": True, "record_mode": "once", - # Match on full request shape so streaming vs non-streaming and - # different prompts produce distinct cassettes. "match_on": ( "method", "scheme", @@ -154,18 +109,12 @@ def vcr_config(): def _vcr_disabled() -> bool: - """VCR is disabled when explicitly opted out or when Redis isn't wired. - - No Redis means no cache to read from or write to — fall back to live - calls instead of silently writing YAML to disk (which we don't ship). - """ if os.environ.get("LITELLM_VCR_DISABLE") == "1": return True return not os.environ.get("REDIS_HOST") def pytest_recording_configure(config, vcr): - """Register the Redis-backed cassette persister.""" if _vcr_disabled(): return vcr.register_persister(make_redis_persister()) @@ -248,33 +197,18 @@ def setup_and_teardown(event_loop): # Add event_loop as a dependency event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) -# Number of attempts a vcr-marked test gets when recording against a live -# provider. Replay-only runs never reach the network so this only matters on -# cache miss / record mode. Tenacity-style exponential backoff is provided by -# the underlying provider SDKs (openai, anthropic) when they see 429/5xx, so -# bumping num_retries propagates retry-with-backoff for free. _VCR_RECORD_RETRIES = 3 @pytest.fixture(autouse=True) def _vcr_record_retries(setup_and_teardown, request): - """Configure record-time retries for ``@pytest.mark.vcr`` tests. - - Depends on ``setup_and_teardown`` so this runs *after* the per-test - ``importlib.reload(litellm)`` resets ``num_retries`` back to None. - """ + # Depends on setup_and_teardown so this runs after litellm is reloaded. if request.node.get_closest_marker("vcr") is None: return litellm.num_retries = _VCR_RECORD_RETRIES def pytest_collection_modifyitems(config, items): - # 1. Auto-apply ``@pytest.mark.vcr`` to every collected test in this - # directory so any provider call lands in the Redis cache. Skip files - # that use respx (it patches the same transport vcrpy does) and the - # persister's own unit tests. Skip entirely if VCR is disabled (no - # REDIS_HOST or LITELLM_VCR_DISABLE=1) so dev runs without Redis - # don't go through cassette logic at all. if not _vcr_disabled(): for item in items: filename = os.path.basename(str(item.fspath)) @@ -284,7 +218,6 @@ def pytest_collection_modifyitems(config, items): continue item.add_marker(pytest.mark.vcr) - # 2. Preserve the historical ordering of custom_logger tests vs the rest. custom_logger_tests = [ item for item in items if "custom_logger" in item.parent.name ] diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index ae42155642..8c5ac83001 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -1888,12 +1888,6 @@ def test_metadata_filter_applies_to_azure_anthropic(): def test_anthropic_basic_completion_replay(): - """Smoke-test that a vanilla Anthropic completion replays from a cassette. - - Exercises the full LiteLLM transformation pipeline (request shaping + - response parsing) against a real-shape Anthropic payload. The cassette - is loaded from the Redis-backed VCR cache configured in conftest.py. - """ response = litellm.completion( model="anthropic/claude-sonnet-4-5-20250929", messages=[{"role": "user", "content": "Hello!"}], @@ -1903,17 +1897,10 @@ def test_anthropic_basic_completion_replay(): assert response.choices[0].message.content == ("Hello! How can I help you today?") assert response.usage.prompt_tokens == 12 assert response.usage.completion_tokens == 11 - # Anthropic sets stop_reason="end_turn" → litellm normalises to "stop" assert response.choices[0].finish_reason == "stop" def test_anthropic_streaming_completion_replay(): - """Replay a streaming Anthropic completion from the VCR cache. - - Exercises the SSE chunk parser and the public streaming surface — any - regression in the streaming transformation surfaces here because the - cassette captures every ``content_block_delta`` event Anthropic emits. - """ stream = litellm.completion( model="anthropic/claude-sonnet-4-5-20250929", messages=[{"role": "user", "content": "Hello!"}], diff --git a/tests/llm_translation/test_vcr_redis_persister.py b/tests/llm_translation/test_vcr_redis_persister.py index 9ca23410bd..6de6283e6b 100644 --- a/tests/llm_translation/test_vcr_redis_persister.py +++ b/tests/llm_translation/test_vcr_redis_persister.py @@ -1,19 +1,3 @@ -"""Tests for the Redis-backed vcrpy cassette persister. - -These cover the three behaviours we actually rely on in CI: - -1. ``save_cassette`` followed by ``load_cassette`` returns the same - request/response pairs (roundtrip via the real vcrpy serializer). -2. Saved keys expire after ~24h so the cache auto-refreshes against live - providers without manual ``make`` runs. -3. ``load_cassette`` raises ``CassetteNotFoundError`` on a miss, so vcrpy's - record-mode machinery falls through to a live HTTP call instead of - silently matching against an empty cassette. - -We also pin the 2xx-only filter so a transient 5xx/429 from the provider -can't be baked into the cache for the rest of the TTL window. -""" - from __future__ import annotations import os @@ -25,8 +9,6 @@ from vcr.persisters.filesystem import CassetteNotFoundError from vcr.request import Request from vcr.serializers import yamlserializer -# Make tests/ importable as a package so we can pull the shared persister -# without depending on pytest's CWD. sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) from tests._vcr_redis_persister import ( # noqa: E402 @@ -38,7 +20,6 @@ from tests._vcr_redis_persister import ( # noqa: E402 def _sample_cassette_dict(): - """Build a minimal cassette payload that exercises serialize/deserialize.""" request = Request( method="POST", uri="https://api.anthropic.com/v1/messages", @@ -48,8 +29,6 @@ def _sample_cassette_dict(): response = { "status": {"code": 200, "message": "OK"}, "headers": {"content-type": ["application/json"]}, - # vcrpy stores response bodies as bytes; mirror that so the - # roundtrip assertion exercises real-world serialization shapes. "body": {"string": b'{"id":"msg_1","type":"message"}'}, } return {"requests": [request], "responses": [response]} @@ -61,10 +40,7 @@ def _persister_with_fake_redis(): def test_save_then_load_roundtrips_cassette_content(): - """A saved cassette must come back from ``load_cassette`` identical to - what was put in. If serialize/deserialize ever drift (e.g. encoding bug) - every replay-mode test in the suite breaks; this catches it cheaply.""" - fake, persister = _persister_with_fake_redis() + _, persister = _persister_with_fake_redis() cassette_path = "tests/llm_translation/cassettes/test_x/test_y.yaml" persister.save_cassette(cassette_path, _sample_cassette_dict(), yamlserializer) @@ -79,27 +55,16 @@ def test_save_then_load_roundtrips_cassette_content(): def test_saved_key_has_24h_ttl(): - """The whole point of the Redis backend is that entries auto-expire after - 24h so each daily CI run re-records against live providers. If the TTL - isn't being applied, the cache never refreshes and we silently mask - upstream API drift.""" fake, persister = _persister_with_fake_redis() cassette_path = "tests/llm_translation/cassettes/test_x/test_ttl.yaml" persister.save_cassette(cassette_path, _sample_cassette_dict(), yamlserializer) ttl = fake.ttl(redis_key_for(cassette_path)) - assert ttl > 0, "key was saved without an expiry — would never refresh" - assert ttl <= CASSETTE_TTL_SECONDS - assert ttl >= CASSETTE_TTL_SECONDS - 5 # allow tiny clock slack + assert CASSETTE_TTL_SECONDS - 5 <= ttl <= CASSETTE_TTL_SECONDS def test_load_missing_key_raises_cassette_not_found(): - """Cache miss must surface as ``CassetteNotFoundError``. vcrpy's record - machinery catches that exception and falls through to the live HTTP - call; if we returned empty/None instead, vcrpy would treat it as a - cassette with zero matching requests and the test would fail with a - confusing ``CannotOverwriteExistingCassetteException``.""" _, persister = _persister_with_fake_redis() with pytest.raises(CassetteNotFoundError): persister.load_cassette("never/recorded.yaml", yamlserializer) @@ -116,24 +81,19 @@ def test_load_missing_key_raises_cassette_not_found(): (400, True), (401, True), (404, True), - (429, True), # rate limit — must never be cached - (500, True), # transient 5xx — must never be cached + (429, True), + (500, True), (502, True), (503, True), ], ) def test_only_2xx_responses_are_cached(status_code, expect_dropped): - """Pin the cache-poisoning protection: a non-2xx must be dropped from - the cassette (returned as ``None`` from the hook) so a transient 429 - or 503 doesn't get pinned for the rest of the TTL window. 2xx - responses must pass through untouched.""" response = { "status": {"code": status_code, "message": "X"}, "headers": {}, "body": {"string": ""}, } result = filter_non_2xx_response(response) - if expect_dropped: - assert result is None - else: + assert (result is None) == expect_dropped + if not expect_dropped: assert result is response From 73594262ee5ec8d6ae8633a82439e5770c527063 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Apr 2026 22:42:30 +0000 Subject: [PATCH 08/30] tests(vcr): drop redundant num_retries=3 layer for vcr-marked tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provider SDKs already retry transient 5xx/429 with exponential backoff (default max_retries=2), and pytest.mark.flaky covers test-level retries on top of that. Setting litellm.num_retries=3 here just multiplied the existing layers — worst case 6 (flaky) x 3 (this) x 2 (CI rerunfailures) = 36 attempts on a single test. Removing it keeps SDK-level network-blip protection intact and shortens worst-case latency on cache-miss runs. --- tests/llm_translation/conftest.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 98b7fb16bc..7f7a742404 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -197,17 +197,6 @@ def setup_and_teardown(event_loop): # Add event_loop as a dependency event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) -_VCR_RECORD_RETRIES = 3 - - -@pytest.fixture(autouse=True) -def _vcr_record_retries(setup_and_teardown, request): - # Depends on setup_and_teardown so this runs after litellm is reloaded. - if request.node.get_closest_marker("vcr") is None: - return - litellm.num_retries = _VCR_RECORD_RETRIES - - def pytest_collection_modifyitems(config, items): if not _vcr_disabled(): for item in items: From 59d59017662bcba72e6b32e82c0d2636bb826795 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Apr 2026 22:57:21 +0000 Subject: [PATCH 09/30] tests(vcr): allow playback repeats so duplicate intra-test requests serve from cache --- tests/llm_responses_api_testing/conftest.py | 1 + tests/llm_translation/conftest.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index 7209d2f957..c8d3bb1b10 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -71,6 +71,7 @@ def vcr_config(): "filter_headers": list(_FILTERED_REQUEST_HEADERS), "decode_compressed_response": True, "record_mode": "once", + "allow_playback_repeats": True, "match_on": ( "method", "scheme", diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 7f7a742404..ed8836e2fe 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -95,6 +95,7 @@ def vcr_config(): "filter_headers": list(_FILTERED_REQUEST_HEADERS), "decode_compressed_response": True, "record_mode": "once", + "allow_playback_repeats": True, "match_on": ( "method", "scheme", From efdeff89d81b08fe8c1ecd8eff53f96446daeb11 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Apr 2026 23:01:10 +0000 Subject: [PATCH 10/30] fix(llm_request_utils): handle None proxy_server_request without AttributeError --- litellm/litellm_core_utils/llm_request_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index f5f28822ca..c4533964d4 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -78,7 +78,7 @@ def get_proxy_server_request_headers(litellm_params: Optional[dict]) -> dict: return {} proxy_request_headers = ( - litellm_params.get("proxy_server_request", {}).get("headers", {}) or {} + (litellm_params.get("proxy_server_request") or {}).get("headers") or {} ) return proxy_request_headers From f55a710e9241980dfcd993c1b3c76f2982ad9d2f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Apr 2026 23:08:01 +0000 Subject: [PATCH 11/30] tests(vcr): accept REDIS_URL / REDIS_SSL_URL for managed Redis with TLS --- tests/_flush_vcr_cache.py | 16 ++++++----- tests/_vcr_redis_persister.py | 31 ++++++++++++++++----- tests/llm_responses_api_testing/conftest.py | 4 ++- tests/llm_translation/conftest.py | 4 ++- 4 files changed, 39 insertions(+), 16 deletions(-) diff --git a/tests/_flush_vcr_cache.py b/tests/_flush_vcr_cache.py index dfaba2367c..c78958fb01 100644 --- a/tests/_flush_vcr_cache.py +++ b/tests/_flush_vcr_cache.py @@ -5,18 +5,20 @@ import sys import redis +from tests._vcr_redis_persister import _redis_url_from_env + PREFIX = "litellm:vcr:cassette:" SCAN_BATCH = 500 def _client() -> redis.Redis: - host = os.environ.get("REDIS_HOST") - if not host: - sys.exit("REDIS_HOST is not set; cannot flush VCR cache") - return redis.Redis( - host=host, - port=int(os.environ.get("REDIS_PORT", 6379)), - password=os.environ.get("REDIS_PASSWORD") or None, + url = _redis_url_from_env() + if not url: + sys.exit( + "Set REDIS_URL, REDIS_SSL_URL, or REDIS_HOST to flush the VCR cache" + ) + return redis.Redis.from_url( + url, socket_timeout=5, socket_connect_timeout=5, decode_responses=False, diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index 3b1e456c0a..32cf5c1c0e 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -14,16 +14,33 @@ def redis_key_for(cassette_path: str) -> str: return f"{REDIS_KEY_PREFIX}{os.path.relpath(str(cassette_path))}" +def _redis_url_from_env() -> Optional[str]: + for var in ("REDIS_URL", "REDIS_SSL_URL"): + url = os.environ.get(var) + if url: + return url + host = os.environ.get("REDIS_HOST") + if not host: + return None + scheme = "rediss" if os.environ.get("REDIS_SSL", "").lower() == "true" else "redis" + auth = "" + if os.environ.get("REDIS_PASSWORD"): + user = os.environ.get("REDIS_USERNAME", "") + auth = f"{user}:{os.environ['REDIS_PASSWORD']}@" + port = os.environ.get("REDIS_PORT", "6379") + return f"{scheme}://{auth}{host}:{port}" + + def _build_default_client(): import redis - host = os.environ.get("REDIS_HOST") - if not host: - raise RuntimeError("REDIS_HOST is not set") - return redis.Redis( - host=host, - port=int(os.environ.get("REDIS_PORT", 6379)), - password=os.environ.get("REDIS_PASSWORD") or None, + url = _redis_url_from_env() + if not url: + raise RuntimeError( + "Set REDIS_URL, REDIS_SSL_URL, or REDIS_HOST to enable the VCR persister" + ) + return redis.Redis.from_url( + url, socket_timeout=5, socket_connect_timeout=5, decode_responses=False, diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index c8d3bb1b10..87ed6218ea 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -88,7 +88,9 @@ def vcr_config(): def _vcr_disabled() -> bool: if os.environ.get("LITELLM_VCR_DISABLE") == "1": return True - return not os.environ.get("REDIS_HOST") + return not any( + os.environ.get(var) for var in ("REDIS_URL", "REDIS_SSL_URL", "REDIS_HOST") + ) def pytest_recording_configure(config, vcr): diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index ed8836e2fe..9da7c98138 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -112,7 +112,9 @@ def vcr_config(): def _vcr_disabled() -> bool: if os.environ.get("LITELLM_VCR_DISABLE") == "1": return True - return not os.environ.get("REDIS_HOST") + return not any( + os.environ.get(var) for var in ("REDIS_URL", "REDIS_SSL_URL", "REDIS_HOST") + ) def pytest_recording_configure(config, vcr): From 468b8490728f5692461321e9a20912a7fef8119a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Apr 2026 23:37:17 +0000 Subject: [PATCH 12/30] tests(vcr): drop YAML/cassettes-directory metaphor from Redis keys --- tests/_vcr_redis_persister.py | 6 ++++- tests/llm_translation/Readme.md | 2 +- .../test_vcr_redis_persister.py | 26 ++++++++++++++----- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index 32cf5c1c0e..6cbaff8441 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -11,7 +11,11 @@ REDIS_KEY_PREFIX = "litellm:vcr:cassette:" def redis_key_for(cassette_path: str) -> str: - return f"{REDIS_KEY_PREFIX}{os.path.relpath(str(cassette_path))}" + rel = os.path.relpath(str(cassette_path)) + if rel.endswith(".yaml"): + rel = rel[: -len(".yaml")] + rel = rel.replace("/cassettes/", "/").lstrip("./") + return f"{REDIS_KEY_PREFIX}{rel}" def _redis_url_from_env() -> Optional[str]: diff --git a/tests/llm_translation/Readme.md b/tests/llm_translation/Readme.md index e684c73202..958adbd975 100644 --- a/tests/llm_translation/Readme.md +++ b/tests/llm_translation/Readme.md @@ -7,7 +7,7 @@ Name of the test file is the name of the LLM provider - e.g. `test_openai.py` is Every test in this directory is auto-decorated with `@pytest.mark.vcr` (via `conftest.py`). The first time a test runs we hit the live provider and record the HTTP exchange into Redis under -`litellm:vcr:cassette:`. Every subsequent run within 24h replays +`litellm:vcr:cassette:`. Every subsequent run within 24h replays from Redis without touching the network. The 24h TTL means each new day's first run records again, so upstream API drift surfaces within a day. diff --git a/tests/llm_translation/test_vcr_redis_persister.py b/tests/llm_translation/test_vcr_redis_persister.py index 6de6283e6b..d10aa388c4 100644 --- a/tests/llm_translation/test_vcr_redis_persister.py +++ b/tests/llm_translation/test_vcr_redis_persister.py @@ -41,10 +41,10 @@ def _persister_with_fake_redis(): def test_save_then_load_roundtrips_cassette_content(): _, persister = _persister_with_fake_redis() - cassette_path = "tests/llm_translation/cassettes/test_x/test_y.yaml" + cassette_id = "tests/llm_translation/test_x/test_y" - persister.save_cassette(cassette_path, _sample_cassette_dict(), yamlserializer) - requests, responses = persister.load_cassette(cassette_path, yamlserializer) + persister.save_cassette(cassette_id, _sample_cassette_dict(), yamlserializer) + requests, responses = persister.load_cassette(cassette_id, yamlserializer) assert len(requests) == 1 assert len(responses) == 1 @@ -56,18 +56,30 @@ def test_save_then_load_roundtrips_cassette_content(): def test_saved_key_has_24h_ttl(): fake, persister = _persister_with_fake_redis() - cassette_path = "tests/llm_translation/cassettes/test_x/test_ttl.yaml" + cassette_id = "tests/llm_translation/test_x/test_ttl" - persister.save_cassette(cassette_path, _sample_cassette_dict(), yamlserializer) + persister.save_cassette(cassette_id, _sample_cassette_dict(), yamlserializer) - ttl = fake.ttl(redis_key_for(cassette_path)) + ttl = fake.ttl(redis_key_for(cassette_id)) assert CASSETTE_TTL_SECONDS - 5 <= ttl <= CASSETTE_TTL_SECONDS def test_load_missing_key_raises_cassette_not_found(): _, persister = _persister_with_fake_redis() with pytest.raises(CassetteNotFoundError): - persister.load_cassette("never/recorded.yaml", yamlserializer) + persister.load_cassette("never/recorded", yamlserializer) + + +def test_redis_key_normalizes_path_passed_by_pytest_recording(): + # pytest-recording passes paths shaped like + # ``/cassettes//.yaml``. The persister stores them + # under a clean test-identifier key — no extension, no ``cassettes/`` + # directory segment — so ``redis-cli keys`` reads as test IDs. + raw = "tests/llm_translation/cassettes/test_anthropic/test_streaming.yaml" + assert ( + redis_key_for(raw) + == "litellm:vcr:cassette:tests/llm_translation/test_anthropic/test_streaming" + ) @pytest.mark.parametrize( From f6a37a6a1547cdb56cb7e66cf39ab8c58b72694d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Apr 2026 16:52:25 -0700 Subject: [PATCH 13/30] style: reformat to pass ci --- litellm/litellm_core_utils/llm_request_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index c4533964d4..7be7085297 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -77,8 +77,8 @@ def get_proxy_server_request_headers(litellm_params: Optional[dict]) -> dict: if litellm_params is None: return {} - proxy_request_headers = ( - (litellm_params.get("proxy_server_request") or {}).get("headers") or {} - ) + proxy_request_headers = (litellm_params.get("proxy_server_request") or {}).get( + "headers" + ) or {} return proxy_request_headers From 68db1c5e9e78e07208e497887535369fb4d7610b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Apr 2026 17:22:21 -0700 Subject: [PATCH 14/30] tests(vcr): switch to record_mode=new_episodes to avoid partial-cassette poisoning record_mode='once' refused to add new requests once any cassette existed in Redis. Combined with filter_non_2xx_response (which drops non-2xx responses from the saved cassette) and a 24h shared-Redis TTL, a single transient API failure mid-test left the cassette stuck with only the leading non-API requests (e.g. the model_prices fetch from raw.githubusercontent.com), and every subsequent run for the next 24h errored with 'Can't overwrite existing cassette'. new_episodes records anything not already present, so partially populated cassettes recover on the next run instead of poisoning the suite for a full TTL window. --- tests/llm_responses_api_testing/conftest.py | 2 +- tests/llm_translation/conftest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index 87ed6218ea..01257cd26b 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -70,7 +70,7 @@ def vcr_config(): return { "filter_headers": list(_FILTERED_REQUEST_HEADERS), "decode_compressed_response": True, - "record_mode": "once", + "record_mode": "new_episodes", "allow_playback_repeats": True, "match_on": ( "method", diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 9da7c98138..132ebd57ad 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -94,7 +94,7 @@ def vcr_config(): return { "filter_headers": list(_FILTERED_REQUEST_HEADERS), "decode_compressed_response": True, - "record_mode": "once", + "record_mode": "new_episodes", "allow_playback_repeats": True, "match_on": ( "method", From 265a94cd602f9cdadb58edd6eac34325b5ccaef4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Apr 2026 17:42:17 -0700 Subject: [PATCH 15/30] tests(vcr): force pure-httpx transport when VCR is active litellm's default LiteLLMAiohttpTransport routes requests through aiohttp, which sits below httpx and is invisible to vcrpy's httpx-stub interception. Under vcrpy + aiohttp, requests reach the real network but responses come back through the stubbed httpx transport as empty 200s, surfacing as 'Unable to get json response - Expecting value: line 1 column 1 (char 0)' in providers like Anthropic, Gemini, and any other path that exercises the aiohttp transport. Disabling the aiohttp transport when the VCR persister is registered forces all calls through pure httpx, which vcrpy can record and replay correctly. --- tests/llm_responses_api_testing/conftest.py | 5 +++++ tests/llm_translation/conftest.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index 01257cd26b..948debbf95 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -97,6 +97,11 @@ def pytest_recording_configure(config, vcr): if _vcr_disabled(): return vcr.register_persister(make_redis_persister()) + # vcrpy patches httpx's transport; litellm's default AiohttpTransport + # routes around httpx and produces empty responses under the patched + # transport. Force pure-httpx transport so vcrpy can record/replay. + litellm.disable_aiohttp_transport = True + os.environ["DISABLE_AIOHTTP_TRANSPORT"] = "True" @pytest.fixture(scope="session") diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 132ebd57ad..2215b1c24b 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -121,6 +121,11 @@ def pytest_recording_configure(config, vcr): if _vcr_disabled(): return vcr.register_persister(make_redis_persister()) + # vcrpy patches httpx's transport; litellm's default AiohttpTransport + # routes around httpx and produces empty responses under the patched + # transport. Force pure-httpx transport so vcrpy can record/replay. + litellm.disable_aiohttp_transport = True + os.environ["DISABLE_AIOHTTP_TRANSPORT"] = "True" # --------------------------------------------------------------------------- From 8bdc46ea74155081c4d043a90f171de2c0c9e9fa Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Apr 2026 17:55:27 -0700 Subject: [PATCH 16/30] fix(responses): omit empty JSON body on DELETE response API requests Azure OpenAI's responses-API DELETE endpoint rejects requests that carry a JSON body with: "Unexpected body with size 2. This API method does not accept a request body.". The default LiteLLMAiohttpTransport silently elides empty-dict bodies on DELETE so this was masked, but the pure-httpx transport (used when DISABLE_AIOHTTP_TRANSPORT=True or under vcrpy/respx patching) sends literal '{}' (2 bytes), which Azure rejects. Only attach json= when the provider's transform actually returned a non-empty dict; otherwise issue a bodyless DELETE. --- litellm/llms/custom_httpx/llm_http_handler.py | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index a34b73b531..19939e6d1e 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2536,9 +2536,16 @@ class BaseLLMHTTPHandler: ) try: - response = await async_httpx_client.delete( - url=url, headers=headers, json=data, timeout=timeout - ) + # Only send a JSON body when the provider supplied request data; + # some providers (e.g. Azure OpenAI) reject DELETE with a body. + delete_kwargs: Dict[str, Any] = { + "url": url, + "headers": headers, + "timeout": timeout, + } + if data: + delete_kwargs["json"] = data + response = await async_httpx_client.delete(**delete_kwargs) except Exception as e: raise self._handle_error( @@ -2620,9 +2627,16 @@ class BaseLLMHTTPHandler: ) try: - response = sync_httpx_client.delete( - url=url, headers=headers, json=data, timeout=timeout - ) + # Only send a JSON body when the provider supplied request data; + # some providers (e.g. Azure OpenAI) reject DELETE with a body. + delete_kwargs: Dict[str, Any] = { + "url": url, + "headers": headers, + "timeout": timeout, + } + if data: + delete_kwargs["json"] = data + response = sync_httpx_client.delete(**delete_kwargs) except Exception as e: raise self._handle_error( From 95bce9a72eb8a17b92cc561f9a2a2d3515cb305d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Apr 2026 18:35:01 -0700 Subject: [PATCH 17/30] tests(vcr): assert response shape, not exact bytes, in replay tests The Anthropic replay tests hardcoded specific token counts and content strings ('Hello! How can I help you today?', prompt_tokens == 12). On a fresh CI Redis those values must match a pre-recorded cassette that doesn't exist, so the first run hits the live API and gets different real bytes back. Assert on shape instead: non-empty content, positive token counts, finish_reason in the known set, and (for streaming) more than one chunk. The tests still exercise the full transformation pipeline end-to-end and catch shape regressions; drift in the exact text/token counts is expected and now tolerated. --- .../test_anthropic_completion.py | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index 8c5ac83001..faff76edb6 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -1888,19 +1888,28 @@ def test_metadata_filter_applies_to_azure_anthropic(): def test_anthropic_basic_completion_replay(): + """Smoke-test the Anthropic completion pipeline end-to-end via VCR. + + Asserts on response shape rather than specific bytes, so the test is + valid both on a fresh CI Redis (records on first run) and on a hot + cache (replays). Drift in the *shape* of Anthropic's response surfaces + here; drift in the exact text/token counts is expected and ignored. + """ response = litellm.completion( model="anthropic/claude-sonnet-4-5-20250929", messages=[{"role": "user", "content": "Hello!"}], ) assert response is not None - assert response.choices[0].message.content == ("Hello! How can I help you today?") - assert response.usage.prompt_tokens == 12 - assert response.usage.completion_tokens == 11 - assert response.choices[0].finish_reason == "stop" + content = response.choices[0].message.content + assert isinstance(content, str) and content.strip(), content + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.choices[0].finish_reason in {"stop", "length"} def test_anthropic_streaming_completion_replay(): + """Same as above for the streaming path; asserts on shape, not bytes.""" stream = litellm.completion( model="anthropic/claude-sonnet-4-5-20250929", messages=[{"role": "user", "content": "Hello!"}], @@ -1909,7 +1918,9 @@ def test_anthropic_streaming_completion_replay(): collected_text = "" finish_reason = None + chunk_count = 0 for chunk in stream: + chunk_count += 1 if not chunk.choices: continue delta = chunk.choices[0].delta @@ -1918,5 +1929,6 @@ def test_anthropic_streaming_completion_replay(): if chunk.choices[0].finish_reason: finish_reason = chunk.choices[0].finish_reason - assert collected_text == "Hello from LiteLLM!" - assert finish_reason == "stop" + assert chunk_count > 1, "expected multiple SSE chunks from streaming response" + assert collected_text.strip(), collected_text + assert finish_reason in {"stop", "length"} From 687ff32616d5f05015682763d30abe5d6b1ba5d9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Apr 2026 18:43:06 -0700 Subject: [PATCH 18/30] tests(vcr): patch vcrpy aiohttp record path instead of forcing httpx transport vcrpy's aiohttp stub captures response bodies via 'await response.read()', which drains aiohttp's StreamReader. Downstream consumers of the same ClientResponse (litellm's AiohttpResponseStream, which iterates response.content.iter_chunked) then see an empty body and surface as JSON 'Expecting value: line 1 column 1 (char 0)' errors on every record-path call. The previous workaround set litellm.disable_aiohttp_transport=True for the whole VCR-active session, which made the tests exercise pure httpx instead of the production aiohttp transport. That hid the production transport from coverage and surfaced its own bugs (e.g. the Azure DELETE-with-empty-body case fixed in upstream staging). Replace the workaround with a targeted monkey-patch that re-feeds the captured body into the StreamReader via unread_data after vcrpy records it. Tests now run through the same transport customers do, both on first record and on replay, for both unary and streaming endpoints. Verified locally against api.anthropic.com with the production LiteLLMAiohttpTransport: record path passes (real network, 4.2s), replay path passes (Redis cache, 1.8s). --- tests/_vcr_redis_persister.py | 43 +++++++++++++++++++++ tests/llm_responses_api_testing/conftest.py | 10 ++--- tests/llm_translation/conftest.py | 10 ++--- 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index 6cbaff8441..d8cda93098 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -85,3 +85,46 @@ def filter_non_2xx_response(response): if not isinstance(code, int): return response return response if 200 <= code < 300 else None + + +_PATCHED_AIOHTTP_RECORD = False + + +def patch_vcrpy_aiohttp_record_path() -> None: + """Make vcrpy's aiohttp record path leave the response body re-readable. + + vcrpy.stubs.aiohttp_stubs.record_response calls ``await response.read()`` + to capture the body for the cassette, which drains aiohttp's StreamReader. + Downstream consumers of the same ClientResponse (e.g. + ``litellm.llms.custom_httpx.aiohttp_transport.AiohttpResponseStream``, + which iterates ``response.content.iter_chunked``) then see an empty body + and surface as ``Expecting value: line 1 column 1 (char 0)`` JSON errors. + + Re-feed the captured bytes back into the StreamReader via ``unread_data`` + so the body remains available to whoever holds the ClientResponse next. + Idempotent; safe to call from multiple conftests. + """ + global _PATCHED_AIOHTTP_RECORD + if _PATCHED_AIOHTTP_RECORD: + return + try: + import vcr.stubs.aiohttp_stubs as _aiohttp_stubs + except ImportError: # pragma: no cover - aiohttp not installed in env + return + + _orig_record_response = _aiohttp_stubs.record_response + + async def _record_response_preserving_body(cassette, vcr_request, response): + await _orig_record_response(cassette, vcr_request, response) + body = getattr(response, "_body", None) or b"" + if body: + try: + response.content.unread_data(body) + except Exception: + # If aiohttp removes unread_data in a future release we want + # the test to fail loudly via the original empty-body + # symptom rather than mask the regression here. + pass + + _aiohttp_stubs.record_response = _record_response_preserving_body + _PATCHED_AIOHTTP_RECORD = True diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index 948debbf95..02cc88ee0d 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -16,6 +16,7 @@ import litellm # noqa: E402 from tests._vcr_redis_persister import ( # noqa: E402 filter_non_2xx_response, make_redis_persister, + patch_vcrpy_aiohttp_record_path, ) @@ -97,11 +98,10 @@ def pytest_recording_configure(config, vcr): if _vcr_disabled(): return vcr.register_persister(make_redis_persister()) - # vcrpy patches httpx's transport; litellm's default AiohttpTransport - # routes around httpx and produces empty responses under the patched - # transport. Force pure-httpx transport so vcrpy can record/replay. - litellm.disable_aiohttp_transport = True - os.environ["DISABLE_AIOHTTP_TRANSPORT"] = "True" + # vcrpy's aiohttp record path drains the response stream via + # ``await response.read()``; without the patch, downstream consumers + # (litellm's AiohttpResponseStream) see an empty body on first record. + patch_vcrpy_aiohttp_record_path() @pytest.fixture(scope="session") diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 2215b1c24b..2d8623b836 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -21,6 +21,7 @@ import litellm # noqa: E402 from tests._vcr_redis_persister import ( # noqa: E402 filter_non_2xx_response, make_redis_persister, + patch_vcrpy_aiohttp_record_path, ) @@ -121,11 +122,10 @@ def pytest_recording_configure(config, vcr): if _vcr_disabled(): return vcr.register_persister(make_redis_persister()) - # vcrpy patches httpx's transport; litellm's default AiohttpTransport - # routes around httpx and produces empty responses under the patched - # transport. Force pure-httpx transport so vcrpy can record/replay. - litellm.disable_aiohttp_transport = True - os.environ["DISABLE_AIOHTTP_TRANSPORT"] = "True" + # vcrpy's aiohttp record path drains the response stream via + # ``await response.read()``; without the patch, downstream consumers + # (litellm's AiohttpResponseStream) see an empty body on first record. + patch_vcrpy_aiohttp_record_path() # --------------------------------------------------------------------------- From 67287460e57c55dc09b6f6d9df30bedd2e8d1de5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Apr 2026 18:45:56 -0700 Subject: [PATCH 19/30] tests(vcr): drop redundant comments and docstrings --- tests/_vcr_redis_persister.py | 29 ++++--------------- tests/llm_responses_api_testing/conftest.py | 3 -- tests/llm_translation/conftest.py | 3 -- .../test_anthropic_completion.py | 8 ----- 4 files changed, 5 insertions(+), 38 deletions(-) diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index d8cda93098..f9a5ee1aa8 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -91,26 +91,13 @@ _PATCHED_AIOHTTP_RECORD = False def patch_vcrpy_aiohttp_record_path() -> None: - """Make vcrpy's aiohttp record path leave the response body re-readable. - - vcrpy.stubs.aiohttp_stubs.record_response calls ``await response.read()`` - to capture the body for the cassette, which drains aiohttp's StreamReader. - Downstream consumers of the same ClientResponse (e.g. - ``litellm.llms.custom_httpx.aiohttp_transport.AiohttpResponseStream``, - which iterates ``response.content.iter_chunked``) then see an empty body - and surface as ``Expecting value: line 1 column 1 (char 0)`` JSON errors. - - Re-feed the captured bytes back into the StreamReader via ``unread_data`` - so the body remains available to whoever holds the ClientResponse next. - Idempotent; safe to call from multiple conftests. - """ + """Re-feed the response body into aiohttp's StreamReader after vcrpy's + record_response drains it, so downstream consumers (e.g. + LiteLLMAiohttpTransport.AiohttpResponseStream) can still read it.""" global _PATCHED_AIOHTTP_RECORD if _PATCHED_AIOHTTP_RECORD: return - try: - import vcr.stubs.aiohttp_stubs as _aiohttp_stubs - except ImportError: # pragma: no cover - aiohttp not installed in env - return + import vcr.stubs.aiohttp_stubs as _aiohttp_stubs _orig_record_response = _aiohttp_stubs.record_response @@ -118,13 +105,7 @@ def patch_vcrpy_aiohttp_record_path() -> None: await _orig_record_response(cassette, vcr_request, response) body = getattr(response, "_body", None) or b"" if body: - try: - response.content.unread_data(body) - except Exception: - # If aiohttp removes unread_data in a future release we want - # the test to fail loudly via the original empty-body - # symptom rather than mask the regression here. - pass + response.content.unread_data(body) _aiohttp_stubs.record_response = _record_response_preserving_body _PATCHED_AIOHTTP_RECORD = True diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index 02cc88ee0d..b58508d5ae 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -98,9 +98,6 @@ def pytest_recording_configure(config, vcr): if _vcr_disabled(): return vcr.register_persister(make_redis_persister()) - # vcrpy's aiohttp record path drains the response stream via - # ``await response.read()``; without the patch, downstream consumers - # (litellm's AiohttpResponseStream) see an empty body on first record. patch_vcrpy_aiohttp_record_path() diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 2d8623b836..6969b1167d 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -122,9 +122,6 @@ def pytest_recording_configure(config, vcr): if _vcr_disabled(): return vcr.register_persister(make_redis_persister()) - # vcrpy's aiohttp record path drains the response stream via - # ``await response.read()``; without the patch, downstream consumers - # (litellm's AiohttpResponseStream) see an empty body on first record. patch_vcrpy_aiohttp_record_path() diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index faff76edb6..2c21928210 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -1888,13 +1888,6 @@ def test_metadata_filter_applies_to_azure_anthropic(): def test_anthropic_basic_completion_replay(): - """Smoke-test the Anthropic completion pipeline end-to-end via VCR. - - Asserts on response shape rather than specific bytes, so the test is - valid both on a fresh CI Redis (records on first run) and on a hot - cache (replays). Drift in the *shape* of Anthropic's response surfaces - here; drift in the exact text/token counts is expected and ignored. - """ response = litellm.completion( model="anthropic/claude-sonnet-4-5-20250929", messages=[{"role": "user", "content": "Hello!"}], @@ -1909,7 +1902,6 @@ def test_anthropic_basic_completion_replay(): def test_anthropic_streaming_completion_replay(): - """Same as above for the streaming path; asserts on shape, not bytes.""" stream = litellm.completion( model="anthropic/claude-sonnet-4-5-20250929", messages=[{"role": "user", "content": "Hello!"}], From 4c695576214b9a4b2a1d6959b5107aea305e1a42 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 1 May 2026 12:32:59 -0700 Subject: [PATCH 20/30] tests(vcr): isolate cassette redis to CASSETTE_REDIS_URL Stop falling back to REDIS_URL/REDIS_SSL_URL/REDIS_HOST for the VCR persister. Sharing a Redis with the application cache risks cassettes being wiped by tests that flush the app Redis. --- tests/_flush_vcr_cache.py | 6 ++--- tests/_vcr_redis_persister.py | 26 +++++++++------------ tests/llm_responses_api_testing/conftest.py | 7 +++--- tests/llm_translation/conftest.py | 7 +++--- 4 files changed, 21 insertions(+), 25 deletions(-) diff --git a/tests/_flush_vcr_cache.py b/tests/_flush_vcr_cache.py index c78958fb01..d236c88fa3 100644 --- a/tests/_flush_vcr_cache.py +++ b/tests/_flush_vcr_cache.py @@ -5,7 +5,7 @@ import sys import redis -from tests._vcr_redis_persister import _redis_url_from_env +from tests._vcr_redis_persister import CASSETTE_REDIS_URL_ENV, _redis_url_from_env PREFIX = "litellm:vcr:cassette:" SCAN_BATCH = 500 @@ -14,9 +14,7 @@ SCAN_BATCH = 500 def _client() -> redis.Redis: url = _redis_url_from_env() if not url: - sys.exit( - "Set REDIS_URL, REDIS_SSL_URL, or REDIS_HOST to flush the VCR cache" - ) + sys.exit(f"Set {CASSETTE_REDIS_URL_ENV} to flush the VCR cache") return redis.Redis.from_url( url, socket_timeout=5, diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index f9a5ee1aa8..4903a16068 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -18,21 +18,15 @@ def redis_key_for(cassette_path: str) -> str: return f"{REDIS_KEY_PREFIX}{rel}" +CASSETTE_REDIS_URL_ENV = "CASSETTE_REDIS_URL" + + def _redis_url_from_env() -> Optional[str]: - for var in ("REDIS_URL", "REDIS_SSL_URL"): - url = os.environ.get(var) - if url: - return url - host = os.environ.get("REDIS_HOST") - if not host: - return None - scheme = "rediss" if os.environ.get("REDIS_SSL", "").lower() == "true" else "redis" - auth = "" - if os.environ.get("REDIS_PASSWORD"): - user = os.environ.get("REDIS_USERNAME", "") - auth = f"{user}:{os.environ['REDIS_PASSWORD']}@" - port = os.environ.get("REDIS_PORT", "6379") - return f"{scheme}://{auth}{host}:{port}" + # Use a dedicated cassette Redis URL so the VCR cache is isolated from any + # application Redis used by tests (which may be flushed by other suites). + # Intentionally do NOT fall back to REDIS_URL/REDIS_HOST — sharing a Redis + # with the app cache risks cassettes being wiped by flushdb/flushall. + return os.environ.get(CASSETTE_REDIS_URL_ENV) or None def _build_default_client(): @@ -41,7 +35,9 @@ def _build_default_client(): url = _redis_url_from_env() if not url: raise RuntimeError( - "Set REDIS_URL, REDIS_SSL_URL, or REDIS_HOST to enable the VCR persister" + f"Set {CASSETTE_REDIS_URL_ENV} to enable the VCR persister. " + "Cassette Redis is intentionally separate from the application " + "Redis (REDIS_URL/REDIS_HOST) to avoid being flushed by tests." ) return redis.Redis.from_url( url, diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index b58508d5ae..cbf59182d7 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -89,9 +89,10 @@ def vcr_config(): def _vcr_disabled() -> bool: if os.environ.get("LITELLM_VCR_DISABLE") == "1": return True - return not any( - os.environ.get(var) for var in ("REDIS_URL", "REDIS_SSL_URL", "REDIS_HOST") - ) + # Cassettes live on a dedicated Redis (CASSETTE_REDIS_URL) so the cache + # isn't shared with — and accidentally flushed by — tests that exercise + # the application Redis via REDIS_URL/REDIS_HOST. + return not os.environ.get("CASSETTE_REDIS_URL") def pytest_recording_configure(config, vcr): diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 6969b1167d..0691113a61 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -113,9 +113,10 @@ def vcr_config(): def _vcr_disabled() -> bool: if os.environ.get("LITELLM_VCR_DISABLE") == "1": return True - return not any( - os.environ.get(var) for var in ("REDIS_URL", "REDIS_SSL_URL", "REDIS_HOST") - ) + # Cassettes live on a dedicated Redis (CASSETTE_REDIS_URL) so the cache + # isn't shared with — and accidentally flushed by — tests that exercise + # the application Redis via REDIS_URL/REDIS_HOST. + return not os.environ.get("CASSETTE_REDIS_URL") def pytest_recording_configure(config, vcr): From a7e8189b173d6d2ba16e61e448172796943307b1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 1 May 2026 12:53:36 -0700 Subject: [PATCH 21/30] tests(vcr): make redis persister resilient to transient outages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Managed Redis (e.g. Upstash) drops idle TLS connections, which surfaced in CI as a teardown ERROR on test_gemini_image_size_limit_exceeded: redis.exceptions.ConnectionError: EOF occurred in violation of protocol (_ssl.c:2427) Cassette persistence is a cache, not test correctness, so: - Configure the redis client with Retry(ExponentialBackoff, retries=2) on ConnectionError/TimeoutError to absorb single-socket drops. - Wrap save_cassette so a final failure logs a warning instead of failing teardown — the next run re-records. - Wrap load_cassette so an outage on read becomes a cache miss (CassetteNotFoundError) instead of erroring in setup. --- tests/_vcr_redis_persister.py | 45 ++++++++++++++++++- .../test_vcr_redis_persister.py | 43 ++++++++++++++++++ 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index 4903a16068..537f314bfb 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import os from typing import Any, Optional @@ -9,6 +10,8 @@ from vcr.serialize import deserialize, serialize CASSETTE_TTL_SECONDS = 24 * 60 * 60 REDIS_KEY_PREFIX = "litellm:vcr:cassette:" +_log = logging.getLogger(__name__) + def redis_key_for(cassette_path: str) -> str: rel = os.path.relpath(str(cassette_path)) @@ -31,6 +34,10 @@ def _redis_url_from_env() -> Optional[str]: def _build_default_client(): import redis + from redis.backoff import ExponentialBackoff + from redis.exceptions import ConnectionError as RedisConnectionError + from redis.exceptions import TimeoutError as RedisTimeoutError + from redis.retry import Retry url = _redis_url_from_env() if not url: @@ -39,11 +46,15 @@ def _build_default_client(): "Cassette Redis is intentionally separate from the application " "Redis (REDIS_URL/REDIS_HOST) to avoid being flushed by tests." ) + # Managed Redis providers (e.g. Upstash) drop idle TLS connections; retry on + # connection/timeout errors so a single dropped socket doesn't fail teardown. return redis.Redis.from_url( url, socket_timeout=5, socket_connect_timeout=5, decode_responses=False, + retry=Retry(ExponentialBackoff(cap=2, base=0.1), retries=2), + retry_on_error=[RedisConnectionError, RedisTimeoutError], ) @@ -53,10 +64,30 @@ def make_redis_persister( ): redis_client = client if client is not None else _build_default_client() + # Lazily resolve the redis exception classes so callers can pass any + # client (incl. fakeredis) without importing the real `redis` package. + try: + from redis.exceptions import ConnectionError as RedisConnectionError + from redis.exceptions import TimeoutError as RedisTimeoutError + + _transient_errors: tuple = (RedisConnectionError, RedisTimeoutError) + except ImportError: # pragma: no cover - redis is a hard test dep + _transient_errors = () + class _RedisPersister: @staticmethod def load_cassette(cassette_path, serializer): - data = redis_client.get(redis_key_for(cassette_path)) + try: + data = redis_client.get(redis_key_for(cassette_path)) + except _transient_errors as exc: + # Treat a Redis outage on read as a cassette miss so tests fall + # through to a live call instead of erroring in setup. + _log.warning( + "VCR redis load failed for %s; treating as cache miss: %s", + cassette_path, + exc, + ) + raise CassetteNotFoundError() from exc if data is None: raise CassetteNotFoundError() if isinstance(data, bytes): @@ -67,7 +98,17 @@ def make_redis_persister( def save_cassette(cassette_path, cassette_dict, serializer): data = serialize(cassette_dict, serializer) payload = data.encode("utf-8") if isinstance(data, str) else data - redis_client.set(redis_key_for(cassette_path), payload, ex=ttl_seconds) + try: + redis_client.set(redis_key_for(cassette_path), payload, ex=ttl_seconds) + except _transient_errors as exc: + # Cassette persistence is a cache, not test correctness. A Redis + # outage on save should not fail an otherwise-passing test — + # the next run will simply re-record. + _log.warning( + "VCR redis save failed for %s; cassette not persisted: %s", + cassette_path, + exc, + ) return _RedisPersister diff --git a/tests/llm_translation/test_vcr_redis_persister.py b/tests/llm_translation/test_vcr_redis_persister.py index d10aa388c4..b4dcd4ded8 100644 --- a/tests/llm_translation/test_vcr_redis_persister.py +++ b/tests/llm_translation/test_vcr_redis_persister.py @@ -5,6 +5,7 @@ import sys import fakeredis import pytest +from redis.exceptions import ConnectionError as RedisConnectionError from vcr.persisters.filesystem import CassetteNotFoundError from vcr.request import Request from vcr.serializers import yamlserializer @@ -82,6 +83,48 @@ def test_redis_key_normalizes_path_passed_by_pytest_recording(): ) +class _FlakyRedis: + """Wraps a fake redis but raises ConnectionError on the chosen op.""" + + def __init__(self, inner, fail_on: str): + self._inner = inner + self._fail_on = fail_on + + def get(self, *args, **kwargs): + if self._fail_on == "get": + raise RedisConnectionError("simulated outage") + return self._inner.get(*args, **kwargs) + + def set(self, *args, **kwargs): + if self._fail_on == "set": + raise RedisConnectionError("simulated outage") + return self._inner.set(*args, **kwargs) + + +def test_save_swallows_connection_errors_so_teardown_does_not_fail(): + # Persistence is a cache; an outage shouldn't fail an otherwise-passing test. + flaky = _FlakyRedis(fakeredis.FakeStrictRedis(), fail_on="set") + persister = make_redis_persister(client=flaky) + + persister.save_cassette( + "tests/llm_translation/test_x/test_save_outage", + _sample_cassette_dict(), + yamlserializer, + ) + + +def test_load_treats_connection_errors_as_cassette_miss(): + # An outage on read should fall through to a live call (CassetteNotFound), + # not surface a redis exception in the test setup. + flaky = _FlakyRedis(fakeredis.FakeStrictRedis(), fail_on="get") + persister = make_redis_persister(client=flaky) + + with pytest.raises(CassetteNotFoundError): + persister.load_cassette( + "tests/llm_translation/test_x/test_load_outage", yamlserializer + ) + + @pytest.mark.parametrize( ("status_code", "expect_dropped"), [ From cf4c9ede6102fdfe34e8a8894b0c6701359b5328 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 1 May 2026 13:00:20 -0700 Subject: [PATCH 22/30] tests(vcr): add LITELLM_VCR_VERBOSE per-test hit/miss reporting Set LITELLM_VCR_VERBOSE=1 to print a one-line cassette verdict per test (HIT / MISS / PARTIAL / NOOP) showing replay vs new-recording counts. Useful for local QA to confirm which tests actually exercised the cache and which fell through to the live provider. --- tests/_vcr_redis_persister.py | 39 +++++++++++++++++++++ tests/llm_responses_api_testing/conftest.py | 20 +++++++++++ tests/llm_translation/conftest.py | 20 +++++++++++ 3 files changed, 79 insertions(+) diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index 537f314bfb..fb1183f557 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -146,3 +146,42 @@ def patch_vcrpy_aiohttp_record_path() -> None: _aiohttp_stubs.record_response = _record_response_preserving_body _PATCHED_AIOHTTP_RECORD = True + + +VCR_VERBOSE_ENV = "LITELLM_VCR_VERBOSE" + + +def vcr_verbose_enabled() -> bool: + return os.environ.get(VCR_VERBOSE_ENV) == "1" + + +def format_vcr_verdict(cassette: Any) -> str: + """Build a one-line hit/miss verdict for a vcrpy Cassette. + + HIT — at least one request was served from cache and nothing new was + recorded. (Pure replay.) + MISS — nothing from cache; one or more requests went live and were + recorded. (Cold cache.) + PARTIAL — mix of replay and new recordings. Usually means the cassette + matches some but not all requests for this test (e.g. retries, + new branches, or vcrpy match_on too strict). + NOOP — test made no HTTP calls (or VCR not engaged for it). + """ + if cassette is None: + return "[VCR NOOP]" + played = getattr(cassette, "play_count", 0) or 0 + # cassette.data is the recorded request/response list; len(cassette) counts + # recorded episodes. New recordings during this test = len - prior_len, but + # we don't have prior_len here, so we use cassette.dirty (set when an append + # happened during this run) as the "new recording" signal. + dirty = getattr(cassette, "dirty", False) + total = len(cassette) if hasattr(cassette, "__len__") else 0 + if played == 0 and not dirty: + return "[VCR NOOP] (no http traffic)" + if played > 0 and not dirty: + return f"[VCR HIT] {played} replayed, 0 new ({total} cassette entries)" + if played == 0 and dirty: + return f"[VCR MISS] 0 replayed, recorded new ({total} cassette entries)" + return ( + f"[VCR PARTIAL] {played} replayed + new recordings ({total} cassette entries)" + ) diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index cbf59182d7..05e0597d60 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -15,8 +15,10 @@ import litellm # noqa: E402 from tests._vcr_redis_persister import ( # noqa: E402 filter_non_2xx_response, + format_vcr_verdict, make_redis_persister, patch_vcrpy_aiohttp_record_path, + vcr_verbose_enabled, ) @@ -102,6 +104,24 @@ def pytest_recording_configure(config, vcr): patch_vcrpy_aiohttp_record_path() +@pytest.fixture(autouse=True) +def _vcr_hit_miss_report(request, vcr): + """When LITELLM_VCR_VERBOSE=1, print a one-line cassette verdict per test. + + Runs after the `vcr` fixture (which yields the active Cassette), so we can + inspect play_count / dirty / len in teardown.""" + yield + if not vcr_verbose_enabled(): + return + verdict = format_vcr_verdict(vcr) + reporter = request.config.pluginmanager.get_plugin("terminalreporter") + line = f"{verdict} :: {request.node.nodeid}" + if reporter is not None: + reporter.write_line(line) + else: # pragma: no cover - reporter is always present in normal runs + print(line) + + @pytest.fixture(scope="session") def event_loop(): try: diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 0691113a61..24742438f1 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -20,8 +20,10 @@ import litellm # noqa: E402 from tests._vcr_redis_persister import ( # noqa: E402 filter_non_2xx_response, + format_vcr_verdict, make_redis_persister, patch_vcrpy_aiohttp_record_path, + vcr_verbose_enabled, ) @@ -126,6 +128,24 @@ def pytest_recording_configure(config, vcr): patch_vcrpy_aiohttp_record_path() +@pytest.fixture(autouse=True) +def _vcr_hit_miss_report(request, vcr): + """When LITELLM_VCR_VERBOSE=1, print a one-line cassette verdict per test. + + Runs after the `vcr` fixture (which yields the active Cassette), so we can + inspect play_count / dirty / len in teardown.""" + yield + if not vcr_verbose_enabled(): + return + verdict = format_vcr_verdict(vcr) + reporter = request.config.pluginmanager.get_plugin("terminalreporter") + line = f"{verdict} :: {request.node.nodeid}" + if reporter is not None: + reporter.write_line(line) + else: # pragma: no cover - reporter is always present in normal runs + print(line) + + # --------------------------------------------------------------------------- # Capture TRUE defaults at conftest import time (before test modules pollute). # --------------------------------------------------------------------------- From ff63bdb9840aa48af5062f039ab3aa095b811857 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 1 May 2026 13:35:29 -0700 Subject: [PATCH 23/30] tests(vcr): only persist cassette on test pass to avoid poisoning cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A test that fails (incl. all the failing retries before a passing one) can otherwise overwrite a known-good cassette with a 'bad luck' recording. Tests like test_prompt_caching, which assert on provider state across two calls, can produce a 200 response that semantically fails the assertion — the 2xx filter doesn't catch this because the HTTP layer is fine. - pytest_runtest_makereport hook attaches each phase report to the pytest item. - _vcr_outcome_gate fixture (combining the verbose-mode reporter) reads the call-phase outcome at teardown and informs the persister via mark_test_outcome_for_cassette before vcrpy's Cassette.__exit__ triggers save_cassette. - save_cassette consults the per-key 'did the test pass?' flag and short-circuits when False, leaving any prior good recording intact. - Defaults to passed=True when no marker is present so non-test usage of the persister still works. --- tests/_vcr_redis_persister.py | 37 +++++++++++++- tests/llm_responses_api_testing/conftest.py | 38 +++++++++++--- tests/llm_translation/conftest.py | 38 +++++++++++--- .../test_vcr_redis_persister.py | 50 +++++++++++++++++++ 4 files changed, 150 insertions(+), 13 deletions(-) diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index fb1183f557..08e7d09c6a 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -13,6 +13,25 @@ REDIS_KEY_PREFIX = "litellm:vcr:cassette:" _log = logging.getLogger(__name__) +# Per-process map: cassette key -> "did the test that produced this cassette +# pass?". The conftest's pytest_runtest_makereport hook sets True when the test +# body succeeds; save_cassette consults it to avoid persisting recordings from +# failed runs. We key by the redis cache key so retries (which may produce a +# fresh cassette object each time but write to the same key) interleave +# correctly. +_passed_by_cassette_key: dict[str, bool] = {} + + +def mark_test_outcome_for_cassette(cassette_path: str, passed: bool) -> None: + """Record whether the test that owns ``cassette_path`` passed. + + Called from a pytest hook in conftest. The recorded value is consulted by + ``save_cassette`` so failed-attempt recordings (e.g. a flaky test that + asserts on provider state) don't poison the cache for future runs. + """ + _passed_by_cassette_key[redis_key_for(cassette_path)] = passed + + def redis_key_for(cassette_path: str) -> str: rel = os.path.relpath(str(cassette_path)) if rel.endswith(".yaml"): @@ -96,10 +115,26 @@ def make_redis_persister( @staticmethod def save_cassette(cassette_path, cassette_dict, serializer): + key = redis_key_for(cassette_path) + # Only persist successful runs. A failed test (incl. all the failed + # retries before a passing one) would otherwise poison the cache — + # e.g. a flaky test that observes provider state across two calls + # could capture a "bad luck" response that deterministically fails + # every future replay. We default to True if the hook didn't run + # (e.g. cassette saved outside a test context) so non-test usage + # still works. + passed = _passed_by_cassette_key.pop(key, True) + if not passed: + _log.info( + "VCR redis save skipped for %s; test did not pass — " + "leaving any prior cassette intact", + cassette_path, + ) + return data = serialize(cassette_dict, serializer) payload = data.encode("utf-8") if isinstance(data, str) else data try: - redis_client.set(redis_key_for(cassette_path), payload, ex=ttl_seconds) + redis_client.set(key, payload, ex=ttl_seconds) except _transient_errors as exc: # Cassette persistence is a cache, not test correctness. A Redis # outage on save should not fail an otherwise-passing test — diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index 05e0597d60..bf74f2c6a8 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -17,6 +17,7 @@ from tests._vcr_redis_persister import ( # noqa: E402 filter_non_2xx_response, format_vcr_verdict, make_redis_persister, + mark_test_outcome_for_cassette, patch_vcrpy_aiohttp_record_path, vcr_verbose_enabled, ) @@ -104,16 +105,41 @@ def pytest_recording_configure(config, vcr): patch_vcrpy_aiohttp_record_path() -@pytest.fixture(autouse=True) -def _vcr_hit_miss_report(request, vcr): - """When LITELLM_VCR_VERBOSE=1, print a one-line cassette verdict per test. +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + """Attach each phase's report to the item so fixture teardown can read it. - Runs after the `vcr` fixture (which yields the active Cassette), so we can - inspect play_count / dirty / len in teardown.""" + Used by ``_vcr_outcome_gate`` below to skip persisting cassettes for + failed test runs (incl. failed retries that pytest-rerunfailures will + re-attempt) so a "bad luck" recording can't poison future replays. + """ + outcome = yield + rep = outcome.get_result() + setattr(item, f"rep_{rep.when}", rep) + + +@pytest.fixture(autouse=True) +def _vcr_outcome_gate(request, vcr): + """Tell the persister whether the test that owns this cassette passed. + + Runs after ``vcr`` (which yields the active Cassette). At teardown time + the call-phase report is attached to the item by the makereport hook + above, so we can mark the cassette key passed/failed before vcrpy's + Cassette.__exit__ triggers persister.save_cassette. + + Also prints a per-test hit/miss verdict when LITELLM_VCR_VERBOSE=1. + """ yield + cassette = vcr # name kept for the verbose-output line + rep_call = getattr(request.node, "rep_call", None) + test_passed = bool(rep_call and rep_call.passed) + cassette_path = getattr(cassette, "_path", None) if cassette is not None else None + if cassette_path: + mark_test_outcome_for_cassette(cassette_path, test_passed) + if not vcr_verbose_enabled(): return - verdict = format_vcr_verdict(vcr) + verdict = format_vcr_verdict(cassette) reporter = request.config.pluginmanager.get_plugin("terminalreporter") line = f"{verdict} :: {request.node.nodeid}" if reporter is not None: diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 24742438f1..79e60117c9 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -22,6 +22,7 @@ from tests._vcr_redis_persister import ( # noqa: E402 filter_non_2xx_response, format_vcr_verdict, make_redis_persister, + mark_test_outcome_for_cassette, patch_vcrpy_aiohttp_record_path, vcr_verbose_enabled, ) @@ -128,16 +129,41 @@ def pytest_recording_configure(config, vcr): patch_vcrpy_aiohttp_record_path() -@pytest.fixture(autouse=True) -def _vcr_hit_miss_report(request, vcr): - """When LITELLM_VCR_VERBOSE=1, print a one-line cassette verdict per test. +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + """Attach each phase's report to the item so fixture teardown can read it. - Runs after the `vcr` fixture (which yields the active Cassette), so we can - inspect play_count / dirty / len in teardown.""" + Used by ``_vcr_outcome_gate`` below to skip persisting cassettes for + failed test runs (incl. failed retries that pytest-rerunfailures will + re-attempt) so a "bad luck" recording can't poison future replays. + """ + outcome = yield + rep = outcome.get_result() + setattr(item, f"rep_{rep.when}", rep) + + +@pytest.fixture(autouse=True) +def _vcr_outcome_gate(request, vcr): + """Tell the persister whether the test that owns this cassette passed. + + Runs after ``vcr`` (which yields the active Cassette). At teardown time + the call-phase report is attached to the item by the makereport hook + above, so we can mark the cassette key passed/failed before vcrpy's + Cassette.__exit__ triggers persister.save_cassette. + + Also prints a per-test hit/miss verdict when LITELLM_VCR_VERBOSE=1. + """ yield + cassette = vcr # name kept for the verbose-output line + rep_call = getattr(request.node, "rep_call", None) + test_passed = bool(rep_call and rep_call.passed) + cassette_path = getattr(cassette, "_path", None) if cassette is not None else None + if cassette_path: + mark_test_outcome_for_cassette(cassette_path, test_passed) + if not vcr_verbose_enabled(): return - verdict = format_vcr_verdict(vcr) + verdict = format_vcr_verdict(cassette) reporter = request.config.pluginmanager.get_plugin("terminalreporter") line = f"{verdict} :: {request.node.nodeid}" if reporter is not None: diff --git a/tests/llm_translation/test_vcr_redis_persister.py b/tests/llm_translation/test_vcr_redis_persister.py index b4dcd4ded8..7d345cbcfe 100644 --- a/tests/llm_translation/test_vcr_redis_persister.py +++ b/tests/llm_translation/test_vcr_redis_persister.py @@ -16,6 +16,7 @@ from tests._vcr_redis_persister import ( # noqa: E402 CASSETTE_TTL_SECONDS, filter_non_2xx_response, make_redis_persister, + mark_test_outcome_for_cassette, redis_key_for, ) @@ -113,6 +114,55 @@ def test_save_swallows_connection_errors_so_teardown_does_not_fail(): ) +def test_save_skipped_when_test_marked_failed_and_prior_cassette_preserved(): + # A flaky test that fails should NOT overwrite a previously-good cassette. + fake, persister = _persister_with_fake_redis() + cassette_id = "tests/llm_translation/test_x/test_flaky" + key = redis_key_for(cassette_id) + + # Seed a "known-good" recording from a prior successful run. + good = _sample_cassette_dict() + persister.save_cassette(cassette_id, good, yamlserializer) + good_payload = fake.get(key) + assert good_payload is not None + + # Simulate a failed run: the hook records "did not pass" before save. + mark_test_outcome_for_cassette(cassette_id, passed=False) + bad_response = { + "status": {"code": 200, "message": "OK"}, + "headers": {}, + "body": {"string": b'{"id":"BAD","type":"message"}'}, + } + bad = {"requests": good["requests"], "responses": [bad_response]} + persister.save_cassette(cassette_id, bad, yamlserializer) + + # Prior good payload is still there — the bad save was suppressed. + assert fake.get(key) == good_payload + + +def test_save_proceeds_when_test_marked_passed(): + fake, persister = _persister_with_fake_redis() + cassette_id = "tests/llm_translation/test_x/test_passed" + key = redis_key_for(cassette_id) + + mark_test_outcome_for_cassette(cassette_id, passed=True) + persister.save_cassette(cassette_id, _sample_cassette_dict(), yamlserializer) + + assert fake.get(key) is not None + + +def test_save_proceeds_when_outcome_unknown(): + # Used outside a pytest run (e.g. ad-hoc scripts), the outcome gate is + # bypassed so the persister still works. + fake, persister = _persister_with_fake_redis() + cassette_id = "tests/llm_translation/test_x/test_no_marker" + key = redis_key_for(cassette_id) + + persister.save_cassette(cassette_id, _sample_cassette_dict(), yamlserializer) + + assert fake.get(key) is not None + + def test_load_treats_connection_errors_as_cassette_miss(): # An outage on read should fall through to a live call (CassetteNotFound), # not surface a redis exception in the test setup. From 8c01b027797ab1e0e40dcb9492ebb8a2270493d8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 1 May 2026 13:50:34 -0700 Subject: [PATCH 24/30] tests(vcr): opt out tests that observe live cross-call provider state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some tests can't benefit from cassette replay because they assert on state that only exists in the live provider between two calls (e.g. prompt-cache propagation, intermittent provider quirks). Marking them with @pytest.mark.vcr just wastes cycles trying to record cassettes they will never replay against successfully. Opt-out by nodeid suffix so subclassed/parametrized variants are covered: - ::test_prompt_caching — Anthropic/Bedrock prompt-cache propagation isn't deterministic in the 0–1s window the test gives it. - ::test_async_pdf_handling_with_file_id — flaky upstream Wikipedia fetch through the Anthropic Files API. - TestBedrockInvokeNovaJson::test_json_response_pydantic_obj — Bedrock Nova returns tool_call vs JSON nondeterministically (other providers' subclasses are healthy). - ::test_bedrock_converse__streaming_passthrough — Bedrock streaming response_cost calc returns None intermittently. These tests keep their existing @pytest.mark.flaky retry behavior. --- tests/llm_translation/conftest.py | 34 +++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 79e60117c9..540518124d 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -47,6 +47,38 @@ _VCR_AUTO_MARKER_SKIP_FILES = _RESPX_CONFLICTING_FILES | frozenset( {"test_vcr_redis_persister.py"} ) +# Tests that observe live cross-call provider state (e.g. prompt-cache +# warm-up between two consecutive calls) cannot benefit from cassette +# replay: the second call's "expected" state depends on what the *live* +# provider does between the two calls, not on what was recorded earlier. +# Auto-marking them with @pytest.mark.vcr just wastes cycles and (before +# the outcome gate) used to poison the cache. They go live with their +# existing @pytest.mark.flaky retry logic. +# +# Match by suffix on the pytest nodeid so subclassed/parametrized variants +# are covered: e.g. "::test_prompt_caching" matches all subclasses that +# inherit the base test. +_VCR_INCOMPATIBLE_NODEID_SUFFIXES = frozenset( + { + # Provider prompt-cache propagation isn't deterministic between two + # back-to-back calls; the test is flaky against the live provider. + "::test_prompt_caching", + # Wikipedia URL fetch through Anthropic Files API is flaky upstream. + "::test_async_pdf_handling_with_file_id", + # Bedrock Nova returns tool_call vs JSON nondeterministically; the + # base assertion expects JSON. Other providers' versions of this + # test are healthy, so we narrow with a class-name guard below. + "TestBedrockInvokeNovaJson::test_json_response_pydantic_obj", + # Bedrock streaming response_cost calc returns None intermittently. + "::test_bedrock_converse__streaming_passthrough", + } +) + + +def _is_vcr_incompatible(nodeid: str) -> bool: + return any(nodeid.endswith(suffix) for suffix in _VCR_INCOMPATIBLE_NODEID_SUFFIXES) + + _FILTERED_REQUEST_HEADERS = ( "authorization", "x-api-key", @@ -255,6 +287,8 @@ def pytest_collection_modifyitems(config, items): filename = os.path.basename(str(item.fspath)) if filename in _VCR_AUTO_MARKER_SKIP_FILES: continue + if _is_vcr_incompatible(item.nodeid): + continue if item.get_closest_marker("vcr") is not None: continue item.add_marker(pytest.mark.vcr) From a47c4e7d1b6b349c5225b71434d24147d5afed5c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 1 May 2026 13:56:20 -0700 Subject: [PATCH 25/30] tests(vcr): refuse to persist cassettes past 50 episodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A test that produces non-deterministic request bodies (e.g. uuid in the prompt) under record_mode=new_episodes never replays — every CI run appends fresh unmatched episodes. The cassette grows unbounded over time and silently inflates Redis (we observed one cassette at 22 episodes / ~860KB after ~5 CI runs). Refuse the save when episode count exceeds MAX_EPISODES_PER_CASSETTE so the pathology surfaces with a loud warning that points to the opt-out fix instead of festering invisibly. --- tests/_vcr_redis_persister.py | 26 +++++++++ .../test_vcr_redis_persister.py | 57 +++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index 08e7d09c6a..3483f9d253 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -10,6 +10,15 @@ from vcr.serialize import deserialize, serialize CASSETTE_TTL_SECONDS = 24 * 60 * 60 REDIS_KEY_PREFIX = "litellm:vcr:cassette:" +# Healthy cassettes hold 1–5 episodes (a single test rarely makes more than a +# handful of distinct HTTP calls). When a cassette balloons past this, it +# usually means a test produces non-deterministic request bodies (e.g. uuid) +# under record_mode=new_episodes, and every CI run is appending fresh +# unmatched episodes instead of replaying. That growth is unbounded over +# time and silently inflates Redis. Refuse to persist past this threshold so +# the pathology surfaces loudly instead. +MAX_EPISODES_PER_CASSETTE = 50 + _log = logging.getLogger(__name__) @@ -124,6 +133,23 @@ def make_redis_persister( # (e.g. cassette saved outside a test context) so non-test usage # still works. passed = _passed_by_cassette_key.pop(key, True) + episode_count = len(cassette_dict.get("requests", []) or []) + if episode_count > MAX_EPISODES_PER_CASSETTE: + # Pathology: the test is producing non-deterministic request + # bodies and accumulating unbounded episodes. Refuse the save + # so the cassette can't keep ballooning, and surface a loud + # warning so someone investigates / opts the test out. + _log.warning( + "VCR redis save refused for %s; cassette has %d episodes " + "(> MAX_EPISODES_PER_CASSETTE=%d). The test likely produces " + "non-deterministic request bodies (e.g. uuid) and is " + "appending instead of replaying. Opt it out with the " + "no-vcr list in conftest, or stabilize its request body.", + cassette_path, + episode_count, + MAX_EPISODES_PER_CASSETTE, + ) + return if not passed: _log.info( "VCR redis save skipped for %s; test did not pass — " diff --git a/tests/llm_translation/test_vcr_redis_persister.py b/tests/llm_translation/test_vcr_redis_persister.py index 7d345cbcfe..1e4b6c83aa 100644 --- a/tests/llm_translation/test_vcr_redis_persister.py +++ b/tests/llm_translation/test_vcr_redis_persister.py @@ -14,6 +14,7 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", from tests._vcr_redis_persister import ( # noqa: E402 CASSETTE_TTL_SECONDS, + MAX_EPISODES_PER_CASSETTE, filter_non_2xx_response, make_redis_persister, mark_test_outcome_for_cassette, @@ -151,6 +152,62 @@ def test_save_proceeds_when_test_marked_passed(): assert fake.get(key) is not None +def test_save_refused_when_cassette_exceeds_max_episodes(): + # Pathological cassettes (non-deterministic body → unbounded episode growth) + # should be refused. Any prior good payload stays intact. + fake, persister = _persister_with_fake_redis() + cassette_id = "tests/llm_translation/test_x/test_runaway" + key = redis_key_for(cassette_id) + + persister.save_cassette(cassette_id, _sample_cassette_dict(), yamlserializer) + seed_payload = fake.get(key) + + request = Request( + method="POST", + uri="https://api.anthropic.com/v1/messages", + body=b"x", + headers={"content-type": "application/json"}, + ) + response = { + "status": {"code": 200, "message": "OK"}, + "headers": {}, + "body": {"string": b"{}"}, + } + bloated = { + "requests": [request] * (MAX_EPISODES_PER_CASSETTE + 1), + "responses": [response] * (MAX_EPISODES_PER_CASSETTE + 1), + } + persister.save_cassette(cassette_id, bloated, yamlserializer) + + # Refused — the seed payload is unchanged. + assert fake.get(key) == seed_payload + + +def test_save_proceeds_at_max_episodes_threshold(): + fake, persister = _persister_with_fake_redis() + cassette_id = "tests/llm_translation/test_x/test_at_threshold" + key = redis_key_for(cassette_id) + + request = Request( + method="POST", + uri="https://api.anthropic.com/v1/messages", + body=b"x", + headers={"content-type": "application/json"}, + ) + response = { + "status": {"code": 200, "message": "OK"}, + "headers": {}, + "body": {"string": b"{}"}, + } + at_threshold = { + "requests": [request] * MAX_EPISODES_PER_CASSETTE, + "responses": [response] * MAX_EPISODES_PER_CASSETTE, + } + persister.save_cassette(cassette_id, at_threshold, yamlserializer) + + assert fake.get(key) is not None + + def test_save_proceeds_when_outcome_unknown(): # Used outside a pytest run (e.g. ad-hoc scripts), the outcome gate is # bypassed so the persister still works. From 225d01cb4f6fb5471d637e60cc606027c02858a8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 1 May 2026 14:07:32 -0700 Subject: [PATCH 26/30] fix(tests): use github-hosted PDF fixture for Anthropic Files API test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Anthropic's URL fetcher intermittently returns 400 'Unable to download the file' for the Wikipedia URL the test was using. Point it at the repo's existing tests/llm_translation/fixtures/dummy.pdf via raw GitHub instead — small, deterministic, reliably fetchable. With a stable URL the test no longer needs to be opted out of VCR; remove it from the incompatible list so it can replay from cassette. --- tests/llm_translation/base_llm_unit_tests.py | 5 ++++- tests/llm_translation/conftest.py | 2 -- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index f3b1895323..61ae594532 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -401,7 +401,10 @@ class BaseLLMChatTest(ABC): { "type": "file", "file": { - "file_id": "https://upload.wikimedia.org/wikipedia/commons/2/20/Re_example.pdf" + # GitHub-hosted fixture (small, deterministic, reliably + # fetchable from Anthropic's egress) instead of a Wikipedia + # URL that intermittently returns 400 "Unable to download". + "file_id": "https://raw.githubusercontent.com/BerriAI/litellm/main/tests/llm_translation/fixtures/dummy.pdf" }, }, ] diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 540518124d..afd48bb674 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -63,8 +63,6 @@ _VCR_INCOMPATIBLE_NODEID_SUFFIXES = frozenset( # Provider prompt-cache propagation isn't deterministic between two # back-to-back calls; the test is flaky against the live provider. "::test_prompt_caching", - # Wikipedia URL fetch through Anthropic Files API is flaky upstream. - "::test_async_pdf_handling_with_file_id", # Bedrock Nova returns tool_call vs JSON nondeterministically; the # base assertion expects JSON. Other providers' versions of this # test are healthy, so we narrow with a class-name guard below. From c05c865a1c427ab04aa7021739d1e69838b214f9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 1 May 2026 14:14:44 -0700 Subject: [PATCH 27/30] tests(vcr): emit verbose verdicts to un-redirected stderr Previously, the per-test [VCR HIT/MISS/...] line was written via TerminalReporter.write_line from inside fixture teardown. Pytest captures that stream by default and only surfaces it on FAILED tests (under 'Captured stdout teardown'), so passing tests' verdicts were invisible in CI logs and the user couldn't tell whether the cache was working. Write directly to sys.__stderr__ so the line bypasses pytest's capture entirely. Under xdist each worker has its own __stderr__ which CircleCI aggregates into the live job log alongside the PASSED/FAILED markers. --- tests/_vcr_redis_persister.py | 24 +++++++++++++++++++++ tests/llm_responses_api_testing/conftest.py | 8 ++----- tests/llm_translation/conftest.py | 8 ++----- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index 3483f9d253..03cec23f4d 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -216,6 +216,30 @@ def vcr_verbose_enabled() -> bool: return os.environ.get(VCR_VERBOSE_ENV) == "1" +def emit_vcr_verbose_line(line: str) -> None: + """Write a one-line VCR verdict to the un-redirected stderr. + + Pytest's stdout/stderr capture would otherwise hide this output (and + only surface it as 'Captured stdout teardown' on failing tests). Under + xdist each worker has its own stderr; CircleCI aggregates them. Writing + to ``sys.__stderr__`` bypasses pytest's capture entirely so the line + reaches the live CI log alongside the per-test PASSED/FAILED markers. + """ + import sys + + try: + stream = sys.__stderr__ + if stream is not None: + stream.write(line + "\n") + stream.flush() + return + except Exception: # pragma: no cover - last-ditch fallback + pass + # If __stderr__ isn't writable for some reason, fall back to the + # captured stderr — better than swallowing. + print(line, file=sys.stderr) + + def format_vcr_verdict(cassette: Any) -> str: """Build a one-line hit/miss verdict for a vcrpy Cassette. diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index bf74f2c6a8..3d405decaa 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -14,6 +14,7 @@ sys.path.insert( import litellm # noqa: E402 from tests._vcr_redis_persister import ( # noqa: E402 + emit_vcr_verbose_line, filter_non_2xx_response, format_vcr_verdict, make_redis_persister, @@ -140,12 +141,7 @@ def _vcr_outcome_gate(request, vcr): if not vcr_verbose_enabled(): return verdict = format_vcr_verdict(cassette) - reporter = request.config.pluginmanager.get_plugin("terminalreporter") - line = f"{verdict} :: {request.node.nodeid}" - if reporter is not None: - reporter.write_line(line) - else: # pragma: no cover - reporter is always present in normal runs - print(line) + emit_vcr_verbose_line(f"{verdict} :: {request.node.nodeid}") @pytest.fixture(scope="session") diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index afd48bb674..f17ab32a62 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -19,6 +19,7 @@ sys.path.insert( import litellm # noqa: E402 from tests._vcr_redis_persister import ( # noqa: E402 + emit_vcr_verbose_line, filter_non_2xx_response, format_vcr_verdict, make_redis_persister, @@ -194,12 +195,7 @@ def _vcr_outcome_gate(request, vcr): if not vcr_verbose_enabled(): return verdict = format_vcr_verdict(cassette) - reporter = request.config.pluginmanager.get_plugin("terminalreporter") - line = f"{verdict} :: {request.node.nodeid}" - if reporter is not None: - reporter.write_line(line) - else: # pragma: no cover - reporter is always present in normal runs - print(line) + emit_vcr_verbose_line(f"{verdict} :: {request.node.nodeid}") # --------------------------------------------------------------------------- From 965185c1065e9d51c8ce135dc86ad1719ad89b1e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 1 May 2026 14:16:57 -0700 Subject: [PATCH 28/30] fix(tests): host PDF fixture via jsDelivr with proper application/pdf MIME Raw github serves application/octet-stream which OpenAI/Gemini reject when LiteLLM fetches the URL client-side. jsDelivr serves the same file with content-type: application/pdf. Pin to a commit SHA so the asset is immutable and jsDelivr can cache it for a year. --- tests/llm_translation/base_llm_unit_tests.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index 61ae594532..5e7b2f190e 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -401,10 +401,17 @@ class BaseLLMChatTest(ABC): { "type": "file", "file": { - # GitHub-hosted fixture (small, deterministic, reliably - # fetchable from Anthropic's egress) instead of a Wikipedia - # URL that intermittently returns 400 "Unable to download". - "file_id": "https://raw.githubusercontent.com/BerriAI/litellm/main/tests/llm_translation/fixtures/dummy.pdf" + # jsDelivr serves the repo's tests/llm_translation/fixtures/dummy.pdf + # with `Content-Type: application/pdf`. Two reasons we don't + # use raw.githubusercontent.com or upload.wikimedia.org: + # - raw GitHub returns Content-Type: application/octet-stream, + # which OpenAI/Gemini reject when LiteLLM fetches the URL + # client-side and forwards the bytes. + # - Wikimedia URLs intermittently return 400 from Anthropic's + # server-side URL fetcher. + # The URL is pinned to a specific commit SHA so jsDelivr can + # serve it as immutable (cache-control: immutable, max-age=1y). + "file_id": "https://cdn.jsdelivr.net/gh/BerriAI/litellm@aab3ef8988b12d166b20356a81c53127480f1125/tests/llm_translation/fixtures/dummy.pdf" }, }, ] From 53f71fbf4d721903ae8a7dd942a99e05f642efc0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 1 May 2026 14:29:06 -0700 Subject: [PATCH 29/30] tests(vcr): emit per-test verdicts via xdist controller's terminalreporter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previous attempt wrote to sys.__stderr__ from the test fixture. Under xdist, fixtures run inside worker subprocesses whose stderr is captured by the controller and only released to the live log on test failure — so passing tests' verdicts were silently swallowed. Round-trip via report.user_properties: the worker-side fixture stashes the verdict on user_properties, xdist serializes it onto the report, and a controller-side pytest_runtest_logreport hook writes it via the TerminalReporter (the same plugin that emits PASSED/FAILED markers). TerminalReporter is resolved lazily on first hook call because it's not yet registered when conftest's pytest_configure runs. Verified locally in both serial and xdist modes. --- tests/_vcr_redis_persister.py | 24 ------- tests/llm_responses_api_testing/conftest.py | 55 ++++++++++++++-- tests/llm_translation/conftest.py | 71 +++++++++++++++++++-- 3 files changed, 118 insertions(+), 32 deletions(-) diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index 03cec23f4d..3483f9d253 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -216,30 +216,6 @@ def vcr_verbose_enabled() -> bool: return os.environ.get(VCR_VERBOSE_ENV) == "1" -def emit_vcr_verbose_line(line: str) -> None: - """Write a one-line VCR verdict to the un-redirected stderr. - - Pytest's stdout/stderr capture would otherwise hide this output (and - only surface it as 'Captured stdout teardown' on failing tests). Under - xdist each worker has its own stderr; CircleCI aggregates them. Writing - to ``sys.__stderr__`` bypasses pytest's capture entirely so the line - reaches the live CI log alongside the per-test PASSED/FAILED markers. - """ - import sys - - try: - stream = sys.__stderr__ - if stream is not None: - stream.write(line + "\n") - stream.flush() - return - except Exception: # pragma: no cover - last-ditch fallback - pass - # If __stderr__ isn't writable for some reason, fall back to the - # captured stderr — better than swallowing. - print(line, file=sys.stderr) - - def format_vcr_verdict(cassette: Any) -> str: """Build a one-line hit/miss verdict for a vcrpy Cassette. diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index 3d405decaa..612e8691f9 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -14,7 +14,6 @@ sys.path.insert( import litellm # noqa: E402 from tests._vcr_redis_persister import ( # noqa: E402 - emit_vcr_verbose_line, filter_non_2xx_response, format_vcr_verdict, make_redis_persister, @@ -24,6 +23,12 @@ from tests._vcr_redis_persister import ( # noqa: E402 ) +# Controller-side handles for writing per-test VCR verdicts to the live +# terminal. See the matching comment in tests/llm_translation/conftest.py. +_controller_pluginmanager = None +_controller_terminal_reporter = None + + _FILTERED_REQUEST_HEADERS = ( "authorization", "x-api-key", @@ -128,10 +133,12 @@ def _vcr_outcome_gate(request, vcr): above, so we can mark the cassette key passed/failed before vcrpy's Cassette.__exit__ triggers persister.save_cassette. - Also prints a per-test hit/miss verdict when LITELLM_VCR_VERBOSE=1. + Stashes a per-test hit/miss verdict on ``user_properties`` so the + controller-side ``pytest_runtest_logreport`` hook can surface it to the + live terminal under xdist. """ yield - cassette = vcr # name kept for the verbose-output line + cassette = vcr rep_call = getattr(request.node, "rep_call", None) test_passed = bool(rep_call and rep_call.passed) cassette_path = getattr(cassette, "_path", None) if cassette is not None else None @@ -141,7 +148,47 @@ def _vcr_outcome_gate(request, vcr): if not vcr_verbose_enabled(): return verdict = format_vcr_verdict(cassette) - emit_vcr_verbose_line(f"{verdict} :: {request.node.nodeid}") + request.node.user_properties.append(("vcr_verdict", verdict)) + + +def pytest_configure(config): + """Stash the pluginmanager so the logreport hook can find TerminalReporter.""" + global _controller_pluginmanager + if os.environ.get("PYTEST_XDIST_WORKER"): + return + _controller_pluginmanager = config.pluginmanager + + +def _resolve_terminal_reporter(): + global _controller_terminal_reporter + if _controller_terminal_reporter is not None: + return _controller_terminal_reporter + if _controller_pluginmanager is None: + return None + _controller_terminal_reporter = _controller_pluginmanager.getplugin( + "terminalreporter" + ) + return _controller_terminal_reporter + + +def pytest_runtest_logreport(report): + """Emit per-test VCR verdicts on the controller's live terminal.""" + if report.when != "teardown": + return + if os.environ.get("PYTEST_XDIST_WORKER"): + return + if not vcr_verbose_enabled(): + return + reporter = _resolve_terminal_reporter() + if reporter is None: + return + verdict = next( + (v for k, v in (report.user_properties or []) if k == "vcr_verdict"), + None, + ) + if not verdict: + return + reporter.write_line(f"{verdict} :: {report.nodeid}") @pytest.fixture(scope="session") diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index f17ab32a62..1976dc476a 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -19,7 +19,6 @@ sys.path.insert( import litellm # noqa: E402 from tests._vcr_redis_persister import ( # noqa: E402 - emit_vcr_verbose_line, filter_non_2xx_response, format_vcr_verdict, make_redis_persister, @@ -29,6 +28,16 @@ from tests._vcr_redis_persister import ( # noqa: E402 ) +# Controller-side handles for writing per-test VCR verdicts to the live +# terminal. ``pytest_configure`` stashes the pluginmanager (workers don't get +# a TerminalReporter — their output is captured and aggregated by the +# controller), and ``pytest_runtest_logreport`` resolves the TerminalReporter +# lazily on first use because it isn't registered yet at conftest configure +# time. +_controller_pluginmanager = None +_controller_terminal_reporter = None + + # vcrpy and respx both patch the httpx transport — applying both makes one # silently win. Files in this set use respx and are skipped by the # auto-marker below. @@ -182,10 +191,14 @@ def _vcr_outcome_gate(request, vcr): above, so we can mark the cassette key passed/failed before vcrpy's Cassette.__exit__ triggers persister.save_cassette. - Also prints a per-test hit/miss verdict when LITELLM_VCR_VERBOSE=1. + Stashes a per-test hit/miss verdict on ``user_properties`` so the + controller-side ``pytest_runtest_logreport`` hook can surface it to the + live terminal. xdist serializes ``user_properties`` on each phase's + report back to the controller, which is the only process that has a + TerminalReporter wired to CI's live log. """ yield - cassette = vcr # name kept for the verbose-output line + cassette = vcr rep_call = getattr(request.node, "rep_call", None) test_passed = bool(rep_call and rep_call.passed) cassette_path = getattr(cassette, "_path", None) if cassette is not None else None @@ -195,7 +208,57 @@ def _vcr_outcome_gate(request, vcr): if not vcr_verbose_enabled(): return verdict = format_vcr_verdict(cassette) - emit_vcr_verbose_line(f"{verdict} :: {request.node.nodeid}") + request.node.user_properties.append(("vcr_verdict", verdict)) + + +def pytest_configure(config): + """Stash the pluginmanager so the logreport hook can find TerminalReporter. + + We can't grab TerminalReporter directly here — it's not registered until + pytest's own ``pytest_configure`` runs, and conftest hooks may run first. + Stashing the config is enough; the hook resolves on first use. + """ + global _controller_pluginmanager + if os.environ.get("PYTEST_XDIST_WORKER"): + return # workers don't have a live-log TerminalReporter + _controller_pluginmanager = config.pluginmanager + + +def _resolve_terminal_reporter(): + """Lazy-resolve the TerminalReporter once it's been registered.""" + global _controller_terminal_reporter + if _controller_terminal_reporter is not None: + return _controller_terminal_reporter + if _controller_pluginmanager is None: + return None + _controller_terminal_reporter = _controller_pluginmanager.getplugin( + "terminalreporter" + ) + return _controller_terminal_reporter + + +def pytest_runtest_logreport(report): + """Print VCR verdicts on the controller, alongside PASSED/FAILED markers. + + Runs once per phase per test. We pick teardown so the verdict (appended + in ``_vcr_outcome_gate`` teardown) is present in ``report.user_properties``. + """ + if report.when != "teardown": + return + if os.environ.get("PYTEST_XDIST_WORKER"): + return # only the controller has a live-log TerminalReporter + if not vcr_verbose_enabled(): + return + reporter = _resolve_terminal_reporter() + if reporter is None: + return + verdict = next( + (v for k, v in (report.user_properties or []) if k == "vcr_verdict"), + None, + ) + if not verdict: + return + reporter.write_line(f"{verdict} :: {report.nodeid}") # --------------------------------------------------------------------------- From 80415b472e6d1f3c1f3ef0c811491d73930959e1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 1 May 2026 14:36:48 -0700 Subject: [PATCH 30/30] tests(vcr): drop redundant comments and docstrings Remove explanatory comments that restated what the code already says. Kept only those that document non-obvious external contracts (the aiohttp record-path patch's reason for re-feeding the body, and the warning messages inside save_cassette that reach the user). --- tests/_vcr_redis_persister.py | 70 +------------------ tests/llm_responses_api_testing/conftest.py | 24 ------- tests/llm_translation/base_llm_unit_tests.py | 13 +--- tests/llm_translation/conftest.py | 64 ++--------------- .../test_vcr_redis_persister.py | 18 ----- 5 files changed, 9 insertions(+), 180 deletions(-) diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index 3483f9d253..4d72a1142b 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -9,35 +9,15 @@ from vcr.serialize import deserialize, serialize CASSETTE_TTL_SECONDS = 24 * 60 * 60 REDIS_KEY_PREFIX = "litellm:vcr:cassette:" - -# Healthy cassettes hold 1–5 episodes (a single test rarely makes more than a -# handful of distinct HTTP calls). When a cassette balloons past this, it -# usually means a test produces non-deterministic request bodies (e.g. uuid) -# under record_mode=new_episodes, and every CI run is appending fresh -# unmatched episodes instead of replaying. That growth is unbounded over -# time and silently inflates Redis. Refuse to persist past this threshold so -# the pathology surfaces loudly instead. +CASSETTE_REDIS_URL_ENV = "CASSETTE_REDIS_URL" +VCR_VERBOSE_ENV = "LITELLM_VCR_VERBOSE" MAX_EPISODES_PER_CASSETTE = 50 _log = logging.getLogger(__name__) - - -# Per-process map: cassette key -> "did the test that produced this cassette -# pass?". The conftest's pytest_runtest_makereport hook sets True when the test -# body succeeds; save_cassette consults it to avoid persisting recordings from -# failed runs. We key by the redis cache key so retries (which may produce a -# fresh cassette object each time but write to the same key) interleave -# correctly. _passed_by_cassette_key: dict[str, bool] = {} def mark_test_outcome_for_cassette(cassette_path: str, passed: bool) -> None: - """Record whether the test that owns ``cassette_path`` passed. - - Called from a pytest hook in conftest. The recorded value is consulted by - ``save_cassette`` so failed-attempt recordings (e.g. a flaky test that - asserts on provider state) don't poison the cache for future runs. - """ _passed_by_cassette_key[redis_key_for(cassette_path)] = passed @@ -49,14 +29,7 @@ def redis_key_for(cassette_path: str) -> str: return f"{REDIS_KEY_PREFIX}{rel}" -CASSETTE_REDIS_URL_ENV = "CASSETTE_REDIS_URL" - - def _redis_url_from_env() -> Optional[str]: - # Use a dedicated cassette Redis URL so the VCR cache is isolated from any - # application Redis used by tests (which may be flushed by other suites). - # Intentionally do NOT fall back to REDIS_URL/REDIS_HOST — sharing a Redis - # with the app cache risks cassettes being wiped by flushdb/flushall. return os.environ.get(CASSETTE_REDIS_URL_ENV) or None @@ -74,8 +47,6 @@ def _build_default_client(): "Cassette Redis is intentionally separate from the application " "Redis (REDIS_URL/REDIS_HOST) to avoid being flushed by tests." ) - # Managed Redis providers (e.g. Upstash) drop idle TLS connections; retry on - # connection/timeout errors so a single dropped socket doesn't fail teardown. return redis.Redis.from_url( url, socket_timeout=5, @@ -92,8 +63,6 @@ def make_redis_persister( ): redis_client = client if client is not None else _build_default_client() - # Lazily resolve the redis exception classes so callers can pass any - # client (incl. fakeredis) without importing the real `redis` package. try: from redis.exceptions import ConnectionError as RedisConnectionError from redis.exceptions import TimeoutError as RedisTimeoutError @@ -108,8 +77,6 @@ def make_redis_persister( try: data = redis_client.get(redis_key_for(cassette_path)) except _transient_errors as exc: - # Treat a Redis outage on read as a cassette miss so tests fall - # through to a live call instead of erroring in setup. _log.warning( "VCR redis load failed for %s; treating as cache miss: %s", cassette_path, @@ -125,20 +92,9 @@ def make_redis_persister( @staticmethod def save_cassette(cassette_path, cassette_dict, serializer): key = redis_key_for(cassette_path) - # Only persist successful runs. A failed test (incl. all the failed - # retries before a passing one) would otherwise poison the cache — - # e.g. a flaky test that observes provider state across two calls - # could capture a "bad luck" response that deterministically fails - # every future replay. We default to True if the hook didn't run - # (e.g. cassette saved outside a test context) so non-test usage - # still works. passed = _passed_by_cassette_key.pop(key, True) episode_count = len(cassette_dict.get("requests", []) or []) if episode_count > MAX_EPISODES_PER_CASSETTE: - # Pathology: the test is producing non-deterministic request - # bodies and accumulating unbounded episodes. Refuse the save - # so the cassette can't keep ballooning, and surface a loud - # warning so someone investigates / opts the test out. _log.warning( "VCR redis save refused for %s; cassette has %d episodes " "(> MAX_EPISODES_PER_CASSETTE=%d). The test likely produces " @@ -162,9 +118,6 @@ def make_redis_persister( try: redis_client.set(key, payload, ex=ttl_seconds) except _transient_errors as exc: - # Cassette persistence is a cache, not test correctness. A Redis - # outage on save should not fail an otherwise-passing test — - # the next run will simply re-record. _log.warning( "VCR redis save failed for %s; cassette not persisted: %s", cassette_path, @@ -175,7 +128,6 @@ def make_redis_persister( def filter_non_2xx_response(response): - # Returning None tells vcrpy to skip persisting; see Cassette.append. if not isinstance(response, dict): return response status = response.get("status") @@ -209,32 +161,14 @@ def patch_vcrpy_aiohttp_record_path() -> None: _PATCHED_AIOHTTP_RECORD = True -VCR_VERBOSE_ENV = "LITELLM_VCR_VERBOSE" - - def vcr_verbose_enabled() -> bool: return os.environ.get(VCR_VERBOSE_ENV) == "1" def format_vcr_verdict(cassette: Any) -> str: - """Build a one-line hit/miss verdict for a vcrpy Cassette. - - HIT — at least one request was served from cache and nothing new was - recorded. (Pure replay.) - MISS — nothing from cache; one or more requests went live and were - recorded. (Cold cache.) - PARTIAL — mix of replay and new recordings. Usually means the cassette - matches some but not all requests for this test (e.g. retries, - new branches, or vcrpy match_on too strict). - NOOP — test made no HTTP calls (or VCR not engaged for it). - """ if cassette is None: return "[VCR NOOP]" played = getattr(cassette, "play_count", 0) or 0 - # cassette.data is the recorded request/response list; len(cassette) counts - # recorded episodes. New recordings during this test = len - prior_len, but - # we don't have prior_len here, so we use cassette.dirty (set when an append - # happened during this run) as the "new recording" signal. dirty = getattr(cassette, "dirty", False) total = len(cassette) if hasattr(cassette, "__len__") else 0 if played == 0 and not dirty: diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index 612e8691f9..80f36e159a 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -23,8 +23,6 @@ from tests._vcr_redis_persister import ( # noqa: E402 ) -# Controller-side handles for writing per-test VCR verdicts to the live -# terminal. See the matching comment in tests/llm_translation/conftest.py. _controller_pluginmanager = None _controller_terminal_reporter = None @@ -98,9 +96,6 @@ def vcr_config(): def _vcr_disabled() -> bool: if os.environ.get("LITELLM_VCR_DISABLE") == "1": return True - # Cassettes live on a dedicated Redis (CASSETTE_REDIS_URL) so the cache - # isn't shared with — and accidentally flushed by — tests that exercise - # the application Redis via REDIS_URL/REDIS_HOST. return not os.environ.get("CASSETTE_REDIS_URL") @@ -113,12 +108,6 @@ def pytest_recording_configure(config, vcr): @pytest.hookimpl(hookwrapper=True) def pytest_runtest_makereport(item, call): - """Attach each phase's report to the item so fixture teardown can read it. - - Used by ``_vcr_outcome_gate`` below to skip persisting cassettes for - failed test runs (incl. failed retries that pytest-rerunfailures will - re-attempt) so a "bad luck" recording can't poison future replays. - """ outcome = yield rep = outcome.get_result() setattr(item, f"rep_{rep.when}", rep) @@ -126,17 +115,6 @@ def pytest_runtest_makereport(item, call): @pytest.fixture(autouse=True) def _vcr_outcome_gate(request, vcr): - """Tell the persister whether the test that owns this cassette passed. - - Runs after ``vcr`` (which yields the active Cassette). At teardown time - the call-phase report is attached to the item by the makereport hook - above, so we can mark the cassette key passed/failed before vcrpy's - Cassette.__exit__ triggers persister.save_cassette. - - Stashes a per-test hit/miss verdict on ``user_properties`` so the - controller-side ``pytest_runtest_logreport`` hook can surface it to the - live terminal under xdist. - """ yield cassette = vcr rep_call = getattr(request.node, "rep_call", None) @@ -152,7 +130,6 @@ def _vcr_outcome_gate(request, vcr): def pytest_configure(config): - """Stash the pluginmanager so the logreport hook can find TerminalReporter.""" global _controller_pluginmanager if os.environ.get("PYTEST_XDIST_WORKER"): return @@ -172,7 +149,6 @@ def _resolve_terminal_reporter(): def pytest_runtest_logreport(report): - """Emit per-test VCR verdicts on the controller's live terminal.""" if report.when != "teardown": return if os.environ.get("PYTEST_XDIST_WORKER"): diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index 5e7b2f190e..450e18d0fa 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -401,16 +401,9 @@ class BaseLLMChatTest(ABC): { "type": "file", "file": { - # jsDelivr serves the repo's tests/llm_translation/fixtures/dummy.pdf - # with `Content-Type: application/pdf`. Two reasons we don't - # use raw.githubusercontent.com or upload.wikimedia.org: - # - raw GitHub returns Content-Type: application/octet-stream, - # which OpenAI/Gemini reject when LiteLLM fetches the URL - # client-side and forwards the bytes. - # - Wikimedia URLs intermittently return 400 from Anthropic's - # server-side URL fetcher. - # The URL is pinned to a specific commit SHA so jsDelivr can - # serve it as immutable (cache-control: immutable, max-age=1y). + # SHA-pinned jsDelivr mirror of tests/llm_translation/fixtures/dummy.pdf; + # raw.githubusercontent.com serves PDFs as application/octet-stream + # which OpenAI/Gemini reject when LiteLLM client-fetches the URL. "file_id": "https://cdn.jsdelivr.net/gh/BerriAI/litellm@aab3ef8988b12d166b20356a81c53127480f1125/tests/llm_translation/fixtures/dummy.pdf" }, }, diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 1976dc476a..09da0520be 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -28,19 +28,12 @@ from tests._vcr_redis_persister import ( # noqa: E402 ) -# Controller-side handles for writing per-test VCR verdicts to the live -# terminal. ``pytest_configure`` stashes the pluginmanager (workers don't get -# a TerminalReporter — their output is captured and aggregated by the -# controller), and ``pytest_runtest_logreport`` resolves the TerminalReporter -# lazily on first use because it isn't registered yet at conftest configure -# time. _controller_pluginmanager = None _controller_terminal_reporter = None # vcrpy and respx both patch the httpx transport — applying both makes one -# silently win. Files in this set use respx and are skipped by the -# auto-marker below. +# silently win, so respx-using files opt out of the auto-marker. _RESPX_CONFLICTING_FILES = frozenset( { "test_azure_o_series.py", @@ -58,26 +51,11 @@ _VCR_AUTO_MARKER_SKIP_FILES = _RESPX_CONFLICTING_FILES | frozenset( ) # Tests that observe live cross-call provider state (e.g. prompt-cache -# warm-up between two consecutive calls) cannot benefit from cassette -# replay: the second call's "expected" state depends on what the *live* -# provider does between the two calls, not on what was recorded earlier. -# Auto-marking them with @pytest.mark.vcr just wastes cycles and (before -# the outcome gate) used to poison the cache. They go live with their -# existing @pytest.mark.flaky retry logic. -# -# Match by suffix on the pytest nodeid so subclassed/parametrized variants -# are covered: e.g. "::test_prompt_caching" matches all subclasses that -# inherit the base test. +# warm-up between two consecutive calls); replay can't reproduce that state. _VCR_INCOMPATIBLE_NODEID_SUFFIXES = frozenset( { - # Provider prompt-cache propagation isn't deterministic between two - # back-to-back calls; the test is flaky against the live provider. "::test_prompt_caching", - # Bedrock Nova returns tool_call vs JSON nondeterministically; the - # base assertion expects JSON. Other providers' versions of this - # test are healthy, so we narrow with a class-name guard below. "TestBedrockInvokeNovaJson::test_json_response_pydantic_obj", - # Bedrock streaming response_cost calc returns None intermittently. "::test_bedrock_converse__streaming_passthrough", } ) @@ -156,9 +134,6 @@ def vcr_config(): def _vcr_disabled() -> bool: if os.environ.get("LITELLM_VCR_DISABLE") == "1": return True - # Cassettes live on a dedicated Redis (CASSETTE_REDIS_URL) so the cache - # isn't shared with — and accidentally flushed by — tests that exercise - # the application Redis via REDIS_URL/REDIS_HOST. return not os.environ.get("CASSETTE_REDIS_URL") @@ -171,12 +146,6 @@ def pytest_recording_configure(config, vcr): @pytest.hookimpl(hookwrapper=True) def pytest_runtest_makereport(item, call): - """Attach each phase's report to the item so fixture teardown can read it. - - Used by ``_vcr_outcome_gate`` below to skip persisting cassettes for - failed test runs (incl. failed retries that pytest-rerunfailures will - re-attempt) so a "bad luck" recording can't poison future replays. - """ outcome = yield rep = outcome.get_result() setattr(item, f"rep_{rep.when}", rep) @@ -184,19 +153,6 @@ def pytest_runtest_makereport(item, call): @pytest.fixture(autouse=True) def _vcr_outcome_gate(request, vcr): - """Tell the persister whether the test that owns this cassette passed. - - Runs after ``vcr`` (which yields the active Cassette). At teardown time - the call-phase report is attached to the item by the makereport hook - above, so we can mark the cassette key passed/failed before vcrpy's - Cassette.__exit__ triggers persister.save_cassette. - - Stashes a per-test hit/miss verdict on ``user_properties`` so the - controller-side ``pytest_runtest_logreport`` hook can surface it to the - live terminal. xdist serializes ``user_properties`` on each phase's - report back to the controller, which is the only process that has a - TerminalReporter wired to CI's live log. - """ yield cassette = vcr rep_call = getattr(request.node, "rep_call", None) @@ -212,20 +168,13 @@ def _vcr_outcome_gate(request, vcr): def pytest_configure(config): - """Stash the pluginmanager so the logreport hook can find TerminalReporter. - - We can't grab TerminalReporter directly here — it's not registered until - pytest's own ``pytest_configure`` runs, and conftest hooks may run first. - Stashing the config is enough; the hook resolves on first use. - """ global _controller_pluginmanager if os.environ.get("PYTEST_XDIST_WORKER"): - return # workers don't have a live-log TerminalReporter + return _controller_pluginmanager = config.pluginmanager def _resolve_terminal_reporter(): - """Lazy-resolve the TerminalReporter once it's been registered.""" global _controller_terminal_reporter if _controller_terminal_reporter is not None: return _controller_terminal_reporter @@ -238,15 +187,10 @@ def _resolve_terminal_reporter(): def pytest_runtest_logreport(report): - """Print VCR verdicts on the controller, alongside PASSED/FAILED markers. - - Runs once per phase per test. We pick teardown so the verdict (appended - in ``_vcr_outcome_gate`` teardown) is present in ``report.user_properties``. - """ if report.when != "teardown": return if os.environ.get("PYTEST_XDIST_WORKER"): - return # only the controller has a live-log TerminalReporter + return if not vcr_verbose_enabled(): return reporter = _resolve_terminal_reporter() diff --git a/tests/llm_translation/test_vcr_redis_persister.py b/tests/llm_translation/test_vcr_redis_persister.py index 1e4b6c83aa..853558150c 100644 --- a/tests/llm_translation/test_vcr_redis_persister.py +++ b/tests/llm_translation/test_vcr_redis_persister.py @@ -74,10 +74,6 @@ def test_load_missing_key_raises_cassette_not_found(): def test_redis_key_normalizes_path_passed_by_pytest_recording(): - # pytest-recording passes paths shaped like - # ``/cassettes//.yaml``. The persister stores them - # under a clean test-identifier key — no extension, no ``cassettes/`` - # directory segment — so ``redis-cli keys`` reads as test IDs. raw = "tests/llm_translation/cassettes/test_anthropic/test_streaming.yaml" assert ( redis_key_for(raw) @@ -86,8 +82,6 @@ def test_redis_key_normalizes_path_passed_by_pytest_recording(): class _FlakyRedis: - """Wraps a fake redis but raises ConnectionError on the chosen op.""" - def __init__(self, inner, fail_on: str): self._inner = inner self._fail_on = fail_on @@ -104,7 +98,6 @@ class _FlakyRedis: def test_save_swallows_connection_errors_so_teardown_does_not_fail(): - # Persistence is a cache; an outage shouldn't fail an otherwise-passing test. flaky = _FlakyRedis(fakeredis.FakeStrictRedis(), fail_on="set") persister = make_redis_persister(client=flaky) @@ -116,18 +109,15 @@ def test_save_swallows_connection_errors_so_teardown_does_not_fail(): def test_save_skipped_when_test_marked_failed_and_prior_cassette_preserved(): - # A flaky test that fails should NOT overwrite a previously-good cassette. fake, persister = _persister_with_fake_redis() cassette_id = "tests/llm_translation/test_x/test_flaky" key = redis_key_for(cassette_id) - # Seed a "known-good" recording from a prior successful run. good = _sample_cassette_dict() persister.save_cassette(cassette_id, good, yamlserializer) good_payload = fake.get(key) assert good_payload is not None - # Simulate a failed run: the hook records "did not pass" before save. mark_test_outcome_for_cassette(cassette_id, passed=False) bad_response = { "status": {"code": 200, "message": "OK"}, @@ -137,7 +127,6 @@ def test_save_skipped_when_test_marked_failed_and_prior_cassette_preserved(): bad = {"requests": good["requests"], "responses": [bad_response]} persister.save_cassette(cassette_id, bad, yamlserializer) - # Prior good payload is still there — the bad save was suppressed. assert fake.get(key) == good_payload @@ -153,8 +142,6 @@ def test_save_proceeds_when_test_marked_passed(): def test_save_refused_when_cassette_exceeds_max_episodes(): - # Pathological cassettes (non-deterministic body → unbounded episode growth) - # should be refused. Any prior good payload stays intact. fake, persister = _persister_with_fake_redis() cassette_id = "tests/llm_translation/test_x/test_runaway" key = redis_key_for(cassette_id) @@ -179,7 +166,6 @@ def test_save_refused_when_cassette_exceeds_max_episodes(): } persister.save_cassette(cassette_id, bloated, yamlserializer) - # Refused — the seed payload is unchanged. assert fake.get(key) == seed_payload @@ -209,8 +195,6 @@ def test_save_proceeds_at_max_episodes_threshold(): def test_save_proceeds_when_outcome_unknown(): - # Used outside a pytest run (e.g. ad-hoc scripts), the outcome gate is - # bypassed so the persister still works. fake, persister = _persister_with_fake_redis() cassette_id = "tests/llm_translation/test_x/test_no_marker" key = redis_key_for(cassette_id) @@ -221,8 +205,6 @@ def test_save_proceeds_when_outcome_unknown(): def test_load_treats_connection_errors_as_cassette_miss(): - # An outage on read should fall through to a live call (CassetteNotFound), - # not surface a redis exception in the test setup. flaky = _FlakyRedis(fakeredis.FakeStrictRedis(), fail_on="get") persister = make_redis_persister(client=flaky)