From b637d9f64a6cb6d28c96680f877d93e37b8bf61d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 13 May 2026 00:31:47 +0000 Subject: [PATCH 01/11] test(vcr): classify cache verdicts, detect live calls, surface cost leaks Convert the per-test VCR verdict line from a single 'NOOP / HIT / MISS / PARTIAL' tag into a classified outcome that distinguishes the cases that silently bill the live API on every CI run from the ones that don't: HIT pure replay PARTIAL mixed replay + new recordings MISS:RECORDED new cassette saved to Redis (cached next run) MISS:OVERFLOW cassette > MAX_EPISODES_PER_CASSETTE; persister refused to save; re-bills every run MISS:NOT_PERSISTED test failed; save_cassette skipped; re-bills NOOP VCR-marked but no HTTP traffic (mocked elsewhere) UNMARKED:LIVE_CALL test bypassed VCR AND opened a TCP connection to a known LLM provider host -> wasted spend UNMARKED:NO_TRAFFIC test bypassed VCR but didn't call out The UNMARKED:LIVE_CALL signal is what converts 'this test probably hits live' into 'this test connected to api.openai.com'. We install a socket.connect / socket.create_connection wrapper for the duration of each non-VCR-marked test and record any outbound TCP to a known LLM provider hostname. The probe sits below the httpx layer so vcrpy and respx (which both patch above the socket) are unaffected. Replace the file-level _RESPX_CONFLICTING_FILES blacklists in the llm_translation and local_testing conftests with per-item respx detection in apply_vcr_auto_marker_to_items. A test now skips VCR when it actually carries @pytest.mark.respx or has respx_mock in its fixture chain - not just because some other test in the same file imports MockRouter. Items skipped by skip_files are split into respx_conflict (real conflict, the module wires up respx) vs file_opt_out (dead skip- list entry whose module never touches respx) so the session summary makes pruning obvious. Stabilize the AWS SigV4 fingerprint: the Authorization header on Bedrock requests rotates its Credential date and Signature on every call, which previously pushed every Bedrock test past the 50-episode overflow threshold. Extract the access-key id only ('aws-sigv4:AKIA...') so two requests with the same identity match. Always emit verdict logging when VCR is active (set LITELLM_VCR_VERBOSE=0 to opt back into the legacy quiet mode). Add a session-end classification summary that lists overflow tests, unmarked live-call tests, and the skip-reason breakdown. Wire the live-call probe + summary hook into every test directory that already uses the Redis-backed VCR cache (audio_tests, guardrails_tests, image_gen_tests, litellm_utils_tests, llm_responses_api_testing, llm_translation, local_testing, logging_callback_tests, ocr_tests, pass_through_unit_tests, router_unit_tests, search_tests, unified_google_tests). Add tests/llm_translation/test_vcr_classification.py covering the verdict classifier, skip-reason tagging, AWS SigV4 fingerprint stability, live-host classification, and session summary rendering. Co-authored-by: Mateo Wang --- tests/_vcr_conftest_common.py | 520 +++++++++++++++++- tests/audio_tests/conftest.py | 9 + tests/guardrails_tests/conftest.py | 9 + tests/image_gen_tests/conftest.py | 9 + tests/litellm_utils_tests/conftest.py | 9 + tests/llm_responses_api_testing/conftest.py | 9 + tests/llm_translation/conftest.py | 31 +- .../test_vcr_classification.py | 497 +++++++++++++++++ tests/local_testing/conftest.py | 27 +- tests/logging_callback_tests/conftest.py | 9 + tests/ocr_tests/conftest.py | 9 + tests/pass_through_unit_tests/conftest.py | 9 + tests/router_unit_tests/conftest.py | 9 + tests/search_tests/conftest.py | 9 + tests/unified_google_tests/conftest.py | 9 + 15 files changed, 1135 insertions(+), 39 deletions(-) create mode 100644 tests/llm_translation/test_vcr_classification.py diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index b2c7eeb78d..2bdddb69d0 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -10,20 +10,22 @@ import hashlib import json import os import re +import socket import sys +from collections import defaultdict from typing import Iterable import pytest from tests._vcr_redis_persister import ( + MAX_EPISODES_PER_CASSETTE, + VCR_VERBOSE_ENV, cassette_cache_capacity_snapshot, cassette_cache_health, filter_non_2xx_response, - format_vcr_verdict, make_redis_persister, mark_test_outcome_for_cassette, patch_vcrpy_aiohttp_record_path, - vcr_verbose_enabled, ) CASSETTE_CACHE_HIGH_WATER_FRACTION = 0.85 @@ -231,6 +233,29 @@ def _iter_header_values(headers, name: str): yield value +_AWS_SIGV4_CREDENTIAL_RE = re.compile( + r"AWS4-HMAC-SHA256\s+Credential=([^/\s,]+)/", re.IGNORECASE +) + + +def _stable_key_value(header_name: str, raw: str) -> str: + """Return a *stable* identifier for a credential header. + + For Bearer / API-key headers the entire value is stable across calls, + so we hash it as-is. For AWS SigV4 ``Authorization`` headers, only + the access-key portion of ``Credential=AKIA...//...`` is stable + — date, region, signed headers, and signature all rotate per request, + so hashing the full value would push every Bedrock request into a new + cassette episode. Extract just the access-key id when present. + """ + if header_name.lower() != "authorization": + return raw + match = _AWS_SIGV4_CREDENTIAL_RE.search(raw) + if match: + return f"aws-sigv4:{match.group(1)}" + return raw + + def _compute_key_fingerprint(request) -> str: headers = getattr(request, "headers", None) parts: list[str] = [] @@ -242,7 +267,8 @@ def _compute_key_fingerprint(request) -> str: text = text.strip() if not text: continue - parts.append(f"{header_name}={text}") + stable = _stable_key_value(header_name, text) + parts.append(f"{header_name}={stable}") if not parts: return "no-key" digest = hashlib.sha256("\n".join(parts).encode("utf-8")).hexdigest() @@ -470,6 +496,114 @@ def register_persister_if_enabled(vcr) -> None: _atexit_banner_registered = True +VCR_SKIP_REASON_USER_ATTR = "vcr_skip_reason" + +# Marker reasons recorded per-item / per-test for the session summary. +SKIP_REASON_RESPX = "respx_conflict" +SKIP_REASON_RESPX_MODULE = "respx_conflict_module" +SKIP_REASON_INCOMPATIBLE = "incompatible" +SKIP_REASON_FILE_OPT_OUT = "file_opt_out" +SKIP_REASON_DISABLED = "disabled" +SKIP_REASON_PRE_MARKED = "already_marked" + +# Hostnames we consider an "expensive live call" if a non-VCR-marked test +# happens to hit them. Localhost/redis/databases are explicitly excluded. +_LIVE_CALL_HOST_SUFFIXES = ( + ".openai.com", + ".anthropic.com", + ".vertexai.googleapis.com", + ".aiplatform.googleapis.com", + ".googleapis.com", + ".bedrock-runtime.amazonaws.com", + ".x.ai", + ".cohere.ai", + ".cohere.com", + ".voyageai.com", + ".perplexity.ai", + ".mistral.ai", + ".groq.com", + ".huggingface.co", + ".azure.com", + ".tavily.com", + ".serper.dev", + ".searchapi.io", + ".firecrawl.dev", + ".exa.ai", +) +_LIVE_CALL_LOCAL_PREFIXES = ( + "127.", + "localhost", + "::1", + "0.0.0.0", + "10.", + "172.16.", + "172.17.", + "172.18.", + "172.19.", + "172.20.", + "192.168.", +) + + +def _module_uses_respx(item) -> bool: + """Return True if the test's *module* actually wires up respx. + + A bare ``from respx import MockRouter`` import (with no actual usage) + does not patch the httpx transport, so it does not conflict with vcrpy. + We confirm by checking the module's source for any of: + - ``@pytest.mark.respx`` + - ``@respx.mock`` / ``with respx.mock`` + - ``respx_mock`` fixture name + """ + module = getattr(item, "module", None) + src_file = getattr(module, "__file__", None) + if not src_file or not os.path.isfile(src_file): + return False + try: + with open(src_file, encoding="utf-8") as f: + src = f.read() + except OSError: + return False + if "respx_mock" in src: + return True + if "@pytest.mark.respx" in src or "@respx.mock" in src: + return True + if "respx.mock" in src or "with respx" in src: + return True + return False + + +def _item_uses_respx(item) -> bool: + """Return True if *this specific item* will trigger respx. + + Two signals: the ``respx`` pytest marker, and the ``respx_mock`` + fixture appearing in the item's resolved fixture chain. Either alone + causes vcrpy + respx to fight over the httpx transport. + """ + if item.get_closest_marker("respx") is not None: + return True + fixturenames = getattr(item, "fixturenames", None) or () + if "respx_mock" in fixturenames: + return True + return False + + +# Cache the source-scan result so we don't reread each module per item. +_RESPX_MODULE_CACHE: dict[str, bool] = {} + + +def _module_path_uses_respx(item) -> bool: + src_file = str(getattr(item, "path", "") or "") + if not src_file: + return False + cached = _RESPX_MODULE_CACHE.get(src_file) + if cached is not None: + return cached + result = _module_uses_respx(item) + _RESPX_MODULE_CACHE[src_file] = result + return result + + def apply_vcr_auto_marker_to_items( items, *, @@ -478,26 +612,232 @@ def apply_vcr_auto_marker_to_items( ) -> None: """Auto-apply ``pytest.mark.vcr`` to collected items. - ``skip_files`` are basenames to leave un-marked (e.g. respx-using - files, since respx and vcrpy both patch the httpx transport). - ``skip_nodeid_suffixes`` are node-id suffixes for individual tests - that depend on live cross-call provider state. + Skip semantics (in priority order): + + 1. ``vcr_disabled()`` — global env-var off-switch (``LITELLM_VCR_DISABLE=1`` + or no ``CASSETTE_REDIS_URL``). + 2. Item already carries ``@pytest.mark.vcr`` — leave it alone. + 3. Item triggers respx (per-item marker / fixture) — vcrpy and respx + both patch the httpx transport so applying both makes one silently + no-op. We tag the item ``vcr_skip_reason=respx_conflict``. + 4. Module wires up respx anywhere — even tests in the file that don't + themselves use respx still inherit the patched transport when + respx fixtures activate at session level. Tagged + ``respx_conflict_module``. + 5. ``skip_files`` / ``skip_nodeid_suffixes`` opt-out lists from the + caller — used for tests that observe live cross-call provider state + (e.g. prompt-cache warmup) which deterministic replay can't model. + Tagged ``incompatible``. + + Each skipped item gets a ``vcr_skip_reason`` attribute so the + session-end summary can show why it isn't cached. """ if vcr_disabled(): + for item in items: + setattr(item, VCR_SKIP_REASON_USER_ATTR, SKIP_REASON_DISABLED) return skip_files = frozenset(skip_files) skip_nodeid_suffixes = tuple(skip_nodeid_suffixes) for item in items: + if item.get_closest_marker("vcr") is not None: + setattr(item, VCR_SKIP_REASON_USER_ATTR, SKIP_REASON_PRE_MARKED) + continue + if _item_uses_respx(item): + setattr(item, VCR_SKIP_REASON_USER_ATTR, SKIP_REASON_RESPX) + continue filename = os.path.basename(str(item.path)) if filename in skip_files: + # Trust the caller's opt-out, but split by reason: if the + # module actually uses respx, label the conflict precisely so + # the summary surfaces dead respx imports vs. real conflicts. + if _module_path_uses_respx(item): + setattr(item, VCR_SKIP_REASON_USER_ATTR, SKIP_REASON_RESPX_MODULE) + else: + setattr(item, VCR_SKIP_REASON_USER_ATTR, SKIP_REASON_FILE_OPT_OUT) continue if any(item.nodeid.endswith(suffix) for suffix in skip_nodeid_suffixes): - continue - if item.get_closest_marker("vcr") is not None: + setattr(item, VCR_SKIP_REASON_USER_ATTR, SKIP_REASON_INCOMPATIBLE) continue item.add_marker(pytest.mark.vcr) +# --------------------------------------------------------------------------- +# Per-test stats accumulator + verdict classification. +# +# The session-end summary needs richer signal than the line-level verdict: +# - which tests overflowed ``MAX_EPISODES_PER_CASSETTE`` (cassette refused +# to save → live calls every CI run); +# - which tests fired live HTTP at a real LLM endpoint while VCR was not +# active for them (genuine wasted spend, not just "test mocked elsewhere"); +# - skip-reason buckets so we can tell respx-conflict from +# incompatible-by-design from "module imports respx but never uses it". +# --------------------------------------------------------------------------- + +# Verdict tags used in the per-test logline AND in the session summary +# breakdown. +VERDICT_HIT = "VCR HIT" +VERDICT_MISS_RECORDED = "VCR MISS:RECORDED" +VERDICT_MISS_OVERFLOW = "VCR MISS:OVERFLOW" +VERDICT_MISS_NOT_PERSISTED = "VCR MISS:NOT_PERSISTED" +VERDICT_PARTIAL = "VCR PARTIAL" +VERDICT_NOOP_NO_TRAFFIC = "VCR NOOP" +VERDICT_UNMARKED_LIVE_CALL = "VCR UNMARKED:LIVE_CALL" +VERDICT_UNMARKED_NO_TRAFFIC = "VCR UNMARKED:NO_TRAFFIC" +VERDICT_DISABLED = "VCR DISABLED" + +# Per-session stats. Cleared by ``_reset_session_stats`` for unit tests. +_session_stats = { + "verdict_counts": defaultdict(int), + "overflow_tests": [], # list of nodeids + "unmarked_live_call_tests": [], # list of (nodeid, hosts) + "skip_reason_counts": defaultdict(int), + "skip_reason_examples": defaultdict(list), +} + + +def _reset_session_stats() -> None: + _session_stats["verdict_counts"].clear() + _session_stats["overflow_tests"].clear() + _session_stats["unmarked_live_call_tests"].clear() + _session_stats["skip_reason_counts"].clear() + _session_stats["skip_reason_examples"].clear() + + +def session_stats_snapshot() -> dict: + """Read-only copy of the per-session VCR stats. Used by the summary.""" + return { + "verdict_counts": dict(_session_stats["verdict_counts"]), + "overflow_tests": list(_session_stats["overflow_tests"]), + "unmarked_live_call_tests": list(_session_stats["unmarked_live_call_tests"]), + "skip_reason_counts": dict(_session_stats["skip_reason_counts"]), + "skip_reason_examples": { + k: list(v) for k, v in _session_stats["skip_reason_examples"].items() + }, + } + + +def _classify_marked_test(cassette) -> str: + """Map cassette state → verdict tag for tests that *were* VCR-marked.""" + played = getattr(cassette, "play_count", 0) or 0 + dirty = getattr(cassette, "dirty", False) + total = len(cassette) if hasattr(cassette, "__len__") else 0 + + # "OVERFLOW" mirrors ``_RedisPersister.save_cassette``'s + # ``> MAX_EPISODES_PER_CASSETTE`` guard. Cassettes that hit this + # threshold are refused for save, so the test re-records live every + # run. + if total > MAX_EPISODES_PER_CASSETTE: + return VERDICT_MISS_OVERFLOW + if played == 0 and not dirty: + return VERDICT_NOOP_NO_TRAFFIC + if played > 0 and not dirty: + return VERDICT_HIT + if played == 0 and dirty: + return VERDICT_MISS_RECORDED + return VERDICT_PARTIAL + + +def _format_verdict_line(verdict: str, cassette, extra: str = "") -> str: + if cassette is None: + return f"[{verdict}]{(' ' + extra) if extra else ''}" + played = getattr(cassette, "play_count", 0) or 0 + total = len(cassette) if hasattr(cassette, "__len__") else 0 + base = f"[{verdict}] played={played} entries={total}" + if extra: + base = f"{base} {extra}" + return base + + +# --------------------------------------------------------------------------- +# Live-call detection for tests that bypass VCR. +# +# When a test isn't VCR-marked (respx_conflict, incompatible, or just +# plain unmarked), we wrap its socket calls inside the autouse +# ``_vcr_outcome_gate`` fixture so we can flag any outbound TCP connection +# to a known LLM provider. This converts "likely live call" into +# "confirmed: this test connected to host X". +# --------------------------------------------------------------------------- + +_LIVE_CALL_PROBE_INSTALLED = False +_LIVE_CALL_BUFFER_KEY = "vcr_live_call_hosts" + + +def _is_live_call_host(host: str) -> bool: + if not host: + return False + host = host.lower() + if any(host.startswith(p) for p in _LIVE_CALL_LOCAL_PREFIXES): + return False + return any(host.endswith(suffix) for suffix in _LIVE_CALL_HOST_SUFFIXES) + + +class _LiveCallProbe: + """Context manager that monkeypatches ``socket.create_connection`` and + ``socket.socket.connect`` for the lifetime of a test, recording any + outbound TCP connection to a known LLM host. + + We don't intercept HTTP at the application layer because that would + fight with vcrpy/respx in tests that *do* mock httpx — the socket + layer is below both, so this probe is safe regardless of what's + patched above it. We also don't raise: the goal is observability, not + a hard gate. + """ + + def __init__(self) -> None: + self.hosts: list[str] = [] + self._orig_create_connection = None + self._orig_socket_connect = None + + def __enter__(self): + self._orig_create_connection = socket.create_connection + self._orig_socket_connect = socket.socket.connect + + def _wrapped_create_connection(address, *args, **kwargs): + try: + host = address[0] if isinstance(address, tuple) else None + if host and _is_live_call_host(host) and host not in self.hosts: + self.hosts.append(host) + except Exception: + pass + return self._orig_create_connection(address, *args, **kwargs) + + def _wrapped_socket_connect(sock_self, address): + try: + host = address[0] if isinstance(address, tuple) else None + if host and _is_live_call_host(host) and host not in self.hosts: + self.hosts.append(host) + except Exception: + pass + return self._orig_socket_connect(sock_self, address) + + socket.create_connection = _wrapped_create_connection + socket.socket.connect = _wrapped_socket_connect + return self + + def __exit__(self, *exc): + if self._orig_create_connection is not None: + socket.create_connection = self._orig_create_connection + if self._orig_socket_connect is not None: + socket.socket.connect = self._orig_socket_connect + return False + + +def vcr_outcome_logging_enabled() -> bool: + """Verdict logging is on whenever VCR itself is active. + + The old ``LITELLM_VCR_VERBOSE=1`` gate kept logs quiet by default, but + that hides the very signal we need to know whether a paid test ran + against a real provider. CI logs already drop a one-line verdict per + test; that's what makes the cost analysis tractable. Set + ``LITELLM_VCR_VERBOSE=0`` if you really want the legacy quiet mode. + """ + if vcr_disabled(): + return False + if os.environ.get(VCR_VERBOSE_ENV) == "0": + return False + return True + + def record_vcr_outcome(request, vcr) -> None: """Call from the post-yield section of an autouse fixture per test.""" cassette = vcr @@ -507,10 +847,71 @@ def record_vcr_outcome(request, vcr) -> None: if cassette_path: mark_test_outcome_for_cassette(cassette_path, test_passed) - if not vcr_verbose_enabled(): + nodeid = request.node.nodeid + + if cassette is not None: + verdict = _classify_marked_test(cassette) + # Track overflow tests even when verbose logging is off — the + # session summary shows them either way. + if verdict == VERDICT_MISS_OVERFLOW: + _session_stats["overflow_tests"].append(nodeid) + if not test_passed and verdict == VERDICT_MISS_RECORDED: + verdict = VERDICT_MISS_NOT_PERSISTED + _session_stats["verdict_counts"][verdict] += 1 + if vcr_outcome_logging_enabled(): + line = _format_verdict_line(verdict, cassette) + request.node.user_properties.append(("vcr_verdict", line)) return - verdict = format_vcr_verdict(cassette) - request.node.user_properties.append(("vcr_verdict", verdict)) + + # Cassette is None ⇒ test wasn't VCR-marked. Honor the skip reason + # we tagged at collection time, and pull live-call hosts captured by + # the socket probe (if any). + skip_reason = getattr( + request.node, VCR_SKIP_REASON_USER_ATTR, SKIP_REASON_FILE_OPT_OUT + ) + _session_stats["skip_reason_counts"][skip_reason] += 1 + + hosts = getattr(request.node, _LIVE_CALL_BUFFER_KEY, []) or [] + if hosts: + verdict = VERDICT_UNMARKED_LIVE_CALL + _session_stats["unmarked_live_call_tests"].append((nodeid, list(hosts))) + extra = f"reason={skip_reason} hosts={','.join(hosts)}" + else: + verdict = VERDICT_UNMARKED_NO_TRAFFIC + extra = f"reason={skip_reason}" + + _session_stats["verdict_counts"][verdict] += 1 + + examples = _session_stats["skip_reason_examples"][skip_reason] + if len(examples) < 5: + examples.append(nodeid) + + if vcr_outcome_logging_enabled(): + request.node.user_properties.append( + ("vcr_verdict", _format_verdict_line(verdict, None, extra)) + ) + + +def install_live_call_probe(request, vcr) -> None: + """Activate the live-call socket probe for non-VCR-marked tests. + + Call this from inside the per-test autouse ``_vcr_outcome_gate`` + fixture *before* the ``yield``. When ``vcr`` is ``None`` (test isn't + VCR-marked) we patch ``socket.connect`` for the duration of the test + and stash any LLM-host connections on ``request.node`` so + ``record_vcr_outcome`` can include them in the verdict line. + + Tests that *are* VCR-marked don't get the probe — vcrpy itself + intercepts above the socket layer, so any "outbound" socket would be + a recording cycle, not real spend. + """ + if vcr is not None or vcr_disabled(): + return None + probe = _LiveCallProbe() + probe.__enter__() + setattr(request.node, _LIVE_CALL_BUFFER_KEY, probe.hosts) + request.addfinalizer(lambda: probe.__exit__(None, None, None)) + return probe def _format_capacity_line(snapshot: dict) -> str: @@ -525,6 +926,99 @@ def _format_capacity_line(snapshot: dict) -> str: ) +def emit_vcr_classification_summary(terminalreporter) -> None: + """Render the per-classification summary at session end. + + Output sections (only included when non-empty): + + * **Verdict counts** — full breakdown of HIT / MISS:RECORDED / + MISS:OVERFLOW / MISS:NOT_PERSISTED / PARTIAL / NOOP / + UNMARKED:LIVE_CALL / UNMARKED:NO_TRAFFIC. The OVERFLOW and + UNMARKED:LIVE_CALL counts are the cost-leak signals. + * **Cassette overflow** (>``MAX_EPISODES_PER_CASSETTE``) — these tests + fire live every CI run because the persister refuses to save them. + Usually means the request body is non-deterministic (file handle + consumed, AWS SigV4 timestamp, random UUID). + * **Unmarked tests with live API calls** — confirmed live HTTP traffic + to a known LLM host while VCR was *not* active for the test. This + is the "convert likely → confirmed" signal: each entry is real + money the cache would otherwise prevent. + * **Skip-reason breakdown** — how many tests opted out of VCR and + why (respx_conflict, respx_conflict_module, file_opt_out, + incompatible). Bare ``file_opt_out`` entries with zero respx usage + in the module are dead skip-list rows worth pruning. + """ + if vcr_disabled(): + return + if os.environ.get("PYTEST_XDIST_WORKER"): + return + + snapshot = session_stats_snapshot() + counts = snapshot["verdict_counts"] + if not counts: + return + + terminalreporter.write_sep("=", "VCR CACHE CLASSIFICATION SUMMARY", bold=True) + for verdict in ( + VERDICT_HIT, + VERDICT_PARTIAL, + VERDICT_MISS_RECORDED, + VERDICT_MISS_OVERFLOW, + VERDICT_MISS_NOT_PERSISTED, + VERDICT_NOOP_NO_TRAFFIC, + VERDICT_UNMARKED_NO_TRAFFIC, + VERDICT_UNMARKED_LIVE_CALL, + ): + n = counts.get(verdict, 0) + if not n: + continue + terminalreporter.write_line(f" [{verdict}] {n}") + + overflow = snapshot["overflow_tests"] + if overflow: + terminalreporter.write_sep( + "-", + f"CASSETTE OVERFLOW (>{MAX_EPISODES_PER_CASSETTE} episodes, save refused)", + red=True, + bold=True, + ) + terminalreporter.write_line( + " These tests will hit the live provider on every CI run " + "because the persister won't save cassettes that grew past " + "the limit. Stabilize the request body (file handle consumed, " + "SigV4 timestamp, UUID, or boundary leak)." + ) + for nodeid in overflow: + terminalreporter.write_line(f" - {nodeid}") + + live_calls = snapshot["unmarked_live_call_tests"] + if live_calls: + terminalreporter.write_sep( + "-", + "UNMARKED TESTS WITH LIVE API CALLS", + red=True, + bold=True, + ) + terminalreporter.write_line( + " These tests connected to a real LLM provider host while " + "they were NOT VCR-marked. Either add @pytest.mark.vcr " + "explicitly, mock with respx, or move them off the " + "respx_conflict / incompatible skip list." + ) + for nodeid, hosts in live_calls: + terminalreporter.write_line(f" - {nodeid} → {','.join(hosts)}") + + reasons = snapshot["skip_reason_counts"] + if reasons: + terminalreporter.write_sep("-", "SKIP-REASON BREAKDOWN", bold=True) + for reason, n in sorted(reasons.items(), key=lambda kv: -kv[1]): + examples = snapshot["skip_reason_examples"].get(reason, []) + terminalreporter.write_line(f" {reason}: {n}") + for ex in examples: + terminalreporter.write_line(f" - {ex}") + terminalreporter.write_sep("=", bold=True) + + def emit_cassette_cache_session_banner(terminalreporter) -> None: """Call from ``pytest_terminal_summary``. No-op on xdist workers.""" if vcr_disabled(): @@ -600,7 +1094,7 @@ class VerboseReporterState: return if os.environ.get("PYTEST_XDIST_WORKER"): return - if not vcr_verbose_enabled(): + if not vcr_outcome_logging_enabled(): return reporter = self.resolve_terminal_reporter() if reporter is None: diff --git a/tests/audio_tests/conftest.py b/tests/audio_tests/conftest.py index d07057a4b6..ff47853d49 100644 --- a/tests/audio_tests/conftest.py +++ b/tests/audio_tests/conftest.py @@ -8,6 +8,9 @@ sys.path.insert(0, os.path.abspath("../..")) from tests._vcr_conftest_common import ( # noqa: E402 VerboseReporterState, apply_vcr_auto_marker_to_items, + emit_cassette_cache_session_banner, + emit_vcr_classification_summary, + install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, vcr_config_dict, @@ -34,6 +37,7 @@ def pytest_runtest_makereport(item, call): @pytest.fixture(autouse=True) def _vcr_outcome_gate(request, vcr): + install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) @@ -48,3 +52,8 @@ def pytest_runtest_logreport(report): def pytest_collection_modifyitems(config, items): apply_vcr_auto_marker_to_items(items) + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + emit_cassette_cache_session_banner(terminalreporter) + emit_vcr_classification_summary(terminalreporter) diff --git a/tests/guardrails_tests/conftest.py b/tests/guardrails_tests/conftest.py index 674d5500c3..eb563699b2 100644 --- a/tests/guardrails_tests/conftest.py +++ b/tests/guardrails_tests/conftest.py @@ -19,6 +19,9 @@ import litellm from tests._vcr_conftest_common import ( # noqa: E402 VerboseReporterState, apply_vcr_auto_marker_to_items, + emit_cassette_cache_session_banner, + emit_vcr_classification_summary, + install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, vcr_config_dict, @@ -45,6 +48,7 @@ def pytest_runtest_makereport(item, call): @pytest.fixture(autouse=True) def _vcr_outcome_gate(request, vcr): + install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) @@ -151,3 +155,8 @@ def pytest_collection_modifyitems(config, items): # Reorder the items list items[:] = custom_logger_tests + other_tests + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + emit_cassette_cache_session_banner(terminalreporter) + emit_vcr_classification_summary(terminalreporter) diff --git a/tests/image_gen_tests/conftest.py b/tests/image_gen_tests/conftest.py index ae67a4a924..93dec98e70 100644 --- a/tests/image_gen_tests/conftest.py +++ b/tests/image_gen_tests/conftest.py @@ -12,6 +12,9 @@ import litellm # noqa: E402,F401 from tests._vcr_conftest_common import ( # noqa: E402 VerboseReporterState, apply_vcr_auto_marker_to_items, + emit_cassette_cache_session_banner, + emit_vcr_classification_summary, + install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, vcr_config_dict, @@ -48,6 +51,7 @@ def pytest_runtest_makereport(item, call): @pytest.fixture(autouse=True) def _vcr_outcome_gate(request, vcr): + install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) @@ -62,3 +66,8 @@ def pytest_runtest_logreport(report): def pytest_collection_modifyitems(config, items): apply_vcr_auto_marker_to_items(items) + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + emit_cassette_cache_session_banner(terminalreporter) + emit_vcr_classification_summary(terminalreporter) diff --git a/tests/litellm_utils_tests/conftest.py b/tests/litellm_utils_tests/conftest.py index a110128d2f..08745c99c0 100644 --- a/tests/litellm_utils_tests/conftest.py +++ b/tests/litellm_utils_tests/conftest.py @@ -15,6 +15,9 @@ import litellm # noqa: E402,F401 from tests._vcr_conftest_common import ( # noqa: E402 VerboseReporterState, apply_vcr_auto_marker_to_items, + emit_cassette_cache_session_banner, + emit_vcr_classification_summary, + install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, vcr_config_dict, @@ -76,6 +79,7 @@ def pytest_runtest_makereport(item, call): @pytest.fixture(autouse=True) def _vcr_outcome_gate(request, vcr): + install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) @@ -107,3 +111,8 @@ def pytest_collection_modifyitems(config, items): # Reorder the items list items[:] = custom_logger_tests + other_tests + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + emit_cassette_cache_session_banner(terminalreporter) + emit_vcr_classification_summary(terminalreporter) diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index e16d3cb4a3..2a08db5714 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -16,6 +16,9 @@ import litellm # noqa: E402 from tests._vcr_conftest_common import ( # noqa: E402 VerboseReporterState, apply_vcr_auto_marker_to_items, + emit_cassette_cache_session_banner, + emit_vcr_classification_summary, + install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, vcr_config_dict, @@ -42,6 +45,7 @@ def pytest_runtest_makereport(item, call): @pytest.fixture(autouse=True) def _vcr_outcome_gate(request, vcr): + install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) @@ -107,3 +111,8 @@ def pytest_collection_modifyitems(config, items): other_tests.sort(key=lambda x: x.name) items[:] = custom_logger_tests + other_tests + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + emit_cassette_cache_session_banner(terminalreporter) + emit_vcr_classification_summary(terminalreporter) diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index a059c4540c..5fcd31aa32 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -21,27 +21,20 @@ import litellm # noqa: E402 from tests._vcr_conftest_common import ( # noqa: E402 VerboseReporterState, apply_vcr_auto_marker_to_items, + emit_cassette_cache_session_banner, + emit_vcr_classification_summary, + install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, vcr_config_dict, ) -# vcrpy and respx both patch the httpx transport — applying both makes one -# silently win, so respx-using files opt out of the auto-marker. -_RESPX_CONFLICTING_FILES = frozenset( - { - "test_gpt4o_audio.py", - "test_nvidia_nim.py", - "test_openai.py", - "test_openai_o1.py", - "test_prompt_caching.py", - "test_text_completion_unit_tests.py", - "test_xai.py", - } -) -_VCR_AUTO_MARKER_SKIP_FILES = _RESPX_CONFLICTING_FILES | frozenset( - {"test_vcr_redis_persister.py"} -) +# Per-item respx detection (``apply_vcr_auto_marker_to_items``) handles +# the vast majority of respx-vs-vcrpy conflicts automatically. The only +# entry below is the persister's own unit-test file, which exercises +# ``save_cassette`` / ``load_cassette`` against fakeredis and must not +# itself run under a live cassette context. +_VCR_AUTO_MARKER_SKIP_FILES = frozenset({"test_vcr_redis_persister.py"}) # Tests that observe live cross-call provider state (e.g. prompt-cache # warm-up between two consecutive calls); replay can't reproduce that state. @@ -73,6 +66,7 @@ def pytest_runtest_makereport(item, call): @pytest.fixture(autouse=True) def _vcr_outcome_gate(request, vcr): + install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) @@ -85,6 +79,11 @@ def pytest_runtest_logreport(report): _verbose_state.maybe_emit_verdict(report) +def pytest_terminal_summary(terminalreporter, exitstatus, config): + emit_cassette_cache_session_banner(terminalreporter) + emit_vcr_classification_summary(terminalreporter) + + # --------------------------------------------------------------------------- # Capture TRUE defaults at conftest import time (before test modules pollute). # --------------------------------------------------------------------------- diff --git a/tests/llm_translation/test_vcr_classification.py b/tests/llm_translation/test_vcr_classification.py new file mode 100644 index 0000000000..a73a44dff1 --- /dev/null +++ b/tests/llm_translation/test_vcr_classification.py @@ -0,0 +1,497 @@ +"""Unit tests for the VCR classification + observability layer. + +Covers: +- per-item respx detection (module scan, marker, fixture) +- skip-reason tagging in ``apply_vcr_auto_marker_to_items`` +- verdict classification (HIT / MISS:RECORDED / MISS:OVERFLOW / MISS:NOT_PERSISTED / + PARTIAL / NOOP / UNMARKED:LIVE_CALL / UNMARKED:NO_TRAFFIC) +- AWS SigV4 fingerprint stability +- session-end summary rendering +- live-call host classification +""" + +from __future__ import annotations + +import os +import sys +from types import SimpleNamespace +from typing import Optional + +import pytest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) + +from tests._vcr_conftest_common import ( # noqa: E402 + SKIP_REASON_FILE_OPT_OUT, + SKIP_REASON_INCOMPATIBLE, + SKIP_REASON_PRE_MARKED, + SKIP_REASON_RESPX, + SKIP_REASON_RESPX_MODULE, + VCR_SKIP_REASON_USER_ATTR, + VERDICT_HIT, + VERDICT_MISS_NOT_PERSISTED, + VERDICT_MISS_OVERFLOW, + VERDICT_MISS_RECORDED, + VERDICT_NOOP_NO_TRAFFIC, + VERDICT_PARTIAL, + VERDICT_UNMARKED_LIVE_CALL, + VERDICT_UNMARKED_NO_TRAFFIC, + _RESPX_MODULE_CACHE, + _classify_marked_test, + _compute_key_fingerprint, + _is_live_call_host, + _reset_session_stats, + _stable_key_value, + apply_vcr_auto_marker_to_items, + emit_vcr_classification_summary, + install_live_call_probe, + record_vcr_outcome, + session_stats_snapshot, +) + + +# --------------------------------------------------------------------------- +# Test doubles +# --------------------------------------------------------------------------- + + +class _StubItem: + """Pytest item double sufficient for the auto-marker logic.""" + + def __init__( + self, + nodeid: str, + path: str, + *, + markers: Optional[list[str]] = None, + fixturenames: Optional[list[str]] = None, + module=None, + ) -> None: + self.nodeid = nodeid + self.path = path + self._markers = list(markers or []) + self.fixturenames = list(fixturenames or []) + self.module = module + self.user_properties: list = [] + + def get_closest_marker(self, name: str): + return name if name in self._markers else None + + def add_marker(self, marker): + # ``pytest.mark.vcr`` is a MarkDecorator; rely on its ``name``. + name = getattr(marker, "name", str(marker)) + self._markers.append(name) + + +@pytest.fixture +def vcr_enabled(monkeypatch): + monkeypatch.setenv("CASSETTE_REDIS_URL", "redis://stub") + monkeypatch.delenv("LITELLM_VCR_DISABLE", raising=False) + monkeypatch.delenv("PYTEST_XDIST_WORKER", raising=False) + + +@pytest.fixture(autouse=True) +def _reset_module_caches(): + _reset_session_stats() + _RESPX_MODULE_CACHE.clear() + yield + _reset_session_stats() + _RESPX_MODULE_CACHE.clear() + + +# --------------------------------------------------------------------------- +# AWS SigV4 fingerprint stability — the Bedrock cassette overflow root cause +# --------------------------------------------------------------------------- + + +def test_should_extract_only_aws_access_key_from_sigv4_authorization(): + """Two Bedrock requests with the same access key but different + timestamps and signatures must produce the same fingerprint, otherwise + every CI run pushes a new episode into the cassette.""" + auth_today = ( + "AWS4-HMAC-SHA256 Credential=AKIAEXAMPLE12345/20260512/us-east-1/" + "bedrock/aws4_request, SignedHeaders=host;x-amz-date, " + "Signature=AAAAAAAA" + ) + auth_tomorrow = ( + "AWS4-HMAC-SHA256 Credential=AKIAEXAMPLE12345/20260513/us-east-1/" + "bedrock/aws4_request, SignedHeaders=host;x-amz-date, " + "Signature=BBBBBBBB" + ) + today = _stable_key_value("Authorization", auth_today) + tomorrow = _stable_key_value("Authorization", auth_tomorrow) + assert today == tomorrow == "aws-sigv4:AKIAEXAMPLE12345" + + +def test_should_keep_bearer_authorization_unchanged(): + """OpenAI ``Bearer `` headers are stable as-is — keep them.""" + out = _stable_key_value("Authorization", "Bearer sk-1234") + assert out == "Bearer sk-1234" + + +def test_should_produce_stable_fingerprint_across_sigv4_signatures(): + """``_compute_key_fingerprint`` should not change when only the SigV4 + signature/timestamp rotates.""" + req_a = SimpleNamespace( + headers={ + "authorization": ( + "AWS4-HMAC-SHA256 Credential=AKIA1/20260101/us-east-1/" + "bedrock/aws4_request, SignedHeaders=host, Signature=AAA" + ) + } + ) + req_b = SimpleNamespace( + headers={ + "authorization": ( + "AWS4-HMAC-SHA256 Credential=AKIA1/20260512/us-east-1/" + "bedrock/aws4_request, SignedHeaders=host;x-amz-date, " + "Signature=ZZZ" + ) + } + ) + assert _compute_key_fingerprint(req_a) == _compute_key_fingerprint(req_b) + + +def test_should_distinguish_different_aws_access_keys(): + """Two different access keys must produce different fingerprints so + cassettes recorded under one identity never serve another.""" + req_a = SimpleNamespace( + headers={ + "authorization": "AWS4-HMAC-SHA256 Credential=AKIAONE/x/y/z/aws4_request, Signature=A" + } + ) + req_b = SimpleNamespace( + headers={ + "authorization": "AWS4-HMAC-SHA256 Credential=AKIATWO/x/y/z/aws4_request, Signature=A" + } + ) + assert _compute_key_fingerprint(req_a) != _compute_key_fingerprint(req_b) + + +# --------------------------------------------------------------------------- +# Live-call host classification +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "host,expected", + [ + ("api.openai.com", True), + ("api.anthropic.com", True), + ("bedrock-runtime.us-east-1.amazonaws.com", False), + ("api.us-east-1.bedrock-runtime.amazonaws.com", True), + ("foo.bar.openai.com", True), + ("127.0.0.1", False), + ("localhost", False), + ("10.0.0.1", False), + ("172.16.0.1", False), + ("redis.example.com", False), + ("", False), + ], +) +def test_should_classify_live_call_hosts(host, expected): + assert _is_live_call_host(host) is expected + + +# --------------------------------------------------------------------------- +# Verdict classification +# --------------------------------------------------------------------------- + + +def _cassette(played: int, dirty: bool, total: int): + class _Sized: + def __init__(self, n): + self.n = n + self.play_count = played + self.dirty = dirty + + def __len__(self): + return self.n + + return _Sized(total) + + +def test_should_classify_pure_replay_as_hit(): + assert ( + _classify_marked_test(_cassette(played=3, dirty=False, total=3)) == VERDICT_HIT + ) + + +def test_should_classify_no_traffic_as_noop(): + assert ( + _classify_marked_test(_cassette(played=0, dirty=False, total=0)) + == VERDICT_NOOP_NO_TRAFFIC + ) + + +def test_should_classify_pure_record_as_miss_recorded(): + assert ( + _classify_marked_test(_cassette(played=0, dirty=True, total=1)) + == VERDICT_MISS_RECORDED + ) + + +def test_should_classify_mixed_replay_and_record_as_partial(): + assert ( + _classify_marked_test(_cassette(played=2, dirty=True, total=4)) + == VERDICT_PARTIAL + ) + + +def test_should_classify_overflow_as_miss_overflow_regardless_of_play_state(): + """Cassettes that exceed ``MAX_EPISODES_PER_CASSETTE`` (50) are + refused for save — they will hit live every CI run, so the verdict + must override HIT/PARTIAL classification.""" + assert ( + _classify_marked_test(_cassette(played=0, dirty=True, total=51)) + == VERDICT_MISS_OVERFLOW + ) + assert ( + _classify_marked_test(_cassette(played=10, dirty=True, total=52)) + == VERDICT_MISS_OVERFLOW + ) + + +# --------------------------------------------------------------------------- +# apply_vcr_auto_marker_to_items: skip-reason tagging +# --------------------------------------------------------------------------- + + +def _make_module_with_source(tmp_path, src: str, name: str): + p = tmp_path / f"{name}.py" + p.write_text(src) + mod = SimpleNamespace(__file__=str(p)) + return mod, str(p) + + +def test_should_apply_vcr_marker_to_clean_test(vcr_enabled, tmp_path): + mod, p = _make_module_with_source(tmp_path, "def test_x(): pass\n", "clean") + item = _StubItem("clean.py::test_x", p, module=mod) + apply_vcr_auto_marker_to_items([item]) + assert item.get_closest_marker("vcr") == "vcr" + + +def test_should_skip_per_item_when_respx_marker_present(vcr_enabled, tmp_path): + mod, p = _make_module_with_source(tmp_path, "def test_x(): pass\n", "respx_marker") + item = _StubItem("respx_marker.py::test_x", p, markers=["respx"], module=mod) + apply_vcr_auto_marker_to_items([item]) + assert item.get_closest_marker("vcr") is None + assert getattr(item, VCR_SKIP_REASON_USER_ATTR) == SKIP_REASON_RESPX + + +def test_should_skip_per_item_when_respx_mock_fixture_present(vcr_enabled, tmp_path): + mod, p = _make_module_with_source(tmp_path, "def test_x(): pass\n", "respx_fixture") + item = _StubItem( + "respx_fixture.py::test_x", p, fixturenames=["respx_mock"], module=mod + ) + apply_vcr_auto_marker_to_items([item]) + assert item.get_closest_marker("vcr") is None + assert getattr(item, VCR_SKIP_REASON_USER_ATTR) == SKIP_REASON_RESPX + + +def test_should_tag_pre_marked_items_so_summary_can_show_them(vcr_enabled, tmp_path): + mod, p = _make_module_with_source(tmp_path, "def test_x(): pass\n", "premarked") + item = _StubItem("premarked.py::test_x", p, markers=["vcr"], module=mod) + apply_vcr_auto_marker_to_items([item]) + assert getattr(item, VCR_SKIP_REASON_USER_ATTR) == SKIP_REASON_PRE_MARKED + + +def test_should_tag_skip_files_with_respx_module_when_module_actually_uses_respx( + vcr_enabled, tmp_path +): + """A file in ``skip_files`` whose module *does* call respx should be + labeled as a real conflict (respx_conflict_module), not a dead opt-out.""" + mod, p = _make_module_with_source( + tmp_path, + "import respx\n@pytest.mark.respx\ndef test_x(): pass\n", + "real_respx", + ) + item = _StubItem("real_respx.py::test_x", p, module=mod) + apply_vcr_auto_marker_to_items([item], skip_files={"real_respx.py"}) + assert getattr(item, VCR_SKIP_REASON_USER_ATTR) == SKIP_REASON_RESPX_MODULE + + +def test_should_tag_skip_files_with_file_opt_out_when_module_does_not_use_respx( + vcr_enabled, tmp_path +): + """A file in ``skip_files`` whose module never wires up respx is a + dead skip-list entry — surface it so we can prune.""" + mod, p = _make_module_with_source( + tmp_path, + "from respx import MockRouter # dead import\ndef test_x(): pass\n", + "dead_skip", + ) + item = _StubItem("dead_skip.py::test_x", p, module=mod) + apply_vcr_auto_marker_to_items([item], skip_files={"dead_skip.py"}) + assert getattr(item, VCR_SKIP_REASON_USER_ATTR) == SKIP_REASON_FILE_OPT_OUT + + +def test_should_tag_nodeid_suffix_skips_as_incompatible(vcr_enabled, tmp_path): + mod, p = _make_module_with_source(tmp_path, "def test_x(): pass\n", "incompat") + item = _StubItem("incompat.py::test_prompt_caching", p, module=mod) + apply_vcr_auto_marker_to_items( + [item], skip_nodeid_suffixes=("::test_prompt_caching",) + ) + assert getattr(item, VCR_SKIP_REASON_USER_ATTR) == SKIP_REASON_INCOMPATIBLE + + +# --------------------------------------------------------------------------- +# Session-end summary +# --------------------------------------------------------------------------- + + +class _FakeReporter: + def __init__(self): + self.lines: list[str] = [] + + def write_sep(self, sep, title="", **kwargs): + self.lines.append(f"=== {title}" if title else "===") + + def write_line(self, line): + self.lines.append(line) + + @property + def output(self): + return "\n".join(self.lines) + + +def test_should_render_overflow_section_when_any_test_overflowed(vcr_enabled): + """The OVERFLOW section is the cost-leak signal: if it's empty, no + cassettes are silently being refused; if it's not empty, those tests + re-bill on every run.""" + request = SimpleNamespace( + node=SimpleNamespace( + nodeid="t::overflow", + user_properties=[], + rep_call=SimpleNamespace(passed=True), + ) + ) + cassette = _cassette(played=0, dirty=True, total=51) + cassette._path = None # avoid mark_test_outcome side-effects + record_vcr_outcome(request, cassette) + + reporter = _FakeReporter() + emit_vcr_classification_summary(reporter) + assert "VCR CACHE CLASSIFICATION SUMMARY" in reporter.output + assert "VCR MISS:OVERFLOW" in reporter.output + assert "CASSETTE OVERFLOW" in reporter.output + assert "t::overflow" in reporter.output + + +def test_should_render_unmarked_live_call_section_with_hosts(vcr_enabled): + request_node = SimpleNamespace( + nodeid="t::leak", + user_properties=[], + rep_call=SimpleNamespace(passed=True), + ) + setattr(request_node, VCR_SKIP_REASON_USER_ATTR, SKIP_REASON_RESPX) + setattr(request_node, "vcr_live_call_hosts", ["api.openai.com"]) + request = SimpleNamespace(node=request_node) + + record_vcr_outcome(request, None) + + snap = session_stats_snapshot() + assert snap["unmarked_live_call_tests"] == [("t::leak", ["api.openai.com"])] + assert snap["verdict_counts"][VERDICT_UNMARKED_LIVE_CALL] == 1 + + reporter = _FakeReporter() + emit_vcr_classification_summary(reporter) + assert "UNMARKED TESTS WITH LIVE API CALLS" in reporter.output + assert "api.openai.com" in reporter.output + assert "t::leak" in reporter.output + + +def test_should_record_unmarked_no_traffic_when_test_skipped_vcr_but_did_not_call_out( + vcr_enabled, +): + request_node = SimpleNamespace( + nodeid="t::clean_skip", + user_properties=[], + rep_call=SimpleNamespace(passed=True), + ) + setattr(request_node, VCR_SKIP_REASON_USER_ATTR, SKIP_REASON_INCOMPATIBLE) + request = SimpleNamespace(node=request_node) + + record_vcr_outcome(request, None) + + snap = session_stats_snapshot() + assert snap["verdict_counts"][VERDICT_UNMARKED_NO_TRAFFIC] == 1 + assert snap["skip_reason_counts"][SKIP_REASON_INCOMPATIBLE] == 1 + + +def test_should_demote_miss_recorded_to_not_persisted_when_test_failed(vcr_enabled): + """If a test failed, ``save_cassette`` skips persisting — that means + the next CI run will hit live again. The verdict must reflect that.""" + request = SimpleNamespace( + node=SimpleNamespace( + nodeid="t::failed", + user_properties=[], + rep_call=SimpleNamespace(passed=False), + ) + ) + cassette = _cassette(played=0, dirty=True, total=1) + cassette._path = None + record_vcr_outcome(request, cassette) + + snap = session_stats_snapshot() + assert snap["verdict_counts"].get(VERDICT_MISS_NOT_PERSISTED) == 1 + + +def test_should_emit_no_summary_when_no_tests_observed(vcr_enabled): + reporter = _FakeReporter() + emit_vcr_classification_summary(reporter) + assert reporter.output == "" + + +# --------------------------------------------------------------------------- +# Live-call probe +# --------------------------------------------------------------------------- + + +def test_should_skip_live_probe_when_vcr_active(vcr_enabled): + """When the test *is* VCR-marked (cassette truthy), we don't install + the probe — vcrpy intercepts above the socket layer, so any + 'connection' would be vcrpy's own bookkeeping and not real spend.""" + request = SimpleNamespace(node=SimpleNamespace(), addfinalizer=lambda fn: None) + fake_cassette = SimpleNamespace(play_count=0, dirty=False) + probe = install_live_call_probe(request, fake_cassette) + assert probe is None + + +def test_live_call_probe_records_known_llm_hosts(vcr_enabled, monkeypatch): + """The probe should record outbound TCP connections to known LLM + provider hosts (and ignore localhost / RFC1918 / unknown hosts).""" + finalizers = [] + + class _Node: + pass + + request = SimpleNamespace( + node=_Node(), addfinalizer=lambda fn: finalizers.append(fn) + ) + probe = install_live_call_probe(request, None) + assert probe is not None + + import socket + + # Manually invoke the patched function — we don't actually open a + # connection because that would hit the network. The probe records + # at the *call site* before delegating, and the original + # ``socket.create_connection`` will then fail; we swallow that. + try: + socket.create_connection(("api.openai.com", 443), timeout=0.001) + except Exception: + pass + try: + socket.create_connection(("127.0.0.1", 6379), timeout=0.001) + except Exception: + pass + + # Restore via finalizers before asserting so the rest of the test + # session is unaffected. + for fn in finalizers: + fn() + + hosts = getattr(request.node, "vcr_live_call_hosts", []) + assert "api.openai.com" in hosts + assert "127.0.0.1" not in hosts diff --git a/tests/local_testing/conftest.py b/tests/local_testing/conftest.py index cad27869ad..0ff7dff668 100644 --- a/tests/local_testing/conftest.py +++ b/tests/local_testing/conftest.py @@ -25,20 +25,21 @@ import litellm from tests._vcr_conftest_common import ( # noqa: E402 VerboseReporterState, apply_vcr_auto_marker_to_items, + emit_cassette_cache_session_banner, + emit_vcr_classification_summary, + install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, vcr_config_dict, ) -# vcrpy and respx both patch the httpx transport — applying both makes one -# silently win, so respx-using files opt out of the auto-marker. -_RESPX_CONFLICTING_FILES = frozenset( - { - "test_router.py", - "test_amazing_vertex_completion.py", - "test_azure_openai.py", - } -) +# Per-item respx detection (``apply_vcr_auto_marker_to_items``) auto-skips +# tests whose ``@pytest.mark.respx`` marker or ``respx_mock`` fixture +# would conflict with vcrpy's transport patch. We no longer maintain a +# file-level ``_RESPX_CONFLICTING_FILES`` list here — the previous +# entries (``test_router.py``) had only a stale ``from respx import +# MockRouter`` import with no actual respx wiring, so file-level +# blacklisting was masking valid cache opportunities. # Files where VCR replay breaks the test: # - ``test_assistants.py``: polls fresh per-session run IDs that no cassette @@ -76,6 +77,7 @@ def pytest_runtest_makereport(item, call): @pytest.fixture(autouse=True) def _vcr_outcome_gate(request, vcr): + install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) @@ -88,6 +90,11 @@ def pytest_runtest_logreport(report): _verbose_state.maybe_emit_verdict(report) +def pytest_terminal_summary(terminalreporter, exitstatus, config): + emit_cassette_cache_session_banner(terminalreporter) + emit_vcr_classification_summary(terminalreporter) + + # --------------------------------------------------------------------------- # Capture TRUE defaults at conftest import time. This runs before any test # module's top-level code (e.g. `litellm.num_retries = 3`) executes, so @@ -215,7 +222,7 @@ def setup_and_teardown(): def pytest_collection_modifyitems(config, items): apply_vcr_auto_marker_to_items( items, - skip_files=_RESPX_CONFLICTING_FILES | _VCR_INCOMPATIBLE_FILES, + skip_files=_VCR_INCOMPATIBLE_FILES, skip_nodeid_suffixes=_VCR_INCOMPATIBLE_NODEID_SUFFIXES, ) diff --git a/tests/logging_callback_tests/conftest.py b/tests/logging_callback_tests/conftest.py index 7042d6094d..cdb9200bc8 100644 --- a/tests/logging_callback_tests/conftest.py +++ b/tests/logging_callback_tests/conftest.py @@ -22,6 +22,9 @@ import litellm from tests._vcr_conftest_common import ( # noqa: E402 VerboseReporterState, apply_vcr_auto_marker_to_items, + emit_cassette_cache_session_banner, + emit_vcr_classification_summary, + install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, vcr_config_dict, @@ -69,6 +72,7 @@ def pytest_runtest_makereport(item, call): @pytest.fixture(autouse=True) def _vcr_outcome_gate(request, vcr): + install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) @@ -220,3 +224,8 @@ def pytest_collection_modifyitems(config, items): # Reorder the items list items[:] = custom_logger_tests + other_tests + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + emit_cassette_cache_session_banner(terminalreporter) + emit_vcr_classification_summary(terminalreporter) diff --git a/tests/ocr_tests/conftest.py b/tests/ocr_tests/conftest.py index db48e2db2a..66970b8579 100644 --- a/tests/ocr_tests/conftest.py +++ b/tests/ocr_tests/conftest.py @@ -15,6 +15,9 @@ sys.path.insert(0, os.path.abspath("../..")) from tests._vcr_conftest_common import ( # noqa: E402 VerboseReporterState, apply_vcr_auto_marker_to_items, + emit_cassette_cache_session_banner, + emit_vcr_classification_summary, + install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, vcr_config_dict, @@ -41,6 +44,7 @@ def pytest_runtest_makereport(item, call): @pytest.fixture(autouse=True) def _vcr_outcome_gate(request, vcr): + install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) @@ -55,3 +59,8 @@ def pytest_runtest_logreport(report): def pytest_collection_modifyitems(config, items): apply_vcr_auto_marker_to_items(items) + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + emit_cassette_cache_session_banner(terminalreporter) + emit_vcr_classification_summary(terminalreporter) diff --git a/tests/pass_through_unit_tests/conftest.py b/tests/pass_through_unit_tests/conftest.py index d07057a4b6..ff47853d49 100644 --- a/tests/pass_through_unit_tests/conftest.py +++ b/tests/pass_through_unit_tests/conftest.py @@ -8,6 +8,9 @@ sys.path.insert(0, os.path.abspath("../..")) from tests._vcr_conftest_common import ( # noqa: E402 VerboseReporterState, apply_vcr_auto_marker_to_items, + emit_cassette_cache_session_banner, + emit_vcr_classification_summary, + install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, vcr_config_dict, @@ -34,6 +37,7 @@ def pytest_runtest_makereport(item, call): @pytest.fixture(autouse=True) def _vcr_outcome_gate(request, vcr): + install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) @@ -48,3 +52,8 @@ def pytest_runtest_logreport(report): def pytest_collection_modifyitems(config, items): apply_vcr_auto_marker_to_items(items) + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + emit_cassette_cache_session_banner(terminalreporter) + emit_vcr_classification_summary(terminalreporter) diff --git a/tests/router_unit_tests/conftest.py b/tests/router_unit_tests/conftest.py index a210244b3d..fe976515c9 100644 --- a/tests/router_unit_tests/conftest.py +++ b/tests/router_unit_tests/conftest.py @@ -15,6 +15,9 @@ import litellm # noqa: E402,F401 from tests._vcr_conftest_common import ( # noqa: E402 VerboseReporterState, apply_vcr_auto_marker_to_items, + emit_cassette_cache_session_banner, + emit_vcr_classification_summary, + install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, vcr_config_dict, @@ -87,6 +90,7 @@ def pytest_runtest_makereport(item, call): @pytest.fixture(autouse=True) def _vcr_outcome_gate(request, vcr): + install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) @@ -114,3 +118,8 @@ def pytest_collection_modifyitems(config, items): # Reorder the items list items[:] = custom_logger_tests + other_tests + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + emit_cassette_cache_session_banner(terminalreporter) + emit_vcr_classification_summary(terminalreporter) diff --git a/tests/search_tests/conftest.py b/tests/search_tests/conftest.py index 3b4623c53a..e06d3e95ee 100644 --- a/tests/search_tests/conftest.py +++ b/tests/search_tests/conftest.py @@ -16,6 +16,9 @@ sys.path.insert(0, os.path.abspath("../..")) from tests._vcr_conftest_common import ( # noqa: E402 VerboseReporterState, apply_vcr_auto_marker_to_items, + emit_cassette_cache_session_banner, + emit_vcr_classification_summary, + install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, vcr_config_dict, @@ -42,6 +45,7 @@ def pytest_runtest_makereport(item, call): @pytest.fixture(autouse=True) def _vcr_outcome_gate(request, vcr): + install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) @@ -56,3 +60,8 @@ def pytest_runtest_logreport(report): def pytest_collection_modifyitems(config, items): apply_vcr_auto_marker_to_items(items) + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + emit_cassette_cache_session_banner(terminalreporter) + emit_vcr_classification_summary(terminalreporter) diff --git a/tests/unified_google_tests/conftest.py b/tests/unified_google_tests/conftest.py index bae5769ad3..d28f89a77b 100644 --- a/tests/unified_google_tests/conftest.py +++ b/tests/unified_google_tests/conftest.py @@ -15,6 +15,9 @@ import litellm # noqa: E402,F401 from tests._vcr_conftest_common import ( # noqa: E402 VerboseReporterState, apply_vcr_auto_marker_to_items, + emit_cassette_cache_session_banner, + emit_vcr_classification_summary, + install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, vcr_config_dict, @@ -74,6 +77,7 @@ def pytest_runtest_makereport(item, call): @pytest.fixture(autouse=True) def _vcr_outcome_gate(request, vcr): + install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) @@ -101,3 +105,8 @@ def pytest_collection_modifyitems(config, items): # Reorder the items list items[:] = custom_logger_tests + other_tests + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + emit_cassette_cache_session_banner(terminalreporter) + emit_vcr_classification_summary(terminalreporter) From 93300019bd971a8dff7c7369e6b50233529de4b0 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 13 May 2026 00:32:03 +0000 Subject: [PATCH 02/11] test(vcr): drop dead 'from respx import MockRouter' imports These seven test files were on _RESPX_CONFLICTING_FILES, which made the auto-marker skip them entirely. Inspecting the source shows the only respx artifact is a top-level 'from respx import MockRouter' that no test ever uses - no @pytest.mark.respx, no respx_mock fixture, no respx.mock context manager. The import is dead code left over from a previous mocking pattern. Now that apply_vcr_auto_marker_to_items detects respx per-item via the marker / fixture chain (b637d9f64a), the file-level skip is no longer needed for these files - they were the reason the OpenAI tests (test_o3_reasoning_effort, test_streaming_response[o1/o3-mini], TestOpenAIO1::test_streaming, TestOpenAIChatCompletion::test_web_search, TestOpenAIO3::test_web_search, etc.) ran live every CI build despite the cassette cache being healthy. Co-authored-by: Mateo Wang --- tests/llm_translation/test_gpt4o_audio.py | 1 - tests/llm_translation/test_nvidia_nim.py | 1 - tests/llm_translation/test_openai.py | 1 - tests/llm_translation/test_openai_o1.py | 1 - tests/llm_translation/test_prompt_caching.py | 1 - tests/llm_translation/test_xai.py | 1 - tests/local_testing/test_router.py | 1 - 7 files changed, 7 deletions(-) diff --git a/tests/llm_translation/test_gpt4o_audio.py b/tests/llm_translation/test_gpt4o_audio.py index 4b70256335..169fe85516 100644 --- a/tests/llm_translation/test_gpt4o_audio.py +++ b/tests/llm_translation/test_gpt4o_audio.py @@ -11,7 +11,6 @@ sys.path.insert( import httpx import pytest -from respx import MockRouter import litellm from litellm import Choices, Message, ModelResponse diff --git a/tests/llm_translation/test_nvidia_nim.py b/tests/llm_translation/test_nvidia_nim.py index 72981665cb..469516407c 100644 --- a/tests/llm_translation/test_nvidia_nim.py +++ b/tests/llm_translation/test_nvidia_nim.py @@ -11,7 +11,6 @@ sys.path.insert( import httpx import pytest -from respx import MockRouter from unittest.mock import patch, MagicMock, AsyncMock import litellm diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index acbb9c5136..1fec7665da 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -12,7 +12,6 @@ sys.path.insert( import httpx import pytest -from respx import MockRouter import litellm from litellm import Choices, Message, ModelResponse diff --git a/tests/llm_translation/test_openai_o1.py b/tests/llm_translation/test_openai_o1.py index 0e4761bb4c..fccb1c6f1e 100644 --- a/tests/llm_translation/test_openai_o1.py +++ b/tests/llm_translation/test_openai_o1.py @@ -11,7 +11,6 @@ sys.path.insert( import httpx import pytest -from respx import MockRouter import litellm from litellm import Choices, Message, ModelResponse diff --git a/tests/llm_translation/test_prompt_caching.py b/tests/llm_translation/test_prompt_caching.py index e9d22074a3..eb4703fd67 100644 --- a/tests/llm_translation/test_prompt_caching.py +++ b/tests/llm_translation/test_prompt_caching.py @@ -11,7 +11,6 @@ sys.path.insert( import httpx import pytest -from respx import MockRouter import litellm from litellm import Choices, Message, ModelResponse diff --git a/tests/llm_translation/test_xai.py b/tests/llm_translation/test_xai.py index f908bb0959..f0945e6e16 100644 --- a/tests/llm_translation/test_xai.py +++ b/tests/llm_translation/test_xai.py @@ -11,7 +11,6 @@ sys.path.insert( import httpx import pytest -from respx import MockRouter import litellm from litellm import Choices, Message, ModelResponse, EmbeddingResponse, Usage diff --git a/tests/local_testing/test_router.py b/tests/local_testing/test_router.py index d6b239c79c..6d04e6ecaa 100644 --- a/tests/local_testing/test_router.py +++ b/tests/local_testing/test_router.py @@ -20,7 +20,6 @@ import os from collections import defaultdict from concurrent.futures import ThreadPoolExecutor from unittest.mock import AsyncMock, MagicMock, patch -from respx import MockRouter import httpx from dotenv import load_dotenv from pydantic import BaseModel From 656377bb3df4ccdc61f8a43ccc1e362cd7d8ddd1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 13 May 2026 00:32:23 +0000 Subject: [PATCH 03/11] test(image_edits): regenerate fixtures per call instead of holding open module-level file handles Module-level TEST_IMAGES = [ open(os.path.join(pwd, 'ishaan_github.png'), 'rb'), open(os.path.join(pwd, 'litellm_site.png'), 'rb'), ] SINGLE_TEST_IMAGE = open(...) opens the file once at import. After the first multipart upload, the file pointer is at EOF, so every subsequent test in the same xdist worker sends an empty multipart body. That non-determinism (a) blows the recorded cassette past MAX_EPISODES_PER_CASSETTE (50) so _RedisPersister.save_cassette refuses to save it, and (b) re-bills the live image edit endpoint on every CI run. Recent CI runs confirm the leak: tests/image_gen_tests/test_image_edits.py shows six tests parking at 51-52 cassette entries (TestOpenAIImageEditGPTImage1::test_openai_image_edit_litellm_sdk[False], TestOpenAIImageEditDallE2::..., test_openai_image_edit_with_bytesio, test_openai_image_edit_litellm_router, test_multiple_vs_single_image_edit[False], test_multiple_image_edit_with_different_formats). Replace the module-level file handles with _make_test_images() / _make_single_test_image() factories that return fresh _RewindableImage (BytesIO subclass) objects whose pointer always starts at 0. The image bytes are read once at import into module-level constants (_ISHAAN_GITHUB_BYTES, _LITELLM_SITE_BYTES), so disk I/O cost is unchanged. Co-authored-by: Mateo Wang --- tests/image_gen_tests/test_image_edits.py | 86 ++++++++++++++++------- 1 file changed, 60 insertions(+), 26 deletions(-) diff --git a/tests/image_gen_tests/test_image_edits.py b/tests/image_gen_tests/test_image_edits.py index 6900bacdb9..441511b014 100644 --- a/tests/image_gen_tests/test_image_edits.py +++ b/tests/image_gen_tests/test_image_edits.py @@ -102,22 +102,57 @@ class BaseLLMImageEditTest(ABC): # Get the current directory of the file being run pwd = os.path.dirname(os.path.realpath(__file__)) -TEST_IMAGES = [ - open(os.path.join(pwd, "ishaan_github.png"), "rb"), - open(os.path.join(pwd, "litellm_site.png"), "rb"), -] -SINGLE_TEST_IMAGE = open(os.path.join(pwd, "ishaan_github.png"), "rb") +# Image fixtures must be regenerated per access — module-level +# ``open(...)`` handles get consumed after a single multipart upload, leaving +# subsequent tests in the same process to send empty bodies. That non-determinism +# (a) blows the recorded cassette past ``MAX_EPISODES_PER_CASSETTE`` so the +# persister refuses to save (see ``tests/_vcr_redis_persister.py``), and +# (b) re-bills the live image edit endpoint on every CI run. +def _read_image_bytes(filename: str) -> bytes: + with open(os.path.join(pwd, filename), "rb") as f: + return f.read() + + +_ISHAAN_GITHUB_BYTES = _read_image_bytes("ishaan_github.png") +_LITELLM_SITE_BYTES = _read_image_bytes("litellm_site.png") + + +class _RewindableImage(BytesIO): + """``BytesIO`` that re-seeks to 0 on read after exhaustion. + + The OpenAI / Azure SDKs read the file pointer once per request. When we + pass the *same* object to a parametrized or retried test, the second + invocation must see the same bytes from offset 0, not EOF. + """ + + def read(self, size=-1): + if self.tell() and self.tell() >= len(self.getvalue()): + self.seek(0) + return super().read(size) + + +def _make_test_images() -> list: + """Return a fresh pair of rewindable image streams. + + Use this everywhere you'd previously have used the module-level + ``TEST_IMAGES``. Each call returns brand new ``BytesIO`` objects whose + file pointers start at 0, so multipart uploads encode the full image + bytes on every test invocation. + """ + return [ + _RewindableImage(_ISHAAN_GITHUB_BYTES), + _RewindableImage(_LITELLM_SITE_BYTES), + ] + + +def _make_single_test_image() -> BytesIO: + return _RewindableImage(_ISHAAN_GITHUB_BYTES) def get_test_images_as_bytesio(): """Helper function to get test images as BytesIO objects""" - bytesio_images = [] - for image_path in ["ishaan_github.png", "litellm_site.png"]: - with open(os.path.join(pwd, image_path), "rb") as f: - image_bytes = f.read() - bytesio_images.append(BytesIO(image_bytes)) - return bytesio_images + return _make_test_images() class TestOpenAIImageEditGPTImage1(BaseLLMImageEditTest): @@ -129,7 +164,7 @@ class TestOpenAIImageEditGPTImage1(BaseLLMImageEditTest): """Return base call args for OpenAI image edit""" return { "model": "gpt-image-1", - "image": TEST_IMAGES, + "image": _make_test_images(), } @@ -143,7 +178,7 @@ class TestAzureAIFlux2ImageEdit(BaseLLMImageEditTest): """Return base call args for Azure AI FLUX 2 image edit""" return { "model": "azure_ai/flux.2-pro", - "image": SINGLE_TEST_IMAGE, + "image": _make_single_test_image(), "api_base": os.getenv("AZURE_AI_API_BASE"), "api_key": os.getenv("AZURE_AI_API_KEY"), "api_version": "preview", @@ -171,7 +206,7 @@ async def test_openai_image_edit_litellm_router(): result = await router.aimage_edit( prompt=prompt, model="gpt-image-1", - image=TEST_IMAGES, + image=_make_test_images(), ) print("result from image edit", result) @@ -275,7 +310,7 @@ async def test_azure_image_edit_litellm_sdk(): api_base=test_api_base, api_key=test_api_key, api_version=test_api_version, - image=TEST_IMAGES, + image=_make_test_images(), ) # Verify the request was made correctly @@ -386,7 +421,7 @@ async def test_openai_image_edit_cost_tracking(): result = await aimage_edit( prompt=prompt, model="openai/gpt-image-1", - image=TEST_IMAGES, + image=_make_test_images(), ) # Verify the request was made correctly @@ -477,7 +512,7 @@ async def test_azure_image_edit_cost_tracking(): prompt=prompt, model="azure/CUSTOM_AZURE_DEPLOYMENT_NAME", base_model="azure/gpt-image-1", - image=TEST_IMAGES, + image=_make_test_images(), ) # Verify the request was made correctly @@ -525,7 +560,6 @@ async def test_recraft_image_edit_api(): import requests litellm._turn_on_debug() - global TEST_IMAGES try: prompt = """ Create a studio ghibli style image that combines all the reference images. Make sure the person looks like a CTO. @@ -533,7 +567,7 @@ async def test_recraft_image_edit_api(): result = await aimage_edit( prompt=prompt, model="recraft/recraftv3", - image=TEST_IMAGES, + image=_make_test_images(), ) print("result from image edit", result) @@ -631,13 +665,13 @@ async def test_multiple_vs_single_image_edit(sync_mode): single_result = image_edit( prompt=prompt, model="gpt-image-1", - image=SINGLE_TEST_IMAGE, + image=_make_single_test_image(), ) else: single_result = await aimage_edit( prompt=prompt, model="gpt-image-1", - image=SINGLE_TEST_IMAGE, + image=_make_single_test_image(), ) print("Single image result:", single_result) @@ -648,13 +682,13 @@ async def test_multiple_vs_single_image_edit(sync_mode): multiple_result = image_edit( prompt=prompt, model="gpt-image-1", - image=TEST_IMAGES, + image=_make_test_images(), ) else: multiple_result = await aimage_edit( prompt=prompt, model="gpt-image-1", - image=TEST_IMAGES, + image=_make_test_images(), ) print("Multiple images result:", multiple_result) @@ -685,7 +719,7 @@ async def test_multiple_image_edit_with_different_formats(): # Test with mixed BytesIO and file objects mixed_images = [ - SINGLE_TEST_IMAGE, # File object + _make_single_test_image(), # File object get_test_images_as_bytesio()[1], # BytesIO object ] @@ -749,14 +783,14 @@ async def test_image_edit_array_handling(): result1 = await aimage_edit( prompt=prompt, model="gpt-image-1", - image=SINGLE_TEST_IMAGE, + image=_make_single_test_image(), ) # Test 2: Multiple images (already a list) result2 = await aimage_edit( prompt=prompt, model="gpt-image-1", - image=TEST_IMAGES, + image=_make_test_images(), ) # Both valid calls should succeed From dd89b9304f4b963688a4787a5777a3a0ff336a05 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 13 May 2026 00:41:49 +0000 Subject: [PATCH 04/11] fix(vcr): match real Bedrock hostnames in live-call probe The suffix '.bedrock-runtime.amazonaws.com' never matched real Bedrock endpoints, which use the format 'bedrock-runtime[-fips].{region}.amazonaws.com' (region between 'bedrock-runtime' and 'amazonaws.com'). Add an explicit host check for that pattern so Bedrock live calls are visible to the probe, and update the unit test accordingly. Also drop the unused '_LIVE_CALL_PROBE_INSTALLED' module variable. --- tests/_vcr_conftest_common.py | 13 ++++++++++--- tests/llm_translation/test_vcr_classification.py | 5 +++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 2bdddb69d0..0b9b0bcd75 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -514,7 +514,6 @@ _LIVE_CALL_HOST_SUFFIXES = ( ".vertexai.googleapis.com", ".aiplatform.googleapis.com", ".googleapis.com", - ".bedrock-runtime.amazonaws.com", ".x.ai", ".cohere.ai", ".cohere.com", @@ -758,7 +757,6 @@ def _format_verdict_line(verdict: str, cassette, extra: str = "") -> str: # "confirmed: this test connected to host X". # --------------------------------------------------------------------------- -_LIVE_CALL_PROBE_INSTALLED = False _LIVE_CALL_BUFFER_KEY = "vcr_live_call_hosts" @@ -768,7 +766,16 @@ def _is_live_call_host(host: str) -> bool: host = host.lower() if any(host.startswith(p) for p in _LIVE_CALL_LOCAL_PREFIXES): return False - return any(host.endswith(suffix) for suffix in _LIVE_CALL_HOST_SUFFIXES) + if any(host.endswith(suffix) for suffix in _LIVE_CALL_HOST_SUFFIXES): + return True + # AWS Bedrock endpoints are ``bedrock-runtime[-fips].{region}.amazonaws.com`` + # (region between ``bedrock-runtime`` and ``amazonaws.com``), so plain + # suffix matching can't catch them. + if host.endswith(".amazonaws.com") and host.split(".", 1)[0].startswith( + "bedrock-runtime" + ): + return True + return False class _LiveCallProbe: diff --git a/tests/llm_translation/test_vcr_classification.py b/tests/llm_translation/test_vcr_classification.py index a73a44dff1..3b08ef5d37 100644 --- a/tests/llm_translation/test_vcr_classification.py +++ b/tests/llm_translation/test_vcr_classification.py @@ -178,8 +178,9 @@ def test_should_distinguish_different_aws_access_keys(): [ ("api.openai.com", True), ("api.anthropic.com", True), - ("bedrock-runtime.us-east-1.amazonaws.com", False), - ("api.us-east-1.bedrock-runtime.amazonaws.com", True), + ("bedrock-runtime.us-east-1.amazonaws.com", True), + ("bedrock-runtime-fips.us-east-1.amazonaws.com", True), + ("api.us-east-1.bedrock-runtime.amazonaws.com", False), ("foo.bar.openai.com", True), ("127.0.0.1", False), ("localhost", False), From ba0f4c1566b056bffaa8e4d5d24dac18e181ec8a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 13 May 2026 00:47:44 +0000 Subject: [PATCH 05/11] fix(vcr): cover full RFC1918 172.16.0.0/12 range in local prefixes --- tests/_vcr_conftest_common.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 0b9b0bcd75..f90cfa600a 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -540,6 +540,17 @@ _LIVE_CALL_LOCAL_PREFIXES = ( "172.18.", "172.19.", "172.20.", + "172.21.", + "172.22.", + "172.23.", + "172.24.", + "172.25.", + "172.26.", + "172.27.", + "172.28.", + "172.29.", + "172.30.", + "172.31.", "192.168.", ) From b13ae8de507619cc2d0103416c2feed889f8031c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 13 May 2026 01:04:24 +0000 Subject: [PATCH 06/11] fix(image_edits): drop _RewindableImage to prevent infinite multipart upload The _RewindableImage(BytesIO) wrapper auto-rewound on every read after EOF, which made the OpenAI SDK's multipart upload writer read the same bytes forever instead of seeing EOF. Workers OOM'd / SIGKILL'd: [gw0] node down: Not properly terminated replacing crashed worker gw0 ... worker 'gw1' crashed while running 'tests/image_gen_tests/test_image_edits.py::TestOpenAIImageEditGPTImage1::test_openai_image_edit_litellm_sdk[False]' The auto-rewind was added defensively for parametrized + flaky-retried tests, but BaseLLMImageEditTest::test_openai_image_edit_litellm_sdk already calls get_base_image_edit_call_args() once per invocation and that helper now constructs fresh streams via _make_test_images(), so rewinding inside the stream is unnecessary. Replace with plain BytesIO seeded with the cached image bytes. Co-authored-by: Mateo Wang --- tests/image_gen_tests/test_image_edits.py | 28 ++++++++--------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/tests/image_gen_tests/test_image_edits.py b/tests/image_gen_tests/test_image_edits.py index 441511b014..e11250fc51 100644 --- a/tests/image_gen_tests/test_image_edits.py +++ b/tests/image_gen_tests/test_image_edits.py @@ -118,36 +118,26 @@ _ISHAAN_GITHUB_BYTES = _read_image_bytes("ishaan_github.png") _LITELLM_SITE_BYTES = _read_image_bytes("litellm_site.png") -class _RewindableImage(BytesIO): - """``BytesIO`` that re-seeks to 0 on read after exhaustion. - - The OpenAI / Azure SDKs read the file pointer once per request. When we - pass the *same* object to a parametrized or retried test, the second - invocation must see the same bytes from offset 0, not EOF. - """ - - def read(self, size=-1): - if self.tell() and self.tell() >= len(self.getvalue()): - self.seek(0) - return super().read(size) - - def _make_test_images() -> list: - """Return a fresh pair of rewindable image streams. + """Return a fresh pair of image streams seeded with the fixture bytes. Use this everywhere you'd previously have used the module-level ``TEST_IMAGES``. Each call returns brand new ``BytesIO`` objects whose file pointers start at 0, so multipart uploads encode the full image - bytes on every test invocation. + bytes on every test invocation. Parametrized and ``flaky``-retried + test methods call ``get_base_image_edit_call_args`` once per + invocation, so a fresh stream per call is sufficient — the factory + must not auto-rewind on EOF or the SDK's multipart writer will read + the same bytes forever (worker OOM). """ return [ - _RewindableImage(_ISHAAN_GITHUB_BYTES), - _RewindableImage(_LITELLM_SITE_BYTES), + BytesIO(_ISHAAN_GITHUB_BYTES), + BytesIO(_LITELLM_SITE_BYTES), ] def _make_single_test_image() -> BytesIO: - return _RewindableImage(_ISHAAN_GITHUB_BYTES) + return BytesIO(_ISHAAN_GITHUB_BYTES) def get_test_images_as_bytesio(): From 3390fc19722b1bc8b845390285566eb796c9a406 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 13 May 2026 01:19:03 +0000 Subject: [PATCH 07/11] test(vcr): mark Bedrock prompt-caching cross-call tests VCR-incompatible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pass_through prompt-caching tests (test_prompt_caching_returns_cache_read_tokens_on_second_call, test_prompt_caching_streaming_second_call_returns_cache_read) make a warm-up call and then assert the *second* call sees a non-zero cache_read_input_tokens count from the upstream's prompt-cache. VCR replay can't model cross-call provider state — both calls match the same cassette episode, so the second call returns the first call's pre-warmup response and the assertion fails: AssertionError: Expected cache_read_input_tokens > 0 on second call, but got 0. Full usage: {'input_tokens': 4986, 'cache_creation_input_tokens': 4974, 'cache_read_input_tokens': 0} This started biting after the AWS SigV4 fingerprint stabilization (b637d9f64a): Bedrock requests now produce a stable per-access-key fingerprint instead of a per-request signature, so cassettes successfully replay where they previously always missed and re-recorded live. Opt these tests out via skip_nodeid_suffixes so they run live and match the existing pattern in tests/llm_translation/conftest.py (::test_prompt_caching). Co-authored-by: Mateo Wang --- tests/pass_through_unit_tests/conftest.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/pass_through_unit_tests/conftest.py b/tests/pass_through_unit_tests/conftest.py index ff47853d49..42a95343eb 100644 --- a/tests/pass_through_unit_tests/conftest.py +++ b/tests/pass_through_unit_tests/conftest.py @@ -16,6 +16,18 @@ from tests._vcr_conftest_common import ( # noqa: E402 vcr_config_dict, ) +# Tests that observe live cross-call provider state — typically a +# warm-up call followed by an assertion that the *second* call sees the +# upstream's prompt-cache (Anthropic / Bedrock prompt-caching). VCR's +# deterministic replay can't model this: both calls match the same +# cassette episode, so the second call returns the first call's +# pre-warmup response. Opt these out so they run live (no caching). +_VCR_INCOMPATIBLE_NODEID_SUFFIXES = ( + "::test_prompt_caching_returns_cache_read_tokens_on_second_call", + "::test_prompt_caching_streaming_second_call_returns_cache_read", +) + + _verbose_state = VerboseReporterState() @@ -51,7 +63,9 @@ def pytest_runtest_logreport(report): def pytest_collection_modifyitems(config, items): - apply_vcr_auto_marker_to_items(items) + apply_vcr_auto_marker_to_items( + items, skip_nodeid_suffixes=_VCR_INCOMPATIBLE_NODEID_SUFFIXES + ) def pytest_terminal_summary(terminalreporter, exitstatus, config): From a82c240ab0099a6369af205423ca91b49fd5154d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 13 May 2026 01:46:25 +0000 Subject: [PATCH 08/11] test(vcr): tighten OVERFLOW classification and switch respx detection to AST MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address two greptile P2 review concerns on PR #27795: 1. MISS:OVERFLOW was firing whenever total > MAX_EPISODES_PER_CASSETTE regardless of cassette state. A cassette that grew past the cap historically but this run only *replayed* (dirty=False) is healthy — the persister never tries to save, so the cache state is stable and the next run will replay too. Only flag OVERFLOW when dirty=True (new episodes were recorded that the persister would refuse to save). Add a regression test covering the dirty=False + large-total case. 2. _module_uses_respx did substring matching on the module source, which false-positives on comments / docstrings / string literals. A comment like # Previously tried respx.mock but switched to vcrpy would keep a file pinned on the opt-out list, defeating the dead-import pruning goal of this PR. Replace the substring scan with an ast.NodeVisitor (_RespxUsageVisitor) that only counts: - @pytest.mark.respx / @respx.mock decorators - with respx.mock(): ... (sync + async) context managers - respx.mock(...) calls outside a with/decorator - function parameters / fixture names equal to respx_mock Add tests for the comment / docstring / string-literal cases plus each real-usage pattern. Co-authored-by: Mateo Wang --- tests/_vcr_conftest_common.py | 149 ++++++++++++++++-- .../test_vcr_classification.py | 66 +++++++- 2 files changed, 196 insertions(+), 19 deletions(-) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index f90cfa600a..5cc544187b 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -5,6 +5,7 @@ See ``tests/llm_translation/Readme.md`` for the full design and from __future__ import annotations +import ast import atexit import hashlib import json @@ -555,18 +556,126 @@ _LIVE_CALL_LOCAL_PREFIXES = ( ) +class _RespxUsageVisitor(ast.NodeVisitor): + """AST visitor that flags real respx wiring in a test module. + + Substring scans of the source text are unreliable: a comment like + ``# Previously used respx.mock`` or a docstring referencing respx + would falsely flag the module. We only count: + + * ``@pytest.mark.respx`` / ``@respx.mock`` decorators + * ``with respx.mock(): ...`` context managers + * ``respx.mock(...)`` / ``respx.mock`` attribute access + * function parameters / fixture arguments named ``respx_mock`` + """ + + def __init__(self) -> None: + self.uses_respx = False + + def _decorator_is_respx(self, dec: ast.expr) -> bool: + # ``@respx.mock`` (Attribute) or ``@respx.mock(...)`` (Call wrapping Attribute) + if isinstance(dec, ast.Call): + dec = dec.func + if isinstance(dec, ast.Attribute): + return ( + isinstance(dec.value, ast.Name) + and dec.value.id == "respx" + and dec.attr == "mock" + ) + return False + + def _is_pytest_mark_respx(self, dec: ast.expr) -> bool: + # ``@pytest.mark.respx`` or ``@pytest.mark.respx(...)``. + if isinstance(dec, ast.Call): + dec = dec.func + if ( + isinstance(dec, ast.Attribute) + and dec.attr == "respx" + and isinstance(dec.value, ast.Attribute) + and dec.value.attr == "mark" + and isinstance(dec.value.value, ast.Name) + and dec.value.value.id == "pytest" + ): + return True + return False + + def _check_decorators(self, decs: list[ast.expr]) -> None: + for d in decs: + if self._decorator_is_respx(d) or self._is_pytest_mark_respx(d): + self.uses_respx = True + + def _check_args(self, args: ast.arguments) -> None: + # ``def test_foo(respx_mock): ...`` — pytest supplies the fixture + # whenever the parameter name appears, regardless of marker. + all_args = ( + list(args.args) + + list(args.kwonlyargs) + + (list(args.posonlyargs) if hasattr(args, "posonlyargs") else []) + ) + for a in all_args: + if a.arg == "respx_mock": + self.uses_respx = True + return + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + self._check_decorators(node.decorator_list) + self._check_args(node.args) + self.generic_visit(node) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + self._check_decorators(node.decorator_list) + self._check_args(node.args) + self.generic_visit(node) + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + self._check_decorators(node.decorator_list) + self.generic_visit(node) + + def _is_respx_mock_attr(self, node: ast.expr) -> bool: + return ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "respx" + and node.attr == "mock" + ) + + def visit_With(self, node: ast.With) -> None: + for item in node.items: + ctx = item.context_expr + if isinstance(ctx, ast.Call): + ctx = ctx.func + if self._is_respx_mock_attr(ctx): + self.uses_respx = True + self.generic_visit(node) + + def visit_AsyncWith(self, node: ast.AsyncWith) -> None: + for item in node.items: + ctx = item.context_expr + if isinstance(ctx, ast.Call): + ctx = ctx.func + if self._is_respx_mock_attr(ctx): + self.uses_respx = True + self.generic_visit(node) + + def visit_Call(self, node: ast.Call) -> None: + # ``respx.mock(...)`` invocation outside a ``with``/decorator — + # e.g. ``mock = respx.mock()`` at module scope. + if self._is_respx_mock_attr(node.func): + self.uses_respx = True + self.generic_visit(node) + + def _module_uses_respx(item) -> bool: """Return True if the test's *module* actually wires up respx. - A bare ``from respx import MockRouter`` import (with no actual usage) - does not patch the httpx transport, so it does not conflict with vcrpy. - We confirm by checking the module's source for any of: - - ``@pytest.mark.respx`` - - ``@respx.mock`` / ``with respx.mock`` - - ``respx_mock`` fixture name + Uses an ``ast`` walk (not substring matching) so comments and + docstrings that mention respx don't count as real usage. A bare + ``from respx import MockRouter`` import with no other respx + references therefore won't flag the module — that's exactly the + dead-import case this PR is trying to surface. """ module = getattr(item, "module", None) - src_file = getattr(module, "__file__", None) + src_file = getattr(module, "__file__", None) or str(getattr(item, "path", "") or "") if not src_file or not os.path.isfile(src_file): return False try: @@ -574,13 +683,16 @@ def _module_uses_respx(item) -> bool: src = f.read() except OSError: return False - if "respx_mock" in src: - return True - if "@pytest.mark.respx" in src or "@respx.mock" in src: - return True - if "respx.mock" in src or "with respx" in src: - return True - return False + try: + tree = ast.parse(src, filename=src_file) + except SyntaxError: + # If the test file itself is broken, fall back to "no respx" — + # the test will fail collection on its own and we don't want + # the auto-marker to mask that with a misleading skip reason. + return False + visitor = _RespxUsageVisitor() + visitor.visit(tree) + return visitor.uses_respx def _item_uses_respx(item) -> bool: @@ -735,8 +847,13 @@ def _classify_marked_test(cassette) -> str: # "OVERFLOW" mirrors ``_RedisPersister.save_cassette``'s # ``> MAX_EPISODES_PER_CASSETTE`` guard. Cassettes that hit this # threshold are refused for save, so the test re-records live every - # run. - if total > MAX_EPISODES_PER_CASSETTE: + # run. Only flag when ``dirty=True`` — if a cassette grew past the + # cap historically but this run replayed it without adding new + # episodes, the persister never tries to save (no recording + # happened), so the cache state is stable and the next run will + # replay too. Flagging that case as OVERFLOW would tag healthy + # cached tests as cost leaks. + if total > MAX_EPISODES_PER_CASSETTE and dirty: return VERDICT_MISS_OVERFLOW if played == 0 and not dirty: return VERDICT_NOOP_NO_TRAFFIC diff --git a/tests/llm_translation/test_vcr_classification.py b/tests/llm_translation/test_vcr_classification.py index 3b08ef5d37..9ab0fedbbf 100644 --- a/tests/llm_translation/test_vcr_classification.py +++ b/tests/llm_translation/test_vcr_classification.py @@ -239,10 +239,13 @@ def test_should_classify_mixed_replay_and_record_as_partial(): ) -def test_should_classify_overflow_as_miss_overflow_regardless_of_play_state(): +def test_should_classify_overflow_only_when_dirty_episodes_were_recorded(): """Cassettes that exceed ``MAX_EPISODES_PER_CASSETTE`` (50) are - refused for save — they will hit live every CI run, so the verdict - must override HIT/PARTIAL classification.""" + refused for save — but only when ``dirty=True`` (new episodes were + actually recorded that the persister would refuse). Replaying an + already-large cassette with no new traffic is healthy: the persister + never tries to save, so the cache state is stable and the next run + will replay too.""" assert ( _classify_marked_test(_cassette(played=0, dirty=True, total=51)) == VERDICT_MISS_OVERFLOW @@ -253,6 +256,20 @@ def test_should_classify_overflow_as_miss_overflow_regardless_of_play_state(): ) +def test_should_classify_large_cassette_with_no_new_episodes_as_hit(): + """``total > 50`` + ``dirty=False`` means everything was replayed + from cache; no save attempt happens, so this is a healthy HIT, not + OVERFLOW.""" + assert ( + _classify_marked_test(_cassette(played=51, dirty=False, total=51)) + == VERDICT_HIT + ) + assert ( + _classify_marked_test(_cassette(played=60, dirty=False, total=60)) + == VERDICT_HIT + ) + + # --------------------------------------------------------------------------- # apply_vcr_auto_marker_to_items: skip-reason tagging # --------------------------------------------------------------------------- @@ -327,6 +344,49 @@ def test_should_tag_skip_files_with_file_opt_out_when_module_does_not_use_respx( assert getattr(item, VCR_SKIP_REASON_USER_ATTR) == SKIP_REASON_FILE_OPT_OUT +def test_should_not_flag_respx_mentioned_in_comment_or_docstring(vcr_enabled, tmp_path): + """Substring scans of source text false-positive on + ``# Previously used respx.mock`` and similar — defeats the dead + skip-list pruning goal. AST-based detection ignores comments and + string literals.""" + src = ( + '"""Module docstring mentions respx.mock and @pytest.mark.respx and respx_mock."""\n' + "# Previously tried respx.mock but switched to vcrpy\n" + "# Old code did `with respx.mock(): ...`\n" + "x = '@respx.mock' # string literal, not a real decorator\n" + "def test_x():\n" + " pass\n" + ) + mod, p = _make_module_with_source(tmp_path, src, "comment_respx") + item = _StubItem("comment_respx.py::test_x", p, module=mod) + apply_vcr_auto_marker_to_items([item], skip_files={"comment_respx.py"}) + assert getattr(item, VCR_SKIP_REASON_USER_ATTR) == SKIP_REASON_FILE_OPT_OUT + + +def test_should_flag_real_respx_mark_decorator_via_ast(vcr_enabled, tmp_path): + src = "import pytest\n" "@pytest.mark.respx\n" "def test_x(respx_mock): pass\n" + mod, p = _make_module_with_source(tmp_path, src, "real_respx_mark") + item = _StubItem("real_respx_mark.py::test_x", p, module=mod) + apply_vcr_auto_marker_to_items([item], skip_files={"real_respx_mark.py"}) + assert getattr(item, VCR_SKIP_REASON_USER_ATTR) == SKIP_REASON_RESPX_MODULE + + +def test_should_flag_real_respx_with_block_via_ast(vcr_enabled, tmp_path): + src = "import respx\n" "def test_x():\n" " with respx.mock():\n" " pass\n" + mod, p = _make_module_with_source(tmp_path, src, "real_respx_with") + item = _StubItem("real_respx_with.py::test_x", p, module=mod) + apply_vcr_auto_marker_to_items([item], skip_files={"real_respx_with.py"}) + assert getattr(item, VCR_SKIP_REASON_USER_ATTR) == SKIP_REASON_RESPX_MODULE + + +def test_should_flag_respx_mock_call_at_module_scope_via_ast(vcr_enabled, tmp_path): + src = "import respx\nmock = respx.mock()\ndef test_x(): pass\n" + mod, p = _make_module_with_source(tmp_path, src, "real_respx_call") + item = _StubItem("real_respx_call.py::test_x", p, module=mod) + apply_vcr_auto_marker_to_items([item], skip_files={"real_respx_call.py"}) + assert getattr(item, VCR_SKIP_REASON_USER_ATTR) == SKIP_REASON_RESPX_MODULE + + def test_should_tag_nodeid_suffix_skips_as_incompatible(vcr_enabled, tmp_path): mod, p = _make_module_with_source(tmp_path, "def test_x(): pass\n", "incompat") item = _StubItem("incompat.py::test_prompt_caching", p, module=mod) From 1ea600bd707f6cca019d4a357fc1d5162b966f4a Mon Sep 17 00:00:00 2001 From: claude Date: Wed, 13 May 2026 07:24:32 +0000 Subject: [PATCH 09/11] fix(vcr): aggregate worker stats on the controller so the session summary actually renders under xdist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_session_stats` is a module-level dict mutated inside `_vcr_outcome_gate` — which runs in each xdist worker process. The controller's `pytest_terminal_summary` then reads its own empty `_session_stats` and bails on `if not counts: return`, so the OVERFLOW / LIVE_CALL sections the rest of this PR adds never make it into CI logs in the dist mode CI actually uses. Ship a structured `vcr_outcome` payload via `user_properties` (which xdist round-trips) and add `aggregate_report_outcome` on the controller to fold worker outcomes into `_session_stats`. The recording process tags `vcr_recorded_by` with `PYTEST_XDIST_WORKER` so the controller can tell "single-process — already counted locally" apart from "produced by a worker — needs aggregation here", and not double-count when there's no xdist. Covered by 9 new unit tests in test_vcr_classification.py including the end-to-end summary render path. --- tests/_vcr_conftest_common.py | 128 ++++++++- .../test_vcr_classification.py | 248 +++++++++++++++++- 2 files changed, 372 insertions(+), 4 deletions(-) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 5cc544187b..a179a21ba6 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -825,6 +825,110 @@ def _reset_session_stats() -> None: _session_stats["skip_reason_examples"].clear() +# user_properties keys used to ship structured outcome data from xdist workers +# back to the controller. ``vcr_verdict`` is the human-readable line that +# ``VerboseReporterState.maybe_emit_verdict`` writes next to each test; +# ``vcr_outcome`` + ``vcr_recorded_by`` are the structured payload that +# ``aggregate_report_outcome`` folds into the controller's ``_session_stats`` +# so the session-end summary actually has data in xdist mode. +_USER_PROP_VERDICT_LINE = "vcr_verdict" +_USER_PROP_OUTCOME = "vcr_outcome" +_USER_PROP_RECORDED_BY = "vcr_recorded_by" + + +def _emit_outcome_payload( + node, + verdict: str, + *, + skip_reason: str | None = None, + live_call_hosts: Iterable[str] | None = None, +) -> None: + """Stash a structured VCR outcome on a pytest node so the xdist + controller can fold it into ``_session_stats``. + + On a worker, ``record_vcr_outcome`` has already updated the worker-local + ``_session_stats`` — but in xdist mode that state lives in the worker + process and never reaches the controller's ``pytest_terminal_summary``. + We use the report's ``user_properties`` channel (which xdist round-trips + back to the controller) to ship the outcome, and + ``aggregate_report_outcome`` rebuilds the controller's stats from there. + + The recorder tags ``vcr_recorded_by`` with ``PYTEST_XDIST_WORKER`` so + the controller can distinguish "recorded in this same main process — + already counted" from "recorded in a worker — needs aggregation here". + """ + node.user_properties.append( + ( + _USER_PROP_OUTCOME, + { + "verdict": verdict, + "skip_reason": skip_reason, + "live_call_hosts": list(live_call_hosts) if live_call_hosts else [], + }, + ) + ) + node.user_properties.append( + (_USER_PROP_RECORDED_BY, os.environ.get("PYTEST_XDIST_WORKER", "")) + ) + + +def aggregate_report_outcome(report) -> None: + """Fold a worker-produced VCR outcome into the controller's session stats. + + No-op outside the xdist controller path: + + * On a worker, ``_session_stats`` was already updated in-process by + ``record_vcr_outcome`` — and the worker doesn't render the summary + anyway, so there's nothing for us to aggregate. + * In single-process mode, ``vcr_recorded_by`` is the empty string, + which means the same process that ran the test is now handling the + report — ``_session_stats`` already has the entry, double-counting + would be a bug. + * Only when ``vcr_recorded_by`` is a non-empty worker id (``"gw0"`` + etc.) do we know the controller's ``_session_stats`` is missing this + test and needs the outcome folded in. + """ + if os.environ.get("PYTEST_XDIST_WORKER"): + return + if report.when != "teardown": + return + + recorded_by = next( + (v for k, v in (report.user_properties or []) if k == _USER_PROP_RECORDED_BY), + None, + ) + if not recorded_by: + return + + outcome = next( + (v for k, v in (report.user_properties or []) if k == _USER_PROP_OUTCOME), + None, + ) + if not outcome: + return + + verdict = outcome.get("verdict") + if not verdict: + return + + nodeid = report.nodeid + _session_stats["verdict_counts"][verdict] += 1 + + if verdict == VERDICT_MISS_OVERFLOW: + _session_stats["overflow_tests"].append(nodeid) + elif verdict == VERDICT_UNMARKED_LIVE_CALL: + _session_stats["unmarked_live_call_tests"].append( + (nodeid, list(outcome.get("live_call_hosts") or [])) + ) + + skip_reason = outcome.get("skip_reason") + if skip_reason: + _session_stats["skip_reason_counts"][skip_reason] += 1 + examples = _session_stats["skip_reason_examples"][skip_reason] + if len(examples) < 5: + examples.append(nodeid) + + def session_stats_snapshot() -> dict: """Read-only copy of the per-session VCR stats. Used by the summary.""" return { @@ -993,9 +1097,10 @@ def record_vcr_outcome(request, vcr) -> None: if not test_passed and verdict == VERDICT_MISS_RECORDED: verdict = VERDICT_MISS_NOT_PERSISTED _session_stats["verdict_counts"][verdict] += 1 + _emit_outcome_payload(request.node, verdict) if vcr_outcome_logging_enabled(): line = _format_verdict_line(verdict, cassette) - request.node.user_properties.append(("vcr_verdict", line)) + request.node.user_properties.append((_USER_PROP_VERDICT_LINE, line)) return # Cassette is None ⇒ test wasn't VCR-marked. Honor the skip reason @@ -1021,9 +1126,15 @@ def record_vcr_outcome(request, vcr) -> None: if len(examples) < 5: examples.append(nodeid) + _emit_outcome_payload( + request.node, + verdict, + skip_reason=skip_reason, + live_call_hosts=hosts, + ) if vcr_outcome_logging_enabled(): request.node.user_properties.append( - ("vcr_verdict", _format_verdict_line(verdict, None, extra)) + (_USER_PROP_VERDICT_LINE, _format_verdict_line(verdict, None, extra)) ) @@ -1225,6 +1336,13 @@ class VerboseReporterState: return self.terminal_reporter def maybe_emit_verdict(self, report) -> None: + # Aggregate xdist-worker stats into the controller's session counters + # first — this path is independent of verbose logging because the + # structured outcome payload is always attached when VCR is active, + # and ``aggregate_report_outcome`` no-ops outside the xdist-controller + # case on its own. + aggregate_report_outcome(report) + if report.when != "teardown": return if os.environ.get("PYTEST_XDIST_WORKER"): @@ -1235,7 +1353,11 @@ class VerboseReporterState: if reporter is None: return verdict = next( - (v for k, v in (report.user_properties or []) if k == "vcr_verdict"), + ( + v + for k, v in (report.user_properties or []) + if k == _USER_PROP_VERDICT_LINE + ), None, ) if not verdict: diff --git a/tests/llm_translation/test_vcr_classification.py b/tests/llm_translation/test_vcr_classification.py index 9ab0fedbbf..babb342731 100644 --- a/tests/llm_translation/test_vcr_classification.py +++ b/tests/llm_translation/test_vcr_classification.py @@ -42,6 +42,7 @@ from tests._vcr_conftest_common import ( # noqa: E402 _is_live_call_host, _reset_session_stats, _stable_key_value, + aggregate_report_outcome, apply_vcr_auto_marker_to_items, emit_vcr_classification_summary, install_live_call_probe, @@ -49,7 +50,6 @@ from tests._vcr_conftest_common import ( # noqa: E402 session_stats_snapshot, ) - # --------------------------------------------------------------------------- # Test doubles # --------------------------------------------------------------------------- @@ -504,6 +504,252 @@ def test_should_emit_no_summary_when_no_tests_observed(vcr_enabled): assert reporter.output == "" +# --------------------------------------------------------------------------- +# xdist controller aggregation +# +# _session_stats lives in module-global memory. Under xdist that memory is +# per-worker, so the controller's pytest_terminal_summary would render an +# empty summary without these aggregation hooks. The tests below simulate +# the controller receiving teardown reports produced by workers. +# --------------------------------------------------------------------------- + + +def _worker_report(nodeid: str, user_properties, *, when: str = "teardown"): + """Stand-in for a pytest TestReport delivered to the xdist controller. + + Only the attributes ``aggregate_report_outcome`` reads (``nodeid``, + ``when``, ``user_properties``) are populated. + """ + return SimpleNamespace( + nodeid=nodeid, + when=when, + user_properties=list(user_properties), + ) + + +def _outcome_from_worker( + verdict: str, + *, + worker_id: str = "gw0", + skip_reason=None, + live_call_hosts=None, +): + """Build the ``user_properties`` list a worker-side ``record_vcr_outcome`` + would attach. ``worker_id=""`` simulates the single-process case where + the same process that ran the test is handling the report.""" + return [ + ( + "vcr_outcome", + { + "verdict": verdict, + "skip_reason": skip_reason, + "live_call_hosts": list(live_call_hosts) if live_call_hosts else [], + }, + ), + ("vcr_recorded_by", worker_id), + ] + + +def test_controller_aggregates_hit_outcome_from_worker_report(vcr_enabled): + """An xdist controller starts with an empty _session_stats; a teardown + report carrying a worker-produced ``vcr_outcome`` must populate the + controller's verdict counts so the session summary has data to render.""" + report = _worker_report( + "t::hit", + _outcome_from_worker(VERDICT_HIT), + ) + + aggregate_report_outcome(report) + + snap = session_stats_snapshot() + assert snap["verdict_counts"][VERDICT_HIT] == 1 + + +def test_controller_records_overflow_nodeid_from_worker_report(vcr_enabled): + """OVERFLOW outcomes from workers must also populate + ``overflow_tests`` (the named-list the summary surfaces).""" + report = _worker_report( + "t::bedrock_overflow", + _outcome_from_worker(VERDICT_MISS_OVERFLOW), + ) + + aggregate_report_outcome(report) + + snap = session_stats_snapshot() + assert snap["verdict_counts"][VERDICT_MISS_OVERFLOW] == 1 + assert snap["overflow_tests"] == ["t::bedrock_overflow"] + + +def test_controller_records_live_call_hosts_from_worker_report(vcr_enabled): + """LIVE_CALL outcomes must round-trip the destination hosts so the + summary's 'UNMARKED TESTS WITH LIVE API CALLS' section has the same + detail it would in single-process mode.""" + report = _worker_report( + "t::prompt_caching", + _outcome_from_worker( + VERDICT_UNMARKED_LIVE_CALL, + skip_reason=SKIP_REASON_INCOMPATIBLE, + live_call_hosts=["api.anthropic.com", "api.x.ai"], + ), + ) + + aggregate_report_outcome(report) + + snap = session_stats_snapshot() + assert snap["verdict_counts"][VERDICT_UNMARKED_LIVE_CALL] == 1 + assert snap["unmarked_live_call_tests"] == [ + ("t::prompt_caching", ["api.anthropic.com", "api.x.ai"]) + ] + assert snap["skip_reason_counts"][SKIP_REASON_INCOMPATIBLE] == 1 + assert "t::prompt_caching" in snap["skip_reason_examples"][SKIP_REASON_INCOMPATIBLE] + + +def test_controller_does_not_double_count_single_process_reports(vcr_enabled): + """In single-process mode, ``record_vcr_outcome`` updates + ``_session_stats`` in the same process that later handles the report. + The aggregator must detect this (via empty ``vcr_recorded_by``) and + skip — otherwise every verdict would be counted twice.""" + report = _worker_report( + "t::single_proc", + _outcome_from_worker(VERDICT_HIT, worker_id=""), + ) + + aggregate_report_outcome(report) + + snap = session_stats_snapshot() + assert snap["verdict_counts"] == {} + + +def test_controller_ignores_reports_without_vcr_outcome(vcr_enabled): + """Tests outside the VCR plumbing (e.g. when VCR is disabled, or unit + tests that never went through ``_vcr_outcome_gate``) produce reports + with no ``vcr_outcome`` user property. The aggregator must no-op.""" + report = _worker_report("t::unrelated", [("other", "value")]) + + aggregate_report_outcome(report) + + snap = session_stats_snapshot() + assert snap["verdict_counts"] == {} + + +def test_controller_ignores_non_teardown_phases(vcr_enabled): + """Only the teardown report carries the final outcome; setup/call + reports must not contribute to the counts.""" + for phase in ("setup", "call"): + report = _worker_report( + "t::phase", + _outcome_from_worker(VERDICT_HIT), + when=phase, + ) + aggregate_report_outcome(report) + + snap = session_stats_snapshot() + assert snap["verdict_counts"] == {} + + +def test_controller_no_ops_when_running_inside_xdist_worker(vcr_enabled, monkeypatch): + """Workers update their own ``_session_stats`` directly via + ``record_vcr_outcome`` — re-aggregating from the report would + double-count their own work. The aggregator must bail when + ``PYTEST_XDIST_WORKER`` is set.""" + monkeypatch.setenv("PYTEST_XDIST_WORKER", "gw3") + report = _worker_report( + "t::on_worker", + _outcome_from_worker(VERDICT_HIT, worker_id="gw3"), + ) + + aggregate_report_outcome(report) + + snap = session_stats_snapshot() + assert snap["verdict_counts"] == {} + + +def test_controller_aggregated_outcomes_drive_session_summary(vcr_enabled): + """End-to-end: with only worker-produced reports (no in-process + ``record_vcr_outcome``), the session-end summary must still render + the OVERFLOW + LIVE_CALL sections that prove the cost-leak signal + survived the xdist worker→controller hop.""" + aggregate_report_outcome( + _worker_report( + "t::overflow_via_worker", + _outcome_from_worker(VERDICT_MISS_OVERFLOW), + ) + ) + aggregate_report_outcome( + _worker_report( + "t::live_call_via_worker", + _outcome_from_worker( + VERDICT_UNMARKED_LIVE_CALL, + skip_reason=SKIP_REASON_RESPX, + live_call_hosts=["api.openai.com"], + ), + ) + ) + + reporter = _FakeReporter() + emit_vcr_classification_summary(reporter) + + assert "VCR CACHE CLASSIFICATION SUMMARY" in reporter.output + assert "CASSETTE OVERFLOW" in reporter.output + assert "t::overflow_via_worker" in reporter.output + assert "UNMARKED TESTS WITH LIVE API CALLS" in reporter.output + assert "api.openai.com" in reporter.output + assert "t::live_call_via_worker" in reporter.output + + +def test_record_vcr_outcome_emits_structured_payload_for_marked_tests( + vcr_enabled, +): + """``record_vcr_outcome`` must always stash the structured outcome on + ``user_properties`` (independent of verbose logging) so the controller + has something to aggregate from in xdist mode.""" + request = SimpleNamespace( + node=SimpleNamespace( + nodeid="t::marked", + user_properties=[], + rep_call=SimpleNamespace(passed=True), + ) + ) + cassette = _cassette(played=1, dirty=False, total=1) + cassette._path = None + record_vcr_outcome(request, cassette) + + outcomes = [v for k, v in request.node.user_properties if k == "vcr_outcome"] + recorded_by = [v for k, v in request.node.user_properties if k == "vcr_recorded_by"] + assert outcomes == [ + {"verdict": VERDICT_HIT, "skip_reason": None, "live_call_hosts": []} + ] + # No PYTEST_XDIST_WORKER set in the vcr_enabled fixture, so the + # recording-process tag is the empty string (single-process mode). + assert recorded_by == [""] + + +def test_record_vcr_outcome_emits_structured_payload_for_unmarked_live_call( + vcr_enabled, +): + """The unmarked-LIVE_CALL path must ship the hosts list and the + skip-reason so the controller can rebuild both.""" + request_node = SimpleNamespace( + nodeid="t::leak", + user_properties=[], + rep_call=SimpleNamespace(passed=True), + ) + setattr(request_node, VCR_SKIP_REASON_USER_ATTR, SKIP_REASON_RESPX) + setattr(request_node, "vcr_live_call_hosts", ["api.openai.com"]) + request = SimpleNamespace(node=request_node) + + record_vcr_outcome(request, None) + + outcomes = [v for k, v in request.node.user_properties if k == "vcr_outcome"] + assert outcomes == [ + { + "verdict": VERDICT_UNMARKED_LIVE_CALL, + "skip_reason": SKIP_REASON_RESPX, + "live_call_hosts": ["api.openai.com"], + } + ] + + # --------------------------------------------------------------------------- # Live-call probe # --------------------------------------------------------------------------- From 39e1831e843e977fca8ab473cdfb19673b4cd72e Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Thu, 14 May 2026 12:30:47 -0700 Subject: [PATCH 10/11] Emit native web_search_tool_result blocks for Anthropic clients (Claude Desktop / Cowork citations) (#27886) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(custom_logger): add async_post_agentic_loop_response_hook Lets a CustomLogger shape the response returned by the agentic-loop follow-up call without bypassing the loop's safety / observability machinery (depth tracking, fingerprinting, etc.). Default returns the response unchanged. Used by websearch_interception to inject Anthropic-native web_search_tool_result blocks when the originating client requested a native web_search_* tool. * feat(llm_http_handler): call post-agentic-loop hook on the originating callback In _execute_anthropic_agentic_plan, after anthropic_messages.acreate returns, call the originating callback's async_post_agentic_loop_response_hook so it can mutate the final response (e.g. inject native tool_result blocks). Pass the callback through from _call_agentic_completion_hooks. Exceptions in the post-hook are caught and logged so a buggy callback can't kill the request. * feat(websearch_interception): add is_anthropic_native_web_search_tool Identifies tools the Anthropic-native clients (Claude Desktop, the Anthropic SDK, the Anthropic Console) use to request native search: type starts with "web_search_" (e.g. web_search_20250305). Rejects the LiteLLM standard tool, the OpenAI-function variant, the bare "WebSearch" legacy name, and the bare "web_search" Claude Code shape. This lets us decide per-request whether the client expects web_search_tool_result content blocks in the response, without renaming any existing constants or touching native-provider skip logic. * feat(websearch_interception): add build_web_search_tool_result_block Produces the Anthropic-native web_search_tool_result content block from a structured SearchResponse. Anthropic-native clients use this block to populate citations / source links — the existing text-blob flatten path only feeds readable evidence to the model and discards the structure, so this builder gives us the missing piece. Shape matches https://docs.anthropic.com/en/api/web-search-tool — web_search_result items carry url, title, page_age, encrypted_content (empty string when the search provider doesn't supply one). * feat(websearch_interception): emit native web_search_tool_result blocks When the originating client request carried a native Anthropic web_search_* tool, the final response now also carries web_search_tool_result content blocks alongside the model's text answer — so Claude Desktop / Anthropic SDK clients can populate the citations panel and replay conversation history with structured search evidence. Wiring: - Pre-request hooks (both deployment + Anthropic path) set a flag on kwargs when they see a native web_search_* tool, so the signal survives the conversion-to-litellm_web_search step regardless of which hook fires first. - _execute_search now returns (text, SearchResponse) so the structured results aren't lost when the text is flattened for the follow-up model call. - _build_anthropic_request_patch returns the parallel list of SearchResponse objects. - async_build_agentic_loop_plan pre-builds the web_search_tool_result blocks (one per tool_use_id) and stashes them on plan.metadata when the flag is set. - async_post_agentic_loop_response_hook reads the metadata and prepends the blocks to response.content. - _execute_agentic_loop mirrors the injection for the legacy path so both paths behave identically. Clients that send the LiteLLM standard tool keep the existing text-only behavior — no regression. * test(websearch_interception): cover native web_search_tool_result emission 18 tests across: - detector branches (native vs litellm-standard, OpenAI-function shape, Claude Desktop builtin WebSearch, bare web_search, missing type) - block-builder shape (results, none, empty) - pre-request hook flag-setting (native sets, standard does not) - async_build_agentic_loop_plan attaches blocks to plan.metadata when the flag is present, leaves metadata untouched when absent - post-hook injection into dict and object responses - legacy _execute_agentic_loop mirrors the injection so both paths return the same shape * test(websearch_short_circuit): keep _execute_search mocks in sync with new tuple return * test(websearch_thinking_constraint): keep _execute_search mocks in sync with new tuple return * feat(websearch_interception): emit native blocks from try_short_circuit_search The agentic-loop post-hook only fires when the model returns a tool_use block. Cowork / Claude Desktop on Bedrock actually make TWO requests per user turn: the main /v1/messages with their builtin tool, and a separate standalone /v1/messages whose only tool is web_search_20250305. That second request hits try_short_circuit_search — no agentic loop, no post-hook — and was returning text-only, leaving the citations panel empty. When the short-circuit input carries a native web_search_* tool, build a synthetic server_tool_use + web_search_tool_result pair (using the structured SearchResponse already returned by _execute_search) so the client gets the native shape it expects. The legacy text block is preserved so non-native short-circuit callers (Claude Code, github_copilot, etc.) see the same payload as before. Failure path still emits the native block pair (with empty results) plus the text-error block, so the client gets a well-formed response rather than a malformed half-shape. * test(websearch_native_blocks): cover short-circuit native-block emission Three new cases on top of the existing 18: - native web_search_20250305 short-circuit → [server_tool_use, web_search_tool_result, text], ids paired, urls/titles carried. - litellm_web_search short-circuit → text-only (no regression). - native short-circuit on search failure → still emits the native block pair (empty results) plus the text-error block, so the client never sees a malformed half-shape. * test(websearch_short_circuit): index assertions by block type, not by position Native short-circuit responses now have [server_tool_use, web_search_tool_result, text] when the input carries web_search_20250305 — find the text block by type rather than relying on content[0]. * fix(websearch_interception): gate legacy WebSearch name on schema absence Clients like Cowork / Claude Desktop ship a client-side tool named "WebSearch" with a full input_schema — they handle it themselves and expect to make a separate native web_search_20250305 sub-request for the actual search. Today is_web_search_tool matches the bare name regardless of other fields, which hijacks the client's tool server-side. The agentic loop fires on the main request, the model never gets to emit the client-side tool_use, and the separate native sub-request (where citation data flows) is never made. Net: citations panel empty. Real Anthropic client tools always carry input_schema (the API rejects them otherwise), so a bare {name: "WebSearch"} with no schema is the only thing that could be a legacy interception marker. Gate the match on schema absence: legacy callers (if any) keep working, real client-side WebSearch tools pass through untouched. * fix(websearch_interception): drop "WebSearch" from response-detection lists Post-conversion the model always sees ``litellm_web_search``, so the "WebSearch" entry in the response-side tool_use detection lists was dead at best. If a model ever did return ``tool_use(name="WebSearch")`` it would now (incorrectly) hijack the client's own ``WebSearch`` tool again — same Cowork problem we just fixed on the input side. Drop it. * test(websearch_native_blocks): cover the WebSearch legacy-name schema gate Three new cases: - {name: "WebSearch"} (bare interception marker) → still matched - {name: "WebSearch", input_schema: {...}} (Cowork client tool) → passes through untouched - {name: "WebSearch", description: "..."} (no schema) → still matched on the assumption it's a legacy marker rather than a malformed real client tool. --------- Co-authored-by: Ishaan Jaffer --- litellm/integrations/custom_logger.py | 21 + .../websearch_interception/handler.py | 247 ++++++++- .../websearch_interception/tools.py | 47 +- .../websearch_interception/transformation.py | 66 ++- litellm/llms/custom_httpx/llm_http_handler.py | 21 +- .../test_websearch_native_blocks.py | 484 ++++++++++++++++++ .../test_websearch_short_circuit.py | 34 +- .../test_websearch_thinking_constraint.py | 36 +- 8 files changed, 900 insertions(+), 56 deletions(-) create mode 100644 tests/test_litellm/integrations/websearch_interception/test_websearch_native_blocks.py diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 300c311f36..481cf7fce8 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -697,6 +697,27 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac """ return AgenticLoopPlan(run_agentic_loop=False) + async def async_post_agentic_loop_response_hook( + self, + response: Any, + plan: AgenticLoopPlan, + kwargs: Dict, + ) -> Any: + """ + Post-process the response returned by the agentic-loop follow-up call. + + Called after BaseLLMHTTPHandler executes ``AgenticLoopPlan.request_patch`` + and receives the final response from the provider. Lets callbacks shape + what the client sees without bypassing the loop's safety / observability + machinery (depth tracking, fingerprinting, etc.). + + Use ``plan.metadata`` to carry whatever the build step decided to expose + for post-processing (e.g. native tool_result blocks to inject). + + Default returns ``response`` unchanged. + """ + return response + async def async_should_run_chat_completion_agentic_loop( self, response: Any, diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 41618c7262..37528e7dcd 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -19,12 +19,14 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.websearch_interception.tools import ( get_litellm_web_search_tool, get_litellm_web_search_tool_openai, + is_anthropic_native_web_search_tool, is_web_search_tool, is_web_search_tool_chat_completion, ) from litellm.integrations.websearch_interception.transformation import ( WebSearchTransformation, ) +from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.types.integrations.websearch_interception import ( WebSearchInterceptionConfig, ) @@ -36,6 +38,16 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager +# Key used to flag, on per-request kwargs, that the originating client sent +# an Anthropic-native ``web_search_*`` tool — meaning the final response +# should include ``web_search_tool_result`` content blocks so the client +# (e.g. Claude Desktop's citations panel) can render sources. +WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY = "_websearch_interception_emit_native_blocks" + +# Key on ``AgenticLoopPlan.metadata`` carrying the list of pre-built +# ``web_search_tool_result`` blocks to inject into the final response. +WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY = "websearch_native_blocks" + class WebSearchInterceptionLogger(CustomLogger): """ @@ -152,22 +164,55 @@ class WebSearchInterceptionLogger(CustomLogger): f"(provider={provider_str}, query='{query}')" ) - # Execute search + # Native clients (Claude Desktop / Cowork / Anthropic SDK) make a + # standalone /v1/messages sub-request just for the search, and they + # expect the response in native shape with server_tool_use + + # web_search_tool_result content blocks so the citations panel can + # render. The agentic-loop post-hook never fires on this path because + # there is no model call — emit the native blocks here instead. + native_tool = next( + (t for t in tools if is_anthropic_native_web_search_tool(t)), + None, + ) + + # Execute search — keep the structured SearchResponse so the native + # block can carry per-result url/title/page_age. try: - search_result_text = await self._execute_search(query) + search_result_text, structured = await self._execute_search(query) except Exception as e: verbose_logger.error( f"WebSearchInterception: Short-circuit search failed: {e}" ) - search_result_text = f"Search failed: {e}" + search_result_text, structured = f"Search failed: {e}", None + + content: List[Dict[str, Any]] = [] + if native_tool is not None: + tool_use_id = f"srvtoolu_{uuid.uuid4().hex}" + tool_name = native_tool.get("name") or "web_search" + content.append( + { + "type": "server_tool_use", + "id": tool_use_id, + "name": tool_name, + "input": {"query": query}, + } + ) + content.append( + WebSearchTransformation.build_web_search_tool_result_block( + tool_use_id=tool_use_id, + search_response=structured, + ) + ) + # Keep the text block so non-native short-circuit callers (Claude Code, + # github_copilot, etc.) see the same payload they always have. + content.append({"type": "text", "text": search_result_text}) - # Build synthetic Anthropic response response: Dict[str, Any] = { "id": f"msg_{str(uuid.uuid4())}", "type": "message", "role": "assistant", "model": model, - "content": [{"type": "text", "text": search_result_text}], + "content": content, "stop_reason": "end_turn", "stop_sequence": None, "usage": {"input_tokens": 0, "output_tokens": 0}, @@ -175,7 +220,8 @@ class WebSearchInterceptionLogger(CustomLogger): verbose_logger.debug( "WebSearchInterception: Short-circuit search completed, " - f"returning synthetic response ({len(search_result_text)} chars)" + f"returning synthetic response ({len(search_result_text)} chars, " + f"native_blocks={native_tool is not None})" ) return response @@ -219,6 +265,14 @@ class WebSearchInterceptionLogger(CustomLogger): "WebSearchInterception: Converting native web_search tools to LiteLLM standard" ) + # If the client sent an Anthropic-native web_search_* tool, mark the + # request so the agentic loop emits native web_search_tool_result + # blocks in the final response (matches async_pre_request_hook). This + # deployment hook fires before async_pre_request_hook on some paths, + # so flagging here ensures the signal isn't lost regardless of order. + if any(is_anthropic_native_web_search_tool(t) for t in tools): + kwargs[WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY] = True + # Convert native/custom web_search tools to LiteLLM standard converted_tools = [] for tool in tools: @@ -342,6 +396,14 @@ class WebSearchInterceptionLogger(CustomLogger): f"WebSearchInterception: Pre-request hook triggered for provider={custom_llm_provider}" ) + # If the client sent an Anthropic-native web_search_* tool, mark the + # request so the agentic loop emits native web_search_tool_result + # blocks in the final response (for citations panels, etc.). The flag + # is read by async_build_agentic_loop_plan; the leading underscore + # prefix ensures it is stripped before the follow-up call kwargs. + if any(is_anthropic_native_web_search_tool(t) for t in tools): + kwargs[WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY] = True + # Convert native web search tools to LiteLLM standard converted_tools = [] for tool in tools: @@ -591,7 +653,7 @@ class WebSearchInterceptionLogger(CustomLogger): ) -> AgenticLoopPlan: tool_calls = tools["tool_calls"] thinking_blocks = tools.get("thinking_blocks", []) - request_patch = await self._build_anthropic_request_patch( + request_patch, structured_results = await self._build_anthropic_request_patch( model=model, messages=messages, tool_calls=tool_calls, @@ -600,12 +662,92 @@ class WebSearchInterceptionLogger(CustomLogger): logging_obj=logging_obj, kwargs=kwargs, ) + + metadata: Dict[str, Any] = { + "tool_type": "websearch", + "response_format": "anthropic", + } + + # If the client request originally carried a native web_search_* tool, + # pre-build the Anthropic-native ``web_search_tool_result`` blocks now + # (while we still have the structured SearchResponse list) and stash + # them on plan metadata for the post-hook to inject. + if kwargs.get(WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY): + metadata[WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY] = ( + self._build_native_result_blocks( + tool_calls=tool_calls, + structured_results=structured_results, + ) + ) + return AgenticLoopPlan( run_agentic_loop=True, request_patch=request_patch, - metadata={"tool_type": "websearch", "response_format": "anthropic"}, + metadata=metadata, ) + async def async_post_agentic_loop_response_hook( + self, + response: Any, + plan: AgenticLoopPlan, + kwargs: Dict, + ) -> Any: + """ + Inject Anthropic-native ``web_search_tool_result`` blocks into the + final response when the originating client used a native + ``web_search_*`` tool. + + See ``WebSearchTransformation.build_web_search_tool_result_block`` for + the block shape. The blocks are prepended to ``response.content`` so + Anthropic-native clients (Claude Desktop, the Anthropic SDK) can + render citations / sources alongside the model's textual reply. + """ + native_blocks = plan.metadata.get(WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY) + if not native_blocks: + return response + return self._inject_native_blocks(response, native_blocks) + + @staticmethod + def _build_native_result_blocks( + tool_calls: List[Dict], + structured_results: List[Optional[SearchResponse]], + ) -> List[Dict[str, Any]]: + """Build one ``web_search_tool_result`` block per tool_call.""" + blocks: List[Dict[str, Any]] = [] + for i, tool_call in enumerate(tool_calls): + tool_use_id = tool_call.get("id") or "" + structured = structured_results[i] if i < len(structured_results) else None + blocks.append( + WebSearchTransformation.build_web_search_tool_result_block( + tool_use_id=tool_use_id, + search_response=structured, + ) + ) + return blocks + + @staticmethod + def _inject_native_blocks( + response: Any, native_blocks: List[Dict[str, Any]] + ) -> Any: + """Prepend native blocks to response content, dict or object form.""" + if not native_blocks: + return response + if isinstance(response, dict): + existing = response.get("content") or [] + response["content"] = list(native_blocks) + list(existing) + return response + existing = getattr(response, "content", None) or [] + try: + response.content = list(native_blocks) + list(existing) + except (AttributeError, TypeError): + # Object refused write — fall through and leave the response + # untouched rather than crash the request. + verbose_logger.debug( + "WebSearchInterception: could not inject native blocks into " + f"response of type {type(response).__name__}" + ) + return response + async def async_run_chat_completion_agentic_loop( self, tools: Dict, @@ -733,7 +875,7 @@ class WebSearchInterceptionLogger(CustomLogger): kwargs: Dict, ) -> Any: """Legacy path: execute search + build patch + run follow-up call.""" - request_patch = await self._build_anthropic_request_patch( + request_patch, structured_results = await self._build_anthropic_request_patch( model=model, messages=messages, tool_calls=tool_calls, @@ -755,7 +897,7 @@ class WebSearchInterceptionLogger(CustomLogger): if max_tokens is None: max_tokens = cast(int, kwargs.get("max_tokens", 1024)) - return await anthropic_messages.acreate( + response = await anthropic_messages.acreate( max_tokens=max_tokens, messages=request_patch.messages, model=request_patch.model or model, @@ -763,6 +905,18 @@ class WebSearchInterceptionLogger(CustomLogger): **request_patch.kwargs, ) + # Legacy path: the new path goes through the typed plan + core + # dispatcher which runs the post-hook automatically. Mirror the + # native-block injection here so both paths behave identically. + if kwargs.get(WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY): + native_blocks = self._build_native_result_blocks( + tool_calls=tool_calls, + structured_results=structured_results, + ) + response = self._inject_native_blocks(response, native_blocks) + + return response + async def _build_anthropic_request_patch( self, model: str, @@ -772,8 +926,16 @@ class WebSearchInterceptionLogger(CustomLogger): anthropic_messages_optional_request_params: Dict, logging_obj: Any, kwargs: Dict, - ) -> AgenticLoopRequestPatch: - """Execute litellm.search() and build follow-up request patch.""" + ) -> Tuple[AgenticLoopRequestPatch, List[Optional[SearchResponse]]]: + """ + Execute litellm.search() and build follow-up request patch. + + Returns the patch alongside the parallel list of structured + ``SearchResponse`` objects (one per tool_call, ``None`` when the + search failed or the tool_call had no query). The caller uses these + to optionally build Anthropic-native ``web_search_tool_result`` + content blocks for the final response. + """ # Extract search queries from tool_use blocks search_tasks = [] @@ -797,23 +959,38 @@ class WebSearchInterceptionLogger(CustomLogger): ) search_results = await asyncio.gather(*search_tasks, return_exceptions=True) - # Handle any exceptions in search results + # Split the gathered (text, structured) tuples into two parallel lists. + # The text list feeds the follow-up model call; the structured list + # is returned to the caller for native-block emission. final_search_results: List[str] = [] + structured_results: List[Optional[SearchResponse]] = [] for i, result in enumerate(search_results): if isinstance(result, Exception): verbose_logger.error( f"WebSearchInterception: Search {i} failed with error: {str(result)}" ) final_search_results.append(f"Search failed: {str(result)}") - elif isinstance(result, str): - # Explicitly cast to str for type checker - final_search_results.append(cast(str, result)) + structured_results.append(None) + elif isinstance(result, tuple) and len(result) == 2: + text_value, structured_value = result + final_search_results.append( + cast(str, text_value) + if isinstance(text_value, str) + else str(text_value) + ) + structured_results.append( + structured_value + if isinstance(structured_value, SearchResponse) + else None + ) else: - # Should never happen, but handle for type safety + # Defensive: legacy callers / unexpected shape — preserve text, + # drop structure. verbose_logger.debug( f"WebSearchInterception: Unexpected result type {type(result)} at index {i}" ) final_search_results.append(str(result)) + structured_results.append(None) # Build assistant and user messages using transformation assistant_message, user_message = WebSearchTransformation.transform_response( @@ -859,16 +1036,26 @@ class WebSearchInterceptionLogger(CustomLogger): len(follow_up_messages), len(final_search_results), ) - return AgenticLoopRequestPatch( + patch = AgenticLoopRequestPatch( model=full_model_name, messages=follow_up_messages, max_tokens=max_tokens, optional_params=optional_params_without_max_tokens, kwargs=kwargs_for_followup, ) + return patch, structured_results - async def _execute_search(self, query: str) -> str: - """Execute a single web search using router's search tools""" + async def _execute_search(self, query: str) -> Tuple[str, Optional[SearchResponse]]: + """ + Execute a single web search using router's search tools. + + Returns both the formatted text (fed back to the model in the follow-up + call) and the structured ``SearchResponse`` (preserved so callers can + build Anthropic-native ``web_search_tool_result`` blocks for clients + that requested a native ``web_search_*`` tool). The structured value + is None on the failure path so callers can still emit an empty result + block rather than dropping the search entirely. + """ try: # Import router from proxy_server try: @@ -934,7 +1121,7 @@ class WebSearchInterceptionLogger(CustomLogger): verbose_logger.debug( f"WebSearchInterception: Search completed for '{query}', got {len(search_result_text)} chars" ) - return search_result_text + return search_result_text, result except Exception as e: verbose_logger.error( f"WebSearchInterception: Search failed for '{query}': {str(e)}" @@ -1015,7 +1202,8 @@ class WebSearchInterceptionLogger(CustomLogger): ) search_results = await asyncio.gather(*search_tasks, return_exceptions=True) - # Handle any exceptions in search results + # Chat-completion path only needs text — OpenAI tool_result format + # has no equivalent of Anthropic's web_search_tool_result block. final_search_results: List[str] = [] for i, result in enumerate(search_results): if isinstance(result, Exception): @@ -1023,8 +1211,13 @@ class WebSearchInterceptionLogger(CustomLogger): f"WebSearchInterception: Search {i} failed with error: {str(result)}" ) final_search_results.append(f"Search failed: {str(result)}") - elif isinstance(result, str): - final_search_results.append(cast(str, result)) + elif isinstance(result, tuple) and len(result) == 2: + text_value, _ = result + final_search_results.append( + cast(str, text_value) + if isinstance(text_value, str) + else str(text_value) + ) else: verbose_logger.debug( f"WebSearchInterception: Unexpected result type {type(result)} at index {i}" @@ -1112,9 +1305,11 @@ class WebSearchInterceptionLogger(CustomLogger): kwargs=kwargs_for_followup, ) - async def _create_empty_search_result(self) -> str: + async def _create_empty_search_result( + self, + ) -> Tuple[str, Optional[SearchResponse]]: """Create an empty search result for tool calls without queries""" - return "No search query provided" + return "No search query provided", None @staticmethod def initialize_from_proxy_config( diff --git a/litellm/integrations/websearch_interception/tools.py b/litellm/integrations/websearch_interception/tools.py index e373b64cdd..b29372af9e 100644 --- a/litellm/integrations/websearch_interception/tools.py +++ b/litellm/integrations/websearch_interception/tools.py @@ -126,6 +126,27 @@ def is_web_search_tool_chat_completion(tool: Dict[str, Any]) -> bool: return False +def is_anthropic_native_web_search_tool(tool: Dict[str, Any]) -> bool: + """ + Check if a tool is an Anthropic-native ``web_search_*`` tool. + + Native clients (Anthropic SDK, Claude Desktop, Anthropic Console) send + tools like ``{"type": "web_search_20250305", "name": "web_search"}`` and + expect the response to contain ``web_search_tool_result`` content blocks + so that citations can be rendered. This helper identifies that contract + so the agentic loop can emit native-format blocks for those clients + without affecting clients that send the LiteLLM standard tool. + + Returns False for the LiteLLM standard tool (``litellm_web_search``), + the OpenAI-shaped variant, the bare ``WebSearch`` legacy name, and the + bare ``web_search`` name (Claude Code style). + """ + tool_type = tool.get("type", "") + if not isinstance(tool_type, str): + return False + return tool_type.startswith("web_search_") and tool_type != "function" + + def is_web_search_tool(tool: Dict[str, Any]) -> bool: """ Check if a tool is a web search tool (native or LiteLLM standard). @@ -135,7 +156,22 @@ def is_web_search_tool(tool: Dict[str, Any]) -> bool: - OpenAI format: type == "function" with function.name == "litellm_web_search" - Anthropic native: type starts with "web_search_" (e.g., "web_search_20250305") - Claude Code: name == "web_search" with a type field - - Custom: name == "WebSearch" (legacy format) + - Custom: name == "WebSearch" (legacy interception marker — only matched + when input_schema is absent; see note below) + + Note on the legacy ``WebSearch`` name: + Clients like Claude Desktop / Cowork ship a *client-side* tool called + ``WebSearch`` (a fully-formed Anthropic client tool with its own + ``input_schema``) that they handle themselves. Treating that as our + interception marker hijacks it server-side and the client's own tool + handler never fires — which means Cowork's separate native + ``web_search_20250305`` sub-request (where citation data actually + flows) never gets made. + + Real Anthropic client tools always carry an ``input_schema`` (the API + rejects them otherwise), so a bare ``{name: "WebSearch"}`` with no + schema is the only thing that could be a legacy interception marker. + Gate the match on schema absence to keep both groups working. Args: tool: Tool dictionary to check @@ -152,6 +188,10 @@ def is_web_search_tool(tool: Dict[str, Any]) -> bool: True >>> is_web_search_tool({"name": "calculator"}) False + >>> is_web_search_tool({"name": "WebSearch"}) # legacy interception marker + True + >>> is_web_search_tool({"name": "WebSearch", "input_schema": {"type": "object"}}) # Cowork client tool + False """ tool_name = tool.get("name", "") tool_type = tool.get("type", "") @@ -175,8 +215,9 @@ def is_web_search_tool(tool: Dict[str, Any]) -> bool: if tool_name == "web_search" and tool_type: return True - # Check for legacy WebSearch format - if tool_name == "WebSearch": + # Legacy "WebSearch" interception marker — only when no schema is + # present, so real client-side WebSearch tools (Cowork) pass through. + if tool_name == "WebSearch" and "input_schema" not in tool: return True return False diff --git a/litellm/integrations/websearch_interception/transformation.py b/litellm/integrations/websearch_interception/transformation.py index 00d4829ad3..9c20a3f6c7 100644 --- a/litellm/integrations/websearch_interception/transformation.py +++ b/litellm/integrations/websearch_interception/transformation.py @@ -100,11 +100,14 @@ class WebSearchTransformation: block_id = getattr(block, "id", None) block_input = getattr(block, "input", {}) - # Check for LiteLLM standard or legacy web search tools - # Handles: litellm_web_search, WebSearch, web_search + # Detect tool_use blocks that came from interception. After + # pre-request conversion the model always sees + # ``litellm_web_search``; the bare ``web_search`` entry handles + # callers that bypass our pre-request hooks (e.g. direct + # litellm.acompletion). "WebSearch" is intentionally omitted — + # see is_web_search_tool for the Cowork rationale. if block_type == "tool_use" and block_name in ( LITELLM_WEB_SEARCH_TOOL_NAME, - "WebSearch", "web_search", ): # Convert to dict for easier handling @@ -190,10 +193,12 @@ class WebSearchTransformation: getattr(function, "arguments", None) if function else None ) - # Check for LiteLLM standard or legacy web search tools + # Detect function-style web search tool_calls. ``WebSearch`` is + # intentionally omitted — see is_web_search_tool for the Cowork + # rationale (clients ship their own client-side ``WebSearch`` and + # we must not hijack it). if tool_type == "function" and function_name in ( LITELLM_WEB_SEARCH_TOOL_NAME, - "WebSearch", "web_search", ): # Parse arguments (might be JSON string) @@ -350,6 +355,57 @@ class WebSearchTransformation: return assistant_message, tool_messages + @staticmethod + def build_web_search_tool_result_block( + tool_use_id: str, + search_response: Optional[SearchResponse], + ) -> Dict[str, Any]: + """ + Build an Anthropic-native ``web_search_tool_result`` content block. + + Native Anthropic clients (Claude Desktop, the Anthropic SDK, the + Anthropic Console) expect search-tool results to be returned as + structured ``web_search_tool_result`` blocks so that citations and + source links can be rendered. The agentic loop currently feeds the + model a flat text blob in the follow-up call (which is correct — the + model needs readable evidence). This helper produces the *additional* + block that should accompany the model's text reply when the original + request used a native ``web_search_*`` tool. + + Spec reference: + https://docs.anthropic.com/en/api/web-search-tool + + Args: + tool_use_id: The ``tool_use_id`` the model emitted on the first + turn. Must match exactly so the client can pair the result + with its tool_use block. + search_response: Structured ``SearchResponse`` from + ``litellm.asearch()``. If None or empty, the block is still + emitted with an empty result list (signals "search ran, no + results" rather than "search did not run"). + """ + items: List[Dict[str, Any]] = [] + if search_response is not None: + results = getattr(search_response, "results", None) or [] + for r in results: + url = getattr(r, "url", "") or "" + title = getattr(r, "title", "") or "" + page_age = getattr(r, "date", None) or getattr(r, "last_updated", None) + items.append( + { + "type": "web_search_result", + "url": url, + "title": title, + "page_age": page_age, + "encrypted_content": "", + } + ) + return { + "type": "web_search_tool_result", + "tool_use_id": tool_use_id, + "content": items, + } + @staticmethod def format_search_response(result: SearchResponse) -> str: """ diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index fa1253d900..2ff63cc2d7 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -4634,6 +4634,7 @@ class BaseLLMHTTPHandler: fingerprints: List[str], fingerprint: str, stream: bool = False, + callback: Optional[Any] = None, ) -> Any: from litellm.anthropic_interface import messages as anthropic_messages @@ -4675,7 +4676,7 @@ class BaseLLMHTTPHandler: kwargs_for_followup["max_agentic_loops"] = max_loops kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint] - return await anthropic_messages.acreate( + response = await anthropic_messages.acreate( **{ "max_tokens": max_tokens, "messages": patch.messages, @@ -4686,6 +4687,23 @@ class BaseLLMHTTPHandler: } ) + if callback is not None: + try: + response = await callback.async_post_agentic_loop_response_hook( + response=response, plan=plan, kwargs=kwargs + ) + except Exception as e: + _call_id = getattr(logging_obj, "litellm_call_id", "unknown") + verbose_logger.exception( + "LiteLLM.AgenticHookError: Exception in " + "async_post_agentic_loop_response_hook [call_id=%s model=%s]: %s", + _call_id, + model, + str(e), + ) + + return response + async def _execute_chat_completion_agentic_plan( self, plan: AgenticLoopPlan, @@ -4869,6 +4887,7 @@ class BaseLLMHTTPHandler: fingerprints=fingerprints, fingerprint=fingerprint, stream=stream, + callback=callback, ) except Exception as e: _call_id = getattr(logging_obj, "litellm_call_id", "unknown") diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_native_blocks.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_native_blocks.py new file mode 100644 index 0000000000..544abab8dc --- /dev/null +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_native_blocks.py @@ -0,0 +1,484 @@ +""" +Tests for Anthropic-native ``web_search_tool_result`` block emission. + +Covers the path that lets Claude Desktop / Anthropic SDK clients render +citations when their request used a native ``web_search_*`` tool against a +provider (e.g. Bedrock) that can't run web search natively. +""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from litellm.integrations.websearch_interception.handler import ( + WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY, + WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY, + WebSearchInterceptionLogger, +) +from litellm.integrations.websearch_interception.tools import ( + is_anthropic_native_web_search_tool, + is_web_search_tool, +) +from litellm.integrations.websearch_interception.transformation import ( + WebSearchTransformation, +) +from litellm.llms.base_llm.search.transformation import SearchResponse, SearchResult +from litellm.types.integrations.custom_logger import ( + AgenticLoopPlan, + AgenticLoopRequestPatch, +) + + +def _make_search_response() -> SearchResponse: + return SearchResponse( + results=[ + SearchResult( + title="LiteLLM Docs", + url="https://docs.litellm.ai/", + snippet="Unified interface for LLMs.", + date="2025-01-15", + ), + SearchResult( + title="Bedrock Pricing", + url="https://aws.amazon.com/bedrock/pricing/", + snippet="Pay-per-use pricing model.", + date=None, + ), + ] + ) + + +class TestIsAnthropicNativeWebSearchTool: + """The detector must match native tools without catching look-alikes.""" + + def test_matches_web_search_20250305(self): + assert is_anthropic_native_web_search_tool( + {"type": "web_search_20250305", "name": "web_search", "max_uses": 5} + ) + + def test_matches_future_dated_variant(self): + assert is_anthropic_native_web_search_tool( + {"type": "web_search_20260101", "name": "web_search"} + ) + + def test_rejects_litellm_standard(self): + assert not is_anthropic_native_web_search_tool( + {"name": "litellm_web_search", "input_schema": {}} + ) + + def test_rejects_openai_function_shape(self): + assert not is_anthropic_native_web_search_tool( + {"type": "function", "function": {"name": "litellm_web_search"}} + ) + + def test_rejects_claude_desktop_builtin(self): + # Claude Desktop's builtin client-side ``WebSearch`` tool must not be + # misidentified — that's the collision PR #25242 introduced. + assert not is_anthropic_native_web_search_tool({"name": "WebSearch"}) + + def test_rejects_unrelated_tool(self): + assert not is_anthropic_native_web_search_tool( + {"type": "function", "function": {"name": "calculator"}} + ) + + def test_handles_missing_type(self): + assert not is_anthropic_native_web_search_tool({"name": "web_search"}) + + +class TestLegacyWebSearchNameGate: + """The bare ``WebSearch`` name is a legacy interception marker. Real + client-side ``WebSearch`` tools (Cowork, Claude Desktop) carry an + ``input_schema`` and must pass through untouched — otherwise the proxy + hijacks them server-side and the client's own tool handler never fires, + which means the separate ``web_search_20250305`` sub-request (where + citations actually flow) is never made.""" + + def test_bare_legacy_name_still_matched(self): + # Caller deliberately uses the bare-name interception marker — + # back-compat for anyone relying on the old shape. + assert is_web_search_tool({"name": "WebSearch"}) + + def test_real_client_tool_passes_through(self): + # Cowork's client-side WebSearch tool ships with input_schema. + cowork_tool = { + "name": "WebSearch", + "input_schema": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + } + assert not is_web_search_tool(cowork_tool) + + def test_real_client_tool_with_description_passes_through(self): + # description-only client tools (no schema) are not valid Anthropic + # tools; only the schema-bearing shape is the disambiguator. This + # case stays matched on the assumption it's a legacy marker. + assert is_web_search_tool({"name": "WebSearch", "description": "search"}) + + +class TestBuildWebSearchToolResultBlock: + """The block-builder must produce the Anthropic-native shape exactly.""" + + def test_shape_with_results(self): + block = WebSearchTransformation.build_web_search_tool_result_block( + tool_use_id="toolu_abc", + search_response=_make_search_response(), + ) + assert block["type"] == "web_search_tool_result" + assert block["tool_use_id"] == "toolu_abc" + assert len(block["content"]) == 2 + first = block["content"][0] + assert first["type"] == "web_search_result" + assert first["url"] == "https://docs.litellm.ai/" + assert first["title"] == "LiteLLM Docs" + assert first["page_age"] == "2025-01-15" + assert first["encrypted_content"] == "" + + def test_handles_none_search_response(self): + block = WebSearchTransformation.build_web_search_tool_result_block( + tool_use_id="toolu_abc", + search_response=None, + ) + assert block["type"] == "web_search_tool_result" + assert block["tool_use_id"] == "toolu_abc" + assert block["content"] == [] + + def test_handles_empty_results(self): + block = WebSearchTransformation.build_web_search_tool_result_block( + tool_use_id="toolu_xyz", + search_response=SearchResponse(results=[]), + ) + assert block["content"] == [] + + +class TestPreRequestHookFlagsNativeTools: + """The pre-request hook must mark the request when a native tool is used.""" + + @pytest.mark.asyncio + async def test_native_tool_sets_flag(self): + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) + kwargs = { + "tools": [ + {"type": "web_search_20250305", "name": "web_search", "max_uses": 5} + ], + "litellm_params": {"custom_llm_provider": "bedrock"}, + } + out = await logger.async_pre_request_hook( + model="bedrock/claude", messages=[], kwargs=kwargs + ) + assert out is not None + assert out.get(WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY) is True + + @pytest.mark.asyncio + async def test_litellm_standard_tool_does_not_set_flag(self): + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) + kwargs = { + "tools": [{"name": "litellm_web_search", "input_schema": {}}], + "litellm_params": {"custom_llm_provider": "bedrock"}, + } + out = await logger.async_pre_request_hook( + model="bedrock/claude", messages=[], kwargs=kwargs + ) + assert out is not None + assert WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY not in out + + +class TestBuildPlanAttachesBlocks: + """async_build_agentic_loop_plan must put pre-built blocks on metadata.""" + + @pytest.mark.asyncio + async def test_metadata_carries_blocks_when_flag_set(self): + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) + tool_calls = [ + { + "id": "toolu_one", + "type": "tool_use", + "name": "litellm_web_search", + "input": {"query": "what is litellm"}, + } + ] + patch_obj = AgenticLoopRequestPatch( + model="bedrock/claude", + messages=[{"role": "user", "content": "hi"}], + max_tokens=1024, + ) + structured = [_make_search_response()] + + with patch.object( + logger, + "_build_anthropic_request_patch", + new=AsyncMock(return_value=(patch_obj, structured)), + ): + plan = await logger.async_build_agentic_loop_plan( + tools={"tool_calls": tool_calls, "thinking_blocks": []}, + model="bedrock/claude", + messages=[], + response=MagicMock(), + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={}, + logging_obj=MagicMock(model_call_details={}), + stream=False, + kwargs={WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: True}, + ) + + blocks = plan.metadata.get(WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY) + assert isinstance(blocks, list) + assert len(blocks) == 1 + assert blocks[0]["type"] == "web_search_tool_result" + assert blocks[0]["tool_use_id"] == "toolu_one" + assert blocks[0]["content"][0]["url"] == "https://docs.litellm.ai/" + + @pytest.mark.asyncio + async def test_metadata_does_not_carry_blocks_when_flag_absent(self): + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) + tool_calls = [ + { + "id": "toolu_one", + "type": "tool_use", + "name": "litellm_web_search", + "input": {"query": "what is litellm"}, + } + ] + patch_obj = AgenticLoopRequestPatch( + model="bedrock/claude", + messages=[{"role": "user", "content": "hi"}], + max_tokens=1024, + ) + + with patch.object( + logger, + "_build_anthropic_request_patch", + new=AsyncMock(return_value=(patch_obj, [_make_search_response()])), + ): + plan = await logger.async_build_agentic_loop_plan( + tools={"tool_calls": tool_calls, "thinking_blocks": []}, + model="bedrock/claude", + messages=[], + response=MagicMock(), + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={}, + logging_obj=MagicMock(model_call_details={}), + stream=False, + kwargs={}, + ) + + assert WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY not in plan.metadata + + +class TestPostHookInjectsBlocks: + """The post-hook must prepend blocks; absent metadata is a no-op.""" + + @pytest.mark.asyncio + async def test_injects_when_metadata_present(self): + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) + block = WebSearchTransformation.build_web_search_tool_result_block( + tool_use_id="toolu_abc", + search_response=_make_search_response(), + ) + plan = AgenticLoopPlan( + run_agentic_loop=True, + metadata={WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY: [block]}, + ) + response = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "Based on the search..."}], + "stop_reason": "end_turn", + } + + out = await logger.async_post_agentic_loop_response_hook( + response=response, plan=plan, kwargs={} + ) + + # Native block must be first so the client can pair it with the + # tool_use before reading the assistant text. + assert out["content"][0]["type"] == "web_search_tool_result" + assert out["content"][0]["tool_use_id"] == "toolu_abc" + assert out["content"][1]["type"] == "text" + + @pytest.mark.asyncio + async def test_noop_when_metadata_absent(self): + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) + plan = AgenticLoopPlan(run_agentic_loop=True, metadata={}) + response = { + "id": "msg_1", + "content": [{"type": "text", "text": "answer"}], + } + out = await logger.async_post_agentic_loop_response_hook( + response=response, plan=plan, kwargs={} + ) + assert out == response + + @pytest.mark.asyncio + async def test_handles_object_style_response(self): + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) + block = WebSearchTransformation.build_web_search_tool_result_block( + tool_use_id="toolu_obj", + search_response=_make_search_response(), + ) + plan = AgenticLoopPlan( + run_agentic_loop=True, + metadata={WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY: [block]}, + ) + + class _Resp: + def __init__(self): + self.content = [{"type": "text", "text": "ok"}] + + resp = _Resp() + out = await logger.async_post_agentic_loop_response_hook( + response=resp, plan=plan, kwargs={} + ) + assert out.content[0]["type"] == "web_search_tool_result" + assert out.content[1]["type"] == "text" + + +class TestShortCircuitEmitsNativeBlocks: + """Standalone /v1/messages sub-requests (Cowork's separate search call) + hit ``try_short_circuit_search``, which builds a synthetic response and + never enters the agentic loop. The native-block emission must happen + here too, otherwise the citations panel stays empty.""" + + @pytest.mark.asyncio + async def test_native_tool_short_circuit_emits_blocks(self): + logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"]) + + with patch.object( + logger, + "_execute_search", + new=AsyncMock(return_value=("Title: x\nURL: y", _make_search_response())), + ): + result = await logger.try_short_circuit_search( + model="github_copilot/claude-sonnet-4", + messages=[{"role": "user", "content": "search query"}], + tools=[ + { + "type": "web_search_20250305", + "name": "web_search", + "max_uses": 3, + } + ], + custom_llm_provider="github_copilot", + ) + + assert result is not None + block_types = [b["type"] for b in result["content"]] + # Order matters: native clients expect tool_use before tool_result. + assert block_types == ["server_tool_use", "web_search_tool_result", "text"] + server_use, tool_result, _ = result["content"] + assert server_use["name"] == "web_search" + assert server_use["input"] == {"query": "search query"} + # tool_use_id must match between the server_tool_use and the + # web_search_tool_result block so the client can pair them. + assert server_use["id"].startswith("srvtoolu_") + assert tool_result["tool_use_id"] == server_use["id"] + # The actual search results carry through (urls + titles). + assert len(tool_result["content"]) == 2 + assert tool_result["content"][0]["url"] == "https://docs.litellm.ai/" + + @pytest.mark.asyncio + async def test_litellm_standard_tool_short_circuit_stays_text_only(self): + """Non-native tool → existing text-only short-circuit, no regression.""" + logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"]) + + with patch.object( + logger, + "_execute_search", + new=AsyncMock(return_value=("Title: x\nURL: y", _make_search_response())), + ): + result = await logger.try_short_circuit_search( + model="github_copilot/claude-sonnet-4", + messages=[{"role": "user", "content": "search query"}], + tools=[ + { + "name": "litellm_web_search", + "input_schema": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + } + ], + custom_llm_provider="github_copilot", + ) + + assert result is not None + block_types = [b["type"] for b in result["content"]] + assert block_types == ["text"] + + @pytest.mark.asyncio + async def test_native_short_circuit_failure_still_emits_blocks(self): + """Search failure on native path: emit blocks with empty results + + the legacy text-error block, so the client gets a well-formed + response instead of a malformed half-shape.""" + logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"]) + + with patch.object(logger, "_execute_search", side_effect=RuntimeError("boom")): + result = await logger.try_short_circuit_search( + model="github_copilot/claude-sonnet-4", + messages=[{"role": "user", "content": "search query"}], + tools=[{"type": "web_search_20250305", "name": "web_search"}], + custom_llm_provider="github_copilot", + ) + + assert result is not None + block_types = [b["type"] for b in result["content"]] + assert block_types == ["server_tool_use", "web_search_tool_result", "text"] + tool_result = result["content"][1] + assert tool_result["content"] == [] + text_block = result["content"][2] + assert "Search failed" in text_block["text"] + + +class TestLegacyPathMatchesNewPath: + """The legacy ``_execute_agentic_loop`` must inject blocks too.""" + + @pytest.mark.asyncio + async def test_legacy_path_injects_when_flag_set(self): + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) + tool_calls = [ + { + "id": "toolu_legacy", + "type": "tool_use", + "name": "litellm_web_search", + "input": {"query": "q"}, + } + ] + patch_obj = AgenticLoopRequestPatch( + model="bedrock/claude", + messages=[{"role": "user", "content": "hi"}], + max_tokens=1024, + optional_params={}, + ) + followup_response = { + "id": "msg_followup", + "content": [{"type": "text", "text": "final answer"}], + } + + with ( + patch.object( + logger, + "_build_anthropic_request_patch", + new=AsyncMock(return_value=(patch_obj, [_make_search_response()])), + ), + patch( + "litellm.integrations.websearch_interception.handler.anthropic_messages.acreate", + new=AsyncMock(return_value=followup_response), + ), + ): + out = await logger._execute_agentic_loop( + model="bedrock/claude", + messages=[], + tool_calls=tool_calls, + thinking_blocks=[], + anthropic_messages_optional_request_params={}, + logging_obj=MagicMock(model_call_details={}), + stream=False, + kwargs={WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: True}, + ) + + assert out["content"][0]["type"] == "web_search_tool_result" + assert out["content"][0]["tool_use_id"] == "toolu_legacy" + assert out["content"][1]["type"] == "text" diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_short_circuit.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_short_circuit.py index 82c1c9839e..7de8892b8f 100644 --- a/tests/test_litellm/integrations/websearch_interception/test_websearch_short_circuit.py +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_short_circuit.py @@ -30,7 +30,8 @@ class TestTryShortCircuitSearch: logger, "_execute_search", new_callable=AsyncMock ) as mock_search: mock_search.return_value = ( - "Title: Result\nURL: https://example.com\nSnippet: test" + "Title: Result\nURL: https://example.com\nSnippet: test", + None, ) result = await logger.try_short_circuit_search( @@ -48,9 +49,15 @@ class TestTryShortCircuitSearch: assert result["type"] == "message" assert result["role"] == "assistant" assert result["stop_reason"] == "end_turn" - assert len(result["content"]) == 1 - assert result["content"][0]["type"] == "text" - assert "Result" in result["content"][0]["text"] + # Native web_search_20250305 client → short-circuit emits native + # blocks (server_tool_use + web_search_tool_result) plus the legacy + # text block so Cowork / Claude Desktop citations panels populate. + block_types = [b["type"] for b in result["content"]] + assert "server_tool_use" in block_types + assert "web_search_tool_result" in block_types + assert "text" in block_types + text_block = next(b for b in result["content"] if b["type"] == "text") + assert "Result" in text_block["text"] mock_search.assert_called_once_with("Search for Claude Code releases") @pytest.mark.asyncio @@ -173,7 +180,8 @@ class TestTryShortCircuitSearch: ) assert result is not None - assert "Search failed" in result["content"][0]["text"] + text_block = next(b for b in result["content"] if b["type"] == "text") + assert "Search failed" in text_block["text"] @pytest.mark.asyncio async def test_response_has_valid_structure(self): @@ -183,7 +191,7 @@ class TestTryShortCircuitSearch: with patch.object( logger, "_execute_search", new_callable=AsyncMock ) as mock_search: - mock_search.return_value = "search results here" + mock_search.return_value = ("search results here", None) result = await logger.try_short_circuit_search( model="github_copilot/claude-sonnet-4", @@ -246,7 +254,7 @@ class TestShortCircuitEntryPoint: with patch.object( logger, "_execute_search", new_callable=AsyncMock ) as mock_search: - mock_search.return_value = "results" + mock_search.return_value = ("results", None) with patch("litellm.callbacks", [logger]): result = await _try_websearch_short_circuit( model="github_copilot/claude-sonnet-4", @@ -257,7 +265,8 @@ class TestShortCircuitEntryPoint: ) assert isinstance(result, dict) - assert result["content"][0]["text"] == "results" + text_block = next(b for b in result["content"] if b["type"] == "text") + assert text_block["text"] == "results" @pytest.mark.asyncio async def test_returns_stream_iterator_when_streaming(self): @@ -273,7 +282,7 @@ class TestShortCircuitEntryPoint: with patch.object( logger, "_execute_search", new_callable=AsyncMock ) as mock_search: - mock_search.return_value = "streaming results" + mock_search.return_value = ("streaming results", None) with patch("litellm.callbacks", [logger]): result = await _try_websearch_short_circuit( model="github_copilot/claude-sonnet-4", @@ -338,7 +347,7 @@ class TestShortCircuitEntryPoint: with patch.object( logger, "_execute_search", new_callable=AsyncMock ) as mock_search: - mock_search.return_value = "streaming results" + mock_search.return_value = ("streaming results", None) with patch("litellm.callbacks", [logger]): # Simulate what anthropic_messages() does: original_stream=True # is passed to the short-circuit, even though the hook would have @@ -368,7 +377,7 @@ class TestShortCircuitEntryPoint: with patch.object( logger, "_execute_search", new_callable=AsyncMock ) as mock_search: - mock_search.return_value = "results" + mock_search.return_value = ("results", None) with patch("litellm.callbacks", [logger]): # Simulate the caller having derived custom_llm_provider from # the model string before calling _try_websearch_short_circuit @@ -381,4 +390,5 @@ class TestShortCircuitEntryPoint: ) assert result is not None - assert result["content"][0]["text"] == "results" + text_block = next(b for b in result["content"] if b["type"] == "text") + assert text_block["text"] == "results" diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_thinking_constraint.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_thinking_constraint.py index a939951c43..b2d5225070 100644 --- a/tests/test_litellm/integrations/websearch_interception/test_websearch_thinking_constraint.py +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_thinking_constraint.py @@ -68,7 +68,9 @@ class TestThinkingBudgetTokensConstraint: "litellm.integrations.websearch_interception.handler.anthropic_messages.acreate", side_effect=_fake_acreate, ), - patch.object(logger, "_execute_search", return_value="search result"), + patch.object( + logger, "_execute_search", return_value=("search result", None) + ), ): await logger._execute_agentic_loop( @@ -102,7 +104,9 @@ class TestThinkingBudgetTokensConstraint: "litellm.integrations.websearch_interception.handler.anthropic_messages.acreate", side_effect=_fake_acreate, ), - patch.object(logger, "_execute_search", return_value="search result"), + patch.object( + logger, "_execute_search", return_value=("search result", None) + ), ): await logger._execute_agentic_loop( @@ -136,7 +140,9 @@ class TestThinkingBudgetTokensConstraint: "litellm.integrations.websearch_interception.handler.anthropic_messages.acreate", side_effect=_fake_acreate, ), - patch.object(logger, "_execute_search", return_value="search result"), + patch.object( + logger, "_execute_search", return_value=("search result", None) + ), ): await logger._execute_agentic_loop( @@ -170,7 +176,9 @@ class TestThinkingBudgetTokensConstraint: "litellm.integrations.websearch_interception.handler.anthropic_messages.acreate", side_effect=_fake_acreate, ), - patch.object(logger, "_execute_search", return_value="search result"), + patch.object( + logger, "_execute_search", return_value=("search result", None) + ), ): await logger._execute_agentic_loop( @@ -201,7 +209,9 @@ class TestThinkingBudgetTokensConstraint: "litellm.integrations.websearch_interception.handler.anthropic_messages.acreate", side_effect=_fake_acreate, ), - patch.object(logger, "_execute_search", return_value="search result"), + patch.object( + logger, "_execute_search", return_value=("search result", None) + ), ): await logger._execute_agentic_loop( @@ -286,7 +296,9 @@ class TestLoggingObjExcludedFromFollowUp: "litellm.integrations.websearch_interception.handler.anthropic_messages.acreate", side_effect=_fake_acreate, ), - patch.object(logger, "_execute_search", return_value="search result"), + patch.object( + logger, "_execute_search", return_value=("search result", None) + ), ): await logger._execute_agentic_loop( @@ -325,7 +337,9 @@ class TestLoggingObjExcludedFromFollowUp: "litellm.integrations.websearch_interception.handler.anthropic_messages.acreate", side_effect=_fake_acreate, ), - patch.object(logger, "_execute_search", return_value="search result"), + patch.object( + logger, "_execute_search", return_value=("search result", None) + ), ): await logger._execute_agentic_loop( @@ -373,7 +387,9 @@ class TestFollowUpErrorScenarios: "litellm.integrations.websearch_interception.handler.anthropic_messages.acreate", side_effect=_fail_acreate, ), - patch.object(logger, "_execute_search", return_value="search result"), + patch.object( + logger, "_execute_search", return_value=("search result", None) + ), ): with pytest.raises(Exception, match="max_tokens must be greater"): @@ -450,7 +466,9 @@ class TestFollowUpErrorScenarios: "litellm.integrations.websearch_interception.handler.anthropic_messages.acreate", side_effect=_fake_acreate, ), - patch.object(logger, "_execute_search", return_value="search result"), + patch.object( + logger, "_execute_search", return_value=("search result", None) + ), ): await logger._execute_agentic_loop( From 3f6c0090c0fee3cf2d6328889e3a31bbc6c4a714 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 14 May 2026 13:43:45 -0700 Subject: [PATCH 11/11] chore(ci): remove unused GitHub Actions workflows and orphan files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audit of .github/workflows/ via gh run history shows the following have either never run or have been dormant for 10+ weeks. CI coverage that still matters is preserved on CircleCI (e.g. llm_translation_testing). Removed workflows: - test-litellm.yml — workflow_dispatch only, last run 2026-02-12 (cancelled); CCI local_testing_part1/2 covers the same tests - llm-translation-testing.yml — last run 2025-07-10; replaced by CCI llm_translation_testing job (run_llm_translation_tests.py kept for the make test-llm-translation target) - run_observatory_tests.yml — last run 2026-03-03 (cancelled) - scan_duplicate_issues.yml — last run 2026-03-02 (failure) - publish_to_pypi.yml — never run - read_pyproject_version.yml — fires on every push to main but its echoed version output is not consumed by any downstream step Removed orphan files (no callers in workflows, CCI, or Makefile): - .github/workflows/README.md — documented only publish_to_pypi.yml - .github/workflows/update_release.py + results_stats.csv - .github/actions/helm-oci-chart-releaser/ --- .../helm-oci-chart-releaser/action.yml | 94 ------- .github/workflows/README.md | 35 --- .github/workflows/llm-translation-testing.yml | 92 ------- .github/workflows/publish_to_pypi.yml | 153 ------------ .github/workflows/read_pyproject_version.yml | 28 --- .github/workflows/results_stats.csv | 27 --- .github/workflows/run_observatory_tests.yml | 229 ------------------ .github/workflows/scan_duplicate_issues.yml | 48 ---- .github/workflows/test-litellm.yml | 45 ---- .github/workflows/update_release.py | 54 ----- 10 files changed, 805 deletions(-) delete mode 100644 .github/actions/helm-oci-chart-releaser/action.yml delete mode 100644 .github/workflows/README.md delete mode 100644 .github/workflows/llm-translation-testing.yml delete mode 100644 .github/workflows/publish_to_pypi.yml delete mode 100644 .github/workflows/read_pyproject_version.yml delete mode 100644 .github/workflows/results_stats.csv delete mode 100644 .github/workflows/run_observatory_tests.yml delete mode 100644 .github/workflows/scan_duplicate_issues.yml delete mode 100644 .github/workflows/test-litellm.yml delete mode 100644 .github/workflows/update_release.py diff --git a/.github/actions/helm-oci-chart-releaser/action.yml b/.github/actions/helm-oci-chart-releaser/action.yml deleted file mode 100644 index 454c591d43..0000000000 --- a/.github/actions/helm-oci-chart-releaser/action.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: Helm OCI Chart Releaser -description: Push Helm charts to OCI-based (Docker) registries -author: sergeyshaykhullin -branding: - color: yellow - icon: upload-cloud -inputs: - name: - required: true - description: Chart name - repository: - required: true - description: Chart repository name - tag: - required: true - description: Chart version - app_version: - required: true - description: App version - path: - required: false - description: Chart path (Default 'charts/{name}') - registry: - required: true - description: OCI registry - registry_username: - required: true - description: OCI registry username - registry_password: - required: true - description: OCI registry password - update_dependencies: - required: false - default: 'false' - description: Update chart dependencies before packaging (Default 'false') -outputs: - image: - value: ${{ steps.output.outputs.image }} - description: Chart image (Default '{registry}/{repository}/{image}:{tag}') -runs: - using: composite - steps: - - name: Helm | Setup - uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 - with: - version: v3.20.0 - - - name: Helm | Login - shell: bash - env: - REGISTRY_PASSWORD: ${{ inputs.registry_password }} - REGISTRY_USERNAME: ${{ inputs.registry_username }} - REGISTRY: ${{ inputs.registry }} - run: echo "$REGISTRY_PASSWORD" | helm registry login -u "$REGISTRY_USERNAME" --password-stdin "$REGISTRY" - - - name: Helm | Dependency - if: inputs.update_dependencies == 'true' - shell: bash - env: - CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} - run: helm dependency update "$CHART_PATH" - - - name: Helm | Package - shell: bash - env: - CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} - TAG: ${{ inputs.tag }} - APP_VERSION: ${{ inputs.app_version }} - run: helm package "$CHART_PATH" --version "$TAG" --app-version "$APP_VERSION" - - - name: Helm | Push - shell: bash - env: - NAME: ${{ inputs.name }} - TAG: ${{ inputs.tag }} - REGISTRY: ${{ inputs.registry }} - REPOSITORY: ${{ inputs.repository }} - run: helm push "${NAME}-${TAG}.tgz" "oci://${REGISTRY}/${REPOSITORY}" - - - name: Helm | Logout - shell: bash - env: - REGISTRY: ${{ inputs.registry }} - run: helm registry logout "$REGISTRY" - - - name: Helm | Output - id: output - shell: bash - env: - REGISTRY: ${{ inputs.registry }} - REPOSITORY: ${{ inputs.repository }} - NAME: ${{ inputs.name }} - TAG: ${{ inputs.tag }} - run: echo "image=${REGISTRY}/${REPOSITORY}/${NAME}:${TAG}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/README.md b/.github/workflows/README.md deleted file mode 100644 index b4e777969d..0000000000 --- a/.github/workflows/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Simple PyPI Publishing - -A GitHub workflow to manually publish LiteLLM packages to PyPI with a specified version. - -## How to Use - -1. Go to the **Actions** tab in the GitHub repository -2. Select **Simple PyPI Publish** from the workflow list -3. Click **Run workflow** -4. Enter the version to publish (e.g., `1.74.10`) - -## What the Workflow Does - -1. **Updates** the version in `pyproject.toml` -2. **Copies** the model prices backup file -3. **Builds** the Python package -4. **Publishes** to PyPI - -## Prerequisites - -Make sure the following secret is configured in the repository: -- `PYPI_PUBLISH_PASSWORD`: PyPI API token for authentication - -## Example Usage - -- Version: `1.74.11` → Publishes as v1.74.11 -- Version: `1.74.10-hotfix1` → Publishes as v1.74.10-hotfix1 - -## Features - -- ✅ Manual trigger with version input -- ✅ Automatic version updates in `pyproject.toml` -- ✅ Repository safety check (only runs on official repo) -- ✅ Clean package building and publishing -- ✅ Success confirmation with PyPI package link \ No newline at end of file diff --git a/.github/workflows/llm-translation-testing.yml b/.github/workflows/llm-translation-testing.yml deleted file mode 100644 index 8d9d52f4e5..0000000000 --- a/.github/workflows/llm-translation-testing.yml +++ /dev/null @@ -1,92 +0,0 @@ -name: LLM Translation Tests - -on: - workflow_dispatch: - inputs: - release_candidate_tag: - description: "Release candidate tag/version" - required: true - type: string - push: - tags: - - "v*-rc*" # Triggers on release candidate tags like v1.0.0-rc1 - -permissions: - contents: read - -jobs: - run-llm-translation-tests: - runs-on: ubuntu-latest - timeout-minutes: 90 - - steps: - - name: Checkout code - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - ref: ${{ github.event.inputs.release_candidate_tag || github.ref }} - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Set up uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - version: "0.10.9" - enable-cache: false - - - name: Restore uv dependencies cache - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: | - ~/.cache/uv - .venv - key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} - restore-keys: | - ${{ runner.os }}-uv- - - - name: Install dependencies - run: | - uv sync --frozen - - - name: Create test results directory - run: mkdir -p test-results - - - name: Run LLM Translation Tests - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} - GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} - AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }} - AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }} - RC_TAG: ${{ github.event.inputs.release_candidate_tag || github.ref_name }} - COMMIT_SHA: ${{ github.sha }} - run: | - python .github/workflows/run_llm_translation_tests.py \ - --tag "$RC_TAG" \ - --commit "$COMMIT_SHA" \ - || true # Continue even if tests fail - - - name: Display test summary - if: always() - run: | - if [ -f "test-results/llm_translation_report.md" ]; then - echo "Test report generated successfully!" - echo "Artifact will contain:" - echo "- test-results/junit.xml (JUnit XML results)" - echo "- test-results/llm_translation_report.md (Beautiful markdown report)" - else - echo "Warning: Test report was not generated" - fi - - - name: Upload test artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - if: always() - with: - name: LLM-Translation-Artifact-${{ github.event.inputs.release_candidate_tag || github.ref_name }} - path: test-results/ - retention-days: 30 diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml deleted file mode 100644 index d60254a0ac..0000000000 --- a/.github/workflows/publish_to_pypi.yml +++ /dev/null @@ -1,153 +0,0 @@ -name: Publish to PyPI - -on: - workflow_dispatch: - -jobs: - preflight-checks: - name: Preflight Checks - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - # No environment — read-only checks, no approval needed - outputs: - needs_publish: ${{ steps.check-litellm.outputs.needs_publish }} - version: ${{ steps.check-litellm.outputs.version }} - - steps: - - name: Checkout repo - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Set up uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - version: "0.10.9" - enable-cache: false - - - name: Check litellm version on PyPI - id: check-litellm - run: | - VERSION=$(python - <<'PY' - import tomllib - - with open("pyproject.toml", "rb") as f: - print(tomllib.load(f)["project"]["version"]) - PY - ) - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - echo "Checking if litellm $VERSION exists on PyPI..." - - HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm/$VERSION/json") - if [ "$HTTP_STATUS" = "200" ]; then - echo "litellm $VERSION already exists on PyPI. Skipping publish." - echo "needs_publish=false" >> "$GITHUB_OUTPUT" - else - echo "litellm $VERSION not found on PyPI. Publish needed." - echo "needs_publish=true" >> "$GITHUB_OUTPUT" - fi - - - name: Sanity check proxy-extras version - run: | - # Read pinned version from project optional dependencies - PYPROJECT_VERSION=$(python3 - <<'PY' - import sys - import tomllib - - with open("pyproject.toml", "rb") as f: - proxy_requirements = tomllib.load(f)["project"]["optional-dependencies"]["proxy"] - - version = None - for requirement in proxy_requirements: - normalized = requirement.split(";", 1)[0].strip() - if not normalized.startswith("litellm-proxy-extras"): - continue - parts = normalized.split("==", 1) - if len(parts) == 2 and parts[0].strip() == "litellm-proxy-extras": - candidate = parts[1].strip() - if candidate: - version = candidate - break - - if version is None: - print( - "::error::Could not find an exact litellm-proxy-extras pin in project.optional-dependencies.proxy", - file=sys.stderr, - ) - sys.exit(1) - - print(version) - PY - ) - echo "pyproject.toml pins litellm-proxy-extras version: $PYPROJECT_VERSION" - - # Check that the pinned version exists on PyPI - echo "Checking if litellm-proxy-extras $PYPROJECT_VERSION exists on PyPI..." - HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm-proxy-extras/$PYPROJECT_VERSION/json") - if [ "$HTTP_STATUS" != "200" ]; then - echo "::error::litellm-proxy-extras $PYPROJECT_VERSION is not published on PyPI yet. Publish it before releasing litellm." - exit 1 - fi - echo "litellm-proxy-extras $PYPROJECT_VERSION exists on PyPI. Sanity check passed." - - publish-litellm: - name: Publish litellm to PyPI - needs: preflight-checks - if: needs.preflight-checks.outputs.needs_publish == 'true' - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - id-token: write - contents: read - environment: pypi-publish - - steps: - - name: Checkout repo - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Set up uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - version: "0.10.9" - enable-cache: false - - - name: Copy model prices backup - run: cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json - - - name: Build package - run: | - rm -rf build dist - uv build - - - name: Verify build artifacts - env: - EXPECTED_VERSION: ${{ needs.preflight-checks.outputs.version }} - run: | - echo "Contents of dist/:" - ls -la dist/ - # Ensure we have both sdist and wheel - ls dist/*.tar.gz - ls dist/*.whl - # Verify built version matches expected - ls dist/ | grep -q "litellm-${EXPECTED_VERSION}" || { - echo "::error::Built artifacts do not match expected version $EXPECTED_VERSION" - ls dist/ - exit 1 - } - - - name: Validate package metadata - run: | - uv tool run --from 'twine==6.2.0' twine check dist/* - - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 diff --git a/.github/workflows/read_pyproject_version.yml b/.github/workflows/read_pyproject_version.yml deleted file mode 100644 index 04b4a38ce1..0000000000 --- a/.github/workflows/read_pyproject_version.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Read Version from pyproject.toml - -on: - push: - branches: - - main # Change this to the default branch of your repository - -permissions: - contents: read - -jobs: - read-version: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Read version from pyproject.toml - id: read-version - run: | - version=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/') - printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV - - - name: Display version - run: echo "Current version is $LITELLM_VERSION" diff --git a/.github/workflows/results_stats.csv b/.github/workflows/results_stats.csv deleted file mode 100644 index bcef047b0f..0000000000 --- a/.github/workflows/results_stats.csv +++ /dev/null @@ -1,27 +0,0 @@ -Date,"Ben -Ashley",Tom Brooks,Jimmy Cooney,"Sue -Daniels",Berlinda Fong,Terry Jones,Angelina Little,Linda Smith -10/1,FALSE,TRUE,TRUE,TRUE,TRUE,TRUE,FALSE,TRUE -10/2,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/3,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/4,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/5,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/6,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/7,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/8,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/9,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/10,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/11,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/12,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/13,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/14,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/15,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/16,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/17,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/18,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/19,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/20,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/21,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/22,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -10/23,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE -Total,0,1,1,1,1,1,0,1 \ No newline at end of file diff --git a/.github/workflows/run_observatory_tests.yml b/.github/workflows/run_observatory_tests.yml deleted file mode 100644 index a25b96766d..0000000000 --- a/.github/workflows/run_observatory_tests.yml +++ /dev/null @@ -1,229 +0,0 @@ -name: Run Observatory Tests -on: - workflow_dispatch: - inputs: - tag: - description: "Docker image tag to test (e.g. v1.61.0.rc1)" - required: true - type: string - commit_hash: - description: "Commit hash (defaults to HEAD of current branch)" - required: false - type: string - workflow_call: - inputs: - tag: - description: "Docker image tag to test" - required: true - type: string - commit_hash: - description: "Commit hash of the release" - required: true - type: string - -permissions: - contents: read - -env: - LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY_STAGING }} - -jobs: - observatory-tests: - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - name: Checkout repository - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Validate tag input - env: - TAG: ${{ inputs.tag }} - run: | - if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then - echo "Invalid tag format: $TAG (expected vX.Y.Z...)" - exit 1 - fi - - - name: Start LiteLLM container - env: - TAG: ${{ inputs.tag }} - AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} - AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }} - WORKSPACE: ${{ github.workspace }} - run: | - docker run -d \ - --name litellm-rc \ - -p 4000:4000 \ - -v "${WORKSPACE}/.github/observatory/litellm_config.yaml:/app/config.yaml" \ - -e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \ - -e AZURE_API_KEY="${AZURE_API_KEY}" \ - -e AZURE_API_BASE="${AZURE_API_BASE}" \ - "litellm/litellm:${TAG}" \ - --config /app/config.yaml --port 4000 - - - name: Wait for LiteLLM health check - run: | - echo "Waiting for LiteLLM to be ready..." - for i in $(seq 1 30); do - if curl -s -f http://localhost:4000/health/liveliness > /dev/null 2>&1; then - echo "LiteLLM is healthy" - exit 0 - fi - echo "Attempt $i/30 - not ready yet, waiting 10s..." - sleep 10 - done - echo "LiteLLM failed to start within 5 minutes" - docker logs litellm-rc - exit 1 - - - name: Start cloudflared tunnel - run: | - # Install cloudflared (pinned version + checksum) - curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared - echo "afdfadd1ef552e66bffc35246fe30a9bd578356d2d386de95585ccfc432472b8 /usr/local/bin/cloudflared" | sha256sum -c - - chmod +x /usr/local/bin/cloudflared - - # Start a quick tunnel (no account needed) and capture the URL - cloudflared tunnel --url http://localhost:4000 --no-autoupdate > /tmp/cloudflared.log 2>&1 & - CLOUDFLARED_PID=$! - echo "CLOUDFLARED_PID=$CLOUDFLARED_PID" >> $GITHUB_ENV - - # Wait for tunnel URL to appear in logs - echo "Waiting for tunnel URL..." - for i in $(seq 1 30); do - TUNNEL_URL=$(grep -oP 'https://[a-z0-9-]+\.trycloudflare\.com' /tmp/cloudflared.log | head -1 || true) - if [ -n "$TUNNEL_URL" ]; then - echo "Tunnel URL: $TUNNEL_URL" - echo "TUNNEL_URL=$TUNNEL_URL" >> $GITHUB_ENV - exit 0 - fi - sleep 2 - done - echo "Failed to get tunnel URL" - cat /tmp/cloudflared.log - exit 1 - - - name: Verify tunnel connectivity - run: | - echo "Testing tunnel at ${TUNNEL_URL}..." - # Quick tunnels need time for DNS propagation; retry to avoid - # transient NXDOMAIN (curl exit code 6) on first attempt. - for i in $(seq 1 10); do - if curl -sf "${TUNNEL_URL}/health/liveliness" > /dev/null 2>&1; then - echo "Tunnel is working (attempt $i)" - exit 0 - fi - echo "Attempt $i/10 - tunnel not routable yet, waiting 5s..." - sleep 5 - done - echo "Tunnel failed to become reachable after 50s" - cat /tmp/cloudflared.log - exit 1 - - - name: Trigger observatory test run - id: trigger - env: - OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }} - OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }} - run: | - PAYLOAD=$(jq -n \ - --arg url "${TUNNEL_URL}" \ - --arg key "${LITELLM_MASTER_KEY}" \ - '{ - deployment_url: $url, - api_key: $key, - test_suite: "TestOAIAzureRelease", - models: ["gpt-4o-mini", "gpt-4o"] - }') - RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${OBSERVATORY_URL}/run-test" \ - -H "Content-Type: application/json" \ - -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}" \ - -d "$PAYLOAD") - HTTP_CODE=$(echo "$RESPONSE" | tail -1) - BODY=$(echo "$RESPONSE" | head -n -1) - echo "Response ($HTTP_CODE): $BODY" - if [ "$HTTP_CODE" -ge 400 ]; then - echo "Failed to trigger test run" - exit 1 - fi - - # Extract request_id for polling this specific run - REQUEST_ID=$(echo "$BODY" | jq -r '.results.request_id') - if [ -z "$REQUEST_ID" ] || [ "$REQUEST_ID" = "null" ]; then - echo "Failed to extract request_id from response" - exit 1 - fi - echo "Request ID: $REQUEST_ID" - echo "request_id=$REQUEST_ID" >> $GITHUB_OUTPUT - - - name: Poll for test completion - id: poll - env: - OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }} - OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }} - REQUEST_ID: ${{ steps.trigger.outputs.request_id }} - run: | - TIMEOUT=900 # 15 minutes - INTERVAL=30 - ELAPSED=0 - while [ $ELAPSED -lt $TIMEOUT ]; do - STATUS=$(curl -s "${OBSERVATORY_URL}/run-status/${REQUEST_ID}" \ - -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}") - RUN_STATUS=$(echo "$STATUS" | jq -r '.status') - echo "Run status (${ELAPSED}s elapsed): $RUN_STATUS" - - if [ "$RUN_STATUS" = "completed" ] || [ "$RUN_STATUS" = "failed" ]; then - echo "Test finished with status: $RUN_STATUS" - echo "$STATUS" > /tmp/observatory_result.json - exit 0 - fi - - sleep $INTERVAL - ELAPSED=$((ELAPSED + INTERVAL)) - done - echo "Timed out waiting for test to complete after ${TIMEOUT}s" - exit 1 - - - name: Verify test results - run: | - RESULT=$(cat /tmp/observatory_result.json) - echo "Full result: $RESULT" - - STATUS=$(echo "$RESULT" | jq -r '.status') - TEST_PASSED=$(echo "$RESULT" | jq -r '.result.test_passed // false') - FAILURE_RATE=$(echo "$RESULT" | jq -r '.result.failure_rate // "N/A"') - ERROR=$(echo "$RESULT" | jq -r '.error // empty') - - echo "Status: $STATUS" - echo "Test passed: $TEST_PASSED" - echo "Failure rate: $FAILURE_RATE" - - if [ -n "$ERROR" ]; then - echo "Error: $ERROR" - fi - - if [ "$STATUS" = "failed" ]; then - echo "Test run failed" - exit 1 - fi - - if [ "$TEST_PASSED" != "true" ]; then - echo "Tests did not pass (failure rate: $FAILURE_RATE)" - exit 1 - fi - - echo "All tests passed!" - - - name: Print LiteLLM logs on failure - if: failure() - run: | - docker logs litellm-rc 2>/dev/null || true - cat /tmp/cloudflared.log 2>/dev/null || true - - - name: Cleanup - if: always() - run: | - kill "$CLOUDFLARED_PID" 2>/dev/null || true - docker rm -f litellm-rc 2>/dev/null || true diff --git a/.github/workflows/scan_duplicate_issues.yml b/.github/workflows/scan_duplicate_issues.yml deleted file mode 100644 index ab0ac2aa3a..0000000000 --- a/.github/workflows/scan_duplicate_issues.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Scan Duplicate Issues (One-Time) - -on: - workflow_dispatch: - inputs: - threshold: - description: "Similarity threshold (0-1)" - required: false - default: "0.85" - close: - description: "Actually close duplicates (false = dry run)" - required: false - type: boolean - default: false - -jobs: - scan: - runs-on: ubuntu-latest - permissions: - issues: write - contents: read - steps: - - name: Checkout scripts - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - sparse-checkout: .github/scripts - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Scan for duplicate issues - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - INPUT_THRESHOLD: ${{ inputs.threshold }} - INPUT_CLOSE: ${{ inputs.close }} - run: | - CLOSE_FLAG="" - if [ "$INPUT_CLOSE" = "true" ]; then - CLOSE_FLAG="--close" - fi - python3 .github/scripts/close_duplicate_issues.py \ - --scan \ - --repo ${{ github.repository }} \ - --threshold "$INPUT_THRESHOLD" \ - $CLOSE_FLAG diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml deleted file mode 100644 index 938647f5d0..0000000000 --- a/.github/workflows/test-litellm.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: LiteLLM Mock Tests (folder - tests/test_litellm) - -# DEPRECATED: This workflow is replaced by test-litellm-matrix.yml which runs -# the same tests in parallel across 10 jobs for faster CI times. -# Kept for manual debugging only. -on: - workflow_dispatch: # Manual trigger only - # pull_request: - # branches: [ main ] - -permissions: - contents: read - -jobs: - test: - runs-on: ubuntu-latest - timeout-minutes: 25 - - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Thank You Message - run: | - echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY - echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Set up uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - version: "0.10.9" - - - name: Install dependencies - run: | - uv lock --check - uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router - - name: Run tests - run: | - uv run --no-sync pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50 diff --git a/.github/workflows/update_release.py b/.github/workflows/update_release.py deleted file mode 100644 index f70509e8e7..0000000000 --- a/.github/workflows/update_release.py +++ /dev/null @@ -1,54 +0,0 @@ -import os -import requests -from datetime import datetime - -# GitHub API endpoints -GITHUB_API_URL = "https://api.github.com" -REPO_OWNER = "BerriAI" -REPO_NAME = "litellm" - -# GitHub personal access token (required for uploading release assets) -GITHUB_ACCESS_TOKEN = os.environ.get("GITHUB_ACCESS_TOKEN") - -# Headers for GitHub API requests -headers = { - "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {GITHUB_ACCESS_TOKEN}", - "X-GitHub-Api-Version": "2022-11-28", -} - -# Get the latest release -releases_url = f"{GITHUB_API_URL}/repos/{REPO_OWNER}/{REPO_NAME}/releases/latest" -response = requests.get(releases_url, headers=headers) -latest_release = response.json() -print("Latest release:", latest_release) - -# Upload an asset to the latest release -upload_url = latest_release["upload_url"].split("{?")[0] -asset_name = "results_stats.csv" -asset_path = os.path.join(os.getcwd(), asset_name) -print("upload_url:", upload_url) - -with open(asset_path, "rb") as asset_file: - asset_data = asset_file.read() - -upload_payload = { - "name": asset_name, - "label": "Load test results", - "created_at": datetime.utcnow().isoformat() + "Z", -} - -upload_headers = headers.copy() -upload_headers["Content-Type"] = "application/octet-stream" - -upload_response = requests.post( - upload_url, - headers=upload_headers, - data=asset_data, - params=upload_payload, -) - -if upload_response.status_code == 201: - print(f"Asset '{asset_name}' uploaded successfully to the latest release.") -else: - print(f"Failed to upload asset. Response: {upload_response.text}")