From 17f6238d2b429ed36ac544e8a4a0f6348ef8f1ae Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 Oct 2025 13:45:21 -0700 Subject: [PATCH] [Feat] OTEL - Ensure error information is logged on OTEL (#15978) * fix _record_exception_on_span * _record_exception_on_span * test_record_exception_on_span * fix linting errors --- litellm/integrations/_types/open_inference.py | 39 +++++++ litellm/integrations/opentelemetry.py | 85 ++++++++++++++ .../unified_guardrail/unified_guardrail.py | 6 +- .../test_opentelemetry_unit_tests.py | 105 ++++++++++++++++++ 4 files changed, 231 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/_types/open_inference.py b/litellm/integrations/_types/open_inference.py index 65ecadcf37..af2ff2347c 100644 --- a/litellm/integrations/_types/open_inference.py +++ b/litellm/integrations/_types/open_inference.py @@ -387,3 +387,42 @@ class OpenInferenceLLMProviderValues(Enum): GOOGLE = "google" AZURE = "azure" AWS = "aws" + + +class ErrorAttributes: + """ + Attributes for error information in spans. + + These attributes follow OpenTelemetry semantic conventions for exceptions + and are used to record error information from StandardLoggingPayloadErrorInformation. + """ + + ERROR_TYPE = "error.type" + """ + The type/class of the error (e.g., 'ValueError', 'OpenAIError', 'RateLimitError'). + Corresponds to StandardLoggingPayloadErrorInformation.error_class + """ + + ERROR_MESSAGE = "error.message" + """ + The error message describing what went wrong. + Corresponds to StandardLoggingPayloadErrorInformation.error_message + """ + + ERROR_CODE = "error.code" + """ + The error code (e.g., HTTP status code like '500', '429', or provider-specific codes). + Corresponds to StandardLoggingPayloadErrorInformation.error_code + """ + + ERROR_STACK_TRACE = "error.stack_trace" + """ + The full stack trace of the error. + Corresponds to StandardLoggingPayloadErrorInformation.traceback + """ + + ERROR_LLM_PROVIDER = "error.llm_provider" + """ + The LLM provider where the error occurred (e.g., 'openai', 'anthropic', 'azure'). + Corresponds to StandardLoggingPayloadErrorInformation.llm_provider + """ diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 11f2c6b261..9315384ad9 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -841,6 +841,10 @@ class OpenTelemetry(CustomLogger): ) span.set_status(Status(StatusCode.ERROR)) self.set_attributes(span, kwargs, response_obj) + + # Record exception information using OTEL standard method + self._record_exception_on_span(span=span, kwargs=kwargs) + span.end(end_time=self._to_ns(end_time)) # Create span for guardrail information @@ -849,6 +853,87 @@ class OpenTelemetry(CustomLogger): if parent_otel_span is not None: parent_otel_span.end(end_time=self._to_ns(datetime.now())) + def _record_exception_on_span(self, span: Span, kwargs: dict): + """ + Record exception information on the span using OTEL standard methods. + + This extracts error information from StandardLoggingPayload and: + 1. Uses span.record_exception() for the actual exception object (OTEL standard) + 2. Sets structured error attributes from StandardLoggingPayloadErrorInformation + """ + try: + from litellm.integrations._types.open_inference import ErrorAttributes + + # Get the exception object if available + exception = kwargs.get("exception") + + # Record the exception using OTEL's standard method + if exception is not None: + span.record_exception(exception) + + # Get StandardLoggingPayload for structured error information + standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get( + "standard_logging_object" + ) + + if standard_logging_payload is None: + return + + # Extract error_information from StandardLoggingPayload + error_information = standard_logging_payload.get("error_information") + + if error_information is None: + # Fallback to error_str if error_information is not available + error_str = standard_logging_payload.get("error_str") + if error_str: + self.safe_set_attribute( + span=span, + key=ErrorAttributes.ERROR_MESSAGE, + value=error_str, + ) + return + + # Set structured error attributes from StandardLoggingPayloadErrorInformation + if error_information.get("error_code"): + self.safe_set_attribute( + span=span, + key=ErrorAttributes.ERROR_CODE, + value=error_information["error_code"], + ) + + if error_information.get("error_class"): + self.safe_set_attribute( + span=span, + key=ErrorAttributes.ERROR_TYPE, + value=error_information["error_class"], + ) + + if error_information.get("error_message"): + self.safe_set_attribute( + span=span, + key=ErrorAttributes.ERROR_MESSAGE, + value=error_information["error_message"], + ) + + if error_information.get("llm_provider"): + self.safe_set_attribute( + span=span, + key=ErrorAttributes.ERROR_LLM_PROVIDER, + value=error_information["llm_provider"], + ) + + if error_information.get("traceback"): + self.safe_set_attribute( + span=span, + key=ErrorAttributes.ERROR_STACK_TRACE, + value=error_information["traceback"], + ) + + except Exception as e: + verbose_logger.exception( + "OpenTelemetry: Error recording exception on span: %s", str(e) + ) + def set_tools_attributes(self, span: Span, tools): import json diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index 0bd1589ed1..5355f308fd 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -13,15 +13,13 @@ from litellm.caching.caching import DualCache from litellm.cost_calculator import _infer_call_type from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger -from litellm.llms import ( - endpoint_guardrail_translation_mappings, - load_guardrail_translation_mappings, -) +from litellm.llms import load_guardrail_translation_mappings from litellm.proxy._types import UserAPIKeyAuth from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import CallTypes, ModelResponseStream GUARDRAIL_NAME = "unified_llm_guardrails" +endpoint_guardrail_translation_mappings = None class UnifiedLLMGuardrails(CustomLogger): diff --git a/tests/logging_callback_tests/test_opentelemetry_unit_tests.py b/tests/logging_callback_tests/test_opentelemetry_unit_tests.py index a1b80e0d1d..807ca24333 100644 --- a/tests/logging_callback_tests/test_opentelemetry_unit_tests.py +++ b/tests/logging_callback_tests/test_opentelemetry_unit_tests.py @@ -119,3 +119,108 @@ class TestOpentelemetryUnitTests(BaseLoggingCallbackTest): assert detected_span_context.span_id == parent_span_context.span_id, ( "Detected span should have same span_id as parent" ) + + def test_record_exception_on_span(self): + """ + Test that _record_exception_on_span properly records exception information. + + This test verifies that StandardLoggingPayloadErrorInformation is properly + extracted and set as span attributes using ErrorAttributes constants. + """ + from opentelemetry import trace + from opentelemetry.sdk.trace import TracerProvider + from litellm.integrations.opentelemetry import OpenTelemetry + from litellm.integrations._types.open_inference import ErrorAttributes + + # Setup: Create TracerProvider and tracer + tracer_provider = TracerProvider() + trace.set_tracer_provider(tracer_provider) + tracer = trace.get_tracer(__name__) + + # Create OpenTelemetry integration + otel_integration = OpenTelemetry() + + # Create a mock span + mock_span = MagicMock() + + # Create test exception + test_exception = ValueError("Test error message") + + # Create kwargs with exception and error_information + kwargs = { + "exception": test_exception, + "standard_logging_object": { + "error_information": { + "error_code": "500", + "error_class": "ValueError", + "llm_provider": "openai", + "traceback": "Traceback (most recent call last)...", + "error_message": "Test error message", + }, + "error_str": "Test error message", + }, + } + + # Act: Record exception on span + otel_integration._record_exception_on_span(span=mock_span, kwargs=kwargs) + + # Assert: span.record_exception should be called with the exception + mock_span.record_exception.assert_called_once_with(test_exception) + + # Assert: Error attributes should be set using ErrorAttributes constants + expected_calls = [ + (ErrorAttributes.ERROR_CODE, "500"), + (ErrorAttributes.ERROR_TYPE, "ValueError"), + (ErrorAttributes.ERROR_MESSAGE, "Test error message"), + (ErrorAttributes.ERROR_LLM_PROVIDER, "openai"), + (ErrorAttributes.ERROR_STACK_TRACE, "Traceback (most recent call last)..."), + ] + + # Check that set_attribute was called with expected values + actual_calls = [call.args for call in mock_span.set_attribute.call_args_list] + + for expected_call in expected_calls: + assert expected_call in actual_calls, ( + f"Expected set_attribute call {expected_call} not found in actual calls: {actual_calls}" + ) + + def test_record_exception_on_span_with_fallback(self): + """ + Test that _record_exception_on_span falls back to error_str when error_information is None. + """ + from opentelemetry import trace + from opentelemetry.sdk.trace import TracerProvider + from litellm.integrations.opentelemetry import OpenTelemetry + from litellm.integrations._types.open_inference import ErrorAttributes + + # Setup: Create TracerProvider and tracer + tracer_provider = TracerProvider() + trace.set_tracer_provider(tracer_provider) + tracer = trace.get_tracer(__name__) + + # Create OpenTelemetry integration + otel_integration = OpenTelemetry() + + # Create a mock span + mock_span = MagicMock() + + # Create test exception + test_exception = ValueError("Test error message") + + # Create kwargs without error_information (should fallback to error_str) + kwargs = { + "exception": test_exception, + "standard_logging_object": { + "error_information": None, + "error_str": "Fallback error message", + }, + } + + # Act: Record exception on span + otel_integration._record_exception_on_span(span=mock_span, kwargs=kwargs) + + # Assert: span.record_exception should be called + mock_span.record_exception.assert_called_once_with(test_exception) + + # Assert: error.message should be set from error_str using ErrorAttributes constant + mock_span.set_attribute.assert_called_with(ErrorAttributes.ERROR_MESSAGE, "Fallback error message")