From e9328bfa3aa820c4af25e5bae4da9f5cee6557a3 Mon Sep 17 00:00:00 2001 From: Fabricio Ceschin Date: Mon, 19 May 2025 14:48:41 -0400 Subject: [PATCH 1/8] Adding langfuse usage details for cached tokens --- litellm/integrations/langfuse/langfuse.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index d0472ee638..ccc072149c 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -638,6 +638,7 @@ class LangFuseLogger: generation_id = None usage = None + usage_details = None if response_obj is not None: if ( hasattr(response_obj, "id") @@ -654,6 +655,13 @@ class LangFuseLogger: "completion_tokens": _usage_obj.completion_tokens, "total_cost": cost if self._supports_costs() else None, } + usage_details = { + "input": _usage_obj.prompt_tokens, + "output": _usage_obj.completion_tokens, + "cache_creation_input_tokens": _usage_obj.get('cache_creation_input_tokens', 0), + "cache_read_input_tokens": _usage_obj.get('cache_read_input_tokens', 0) + } + generation_name = clean_metadata.pop("generation_name", None) if generation_name is None: # if `generation_name` is None, use sensible default values @@ -686,6 +694,7 @@ class LangFuseLogger: "input": input if not mask_input else "redacted-by-litellm", "output": output if not mask_output else "redacted-by-litellm", "usage": usage, + "usage_details": usage_details, "metadata": log_requester_metadata(clean_metadata), "level": level, "version": clean_metadata.pop("version", None), From d47ad69a1b2d006fbc15acea12bc47cc9d330bc1 Mon Sep 17 00:00:00 2001 From: Fabricio Ceschin Date: Wed, 21 May 2025 10:43:08 -0400 Subject: [PATCH 2/8] Added typed dict for LangfuseUsageDetails --- litellm/integrations/langfuse/langfuse.py | 2 +- litellm/types/integrations/langfuse.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index f4d1a1f79c..17e15e0b03 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -659,7 +659,7 @@ class LangFuseLogger: "completion_tokens": _usage_obj.completion_tokens, "total_cost": cost if self._supports_costs() else None, } - usage_details = { + usage_details : Optional[LangfuseUsageDetails] = { "input": _usage_obj.prompt_tokens, "output": _usage_obj.completion_tokens, "cache_creation_input_tokens": _usage_obj.get('cache_creation_input_tokens', 0), diff --git a/litellm/types/integrations/langfuse.py b/litellm/types/integrations/langfuse.py index ecf42d8cd4..c19cd98d16 100644 --- a/litellm/types/integrations/langfuse.py +++ b/litellm/types/integrations/langfuse.py @@ -5,3 +5,10 @@ class LangfuseLoggingConfig(TypedDict): langfuse_secret: Optional[str] langfuse_public_key: Optional[str] langfuse_host: Optional[str] + + +class LangfuseUsageDetails(TypedDict): + input: Optional[int] + output: Optional[int] + cache_creation_input_tokens: Optional[int] + cache_read_input_tokens: Optional[int] From a30fd5c2ad59682d250493996e55327abd5e645b Mon Sep 17 00:00:00 2001 From: Fabricio Ceschin Date: Wed, 21 May 2025 10:51:59 -0400 Subject: [PATCH 3/8] Fixing lint errors with LangfuseUsageDetails --- litellm/integrations/langfuse/langfuse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 17e15e0b03..001e79cf80 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -642,7 +642,7 @@ class LangFuseLogger: generation_id = None usage = None - usage_details = None + usage_details : Optional[LangfuseUsageDetails] = None if response_obj is not None: if ( hasattr(response_obj, "id") From 90c54d688a4b290312c79a5c50cfbb227aa09670 Mon Sep 17 00:00:00 2001 From: Fabricio Ceschin Date: Wed, 21 May 2025 11:03:14 -0400 Subject: [PATCH 4/8] Fixing lint errors with LangfuseUsageDetails again --- litellm/integrations/langfuse/langfuse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 001e79cf80..a98771387c 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -659,7 +659,7 @@ class LangFuseLogger: "completion_tokens": _usage_obj.completion_tokens, "total_cost": cost if self._supports_costs() else None, } - usage_details : Optional[LangfuseUsageDetails] = { + usage_details = { "input": _usage_obj.prompt_tokens, "output": _usage_obj.completion_tokens, "cache_creation_input_tokens": _usage_obj.get('cache_creation_input_tokens', 0), From bd0369dd887e3fb9f630c6e25d6962f42635a0b5 Mon Sep 17 00:00:00 2001 From: Fabricio Ceschin Date: Wed, 21 May 2025 13:35:21 -0400 Subject: [PATCH 5/8] Using typed dict in usage_details --- litellm/integrations/langfuse/langfuse.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index a98771387c..f116eb8041 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -642,7 +642,7 @@ class LangFuseLogger: generation_id = None usage = None - usage_details : Optional[LangfuseUsageDetails] = None + usage_details = None if response_obj is not None: if ( hasattr(response_obj, "id") @@ -659,12 +659,10 @@ class LangFuseLogger: "completion_tokens": _usage_obj.completion_tokens, "total_cost": cost if self._supports_costs() else None, } - usage_details = { - "input": _usage_obj.prompt_tokens, - "output": _usage_obj.completion_tokens, - "cache_creation_input_tokens": _usage_obj.get('cache_creation_input_tokens', 0), - "cache_read_input_tokens": _usage_obj.get('cache_read_input_tokens', 0) - } + usage_details = LangfuseUsageDetails(input=_usage_obj.prompt_tokens, + output=_usage_obj.completion_tokens, + cache_creation_input_tokens=_usage_obj.get('cache_creation_input_tokens', 0), + cache_read_input_tokens=_usage_obj.get('cache_read_input_tokens', 0)) generation_name = clean_metadata.pop("generation_name", None) if generation_name is None: From 9d73f1e607be57c65c6d8061f060c15c7e92a21d Mon Sep 17 00:00:00 2001 From: Fabricio Ceschin Date: Wed, 21 May 2025 21:23:49 -0400 Subject: [PATCH 6/8] Added unit tests for langfuse usage details integration --- tests/litellm/integrations/test_langfuse.py | 218 ++++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 tests/litellm/integrations/test_langfuse.py diff --git a/tests/litellm/integrations/test_langfuse.py b/tests/litellm/integrations/test_langfuse.py new file mode 100644 index 0000000000..26874d9ba2 --- /dev/null +++ b/tests/litellm/integrations/test_langfuse.py @@ -0,0 +1,218 @@ +import unittest +from unittest.mock import patch, MagicMock +import sys +import os +import datetime + +sys.path.insert(0, os.path.abspath("../..")) +from litellm.integrations.langfuse.langfuse import LangFuseLogger +# Import LangfuseUsageDetails directly from the module where it's defined +from litellm.types.integrations.langfuse import * + + + +class TestLangfuseUsageDetails(unittest.TestCase): + + def setUp(self): + # Set up environment variables for testing + self.env_patcher = patch.dict('os.environ', { + 'LANGFUSE_SECRET_KEY': 'test-secret-key', + 'LANGFUSE_PUBLIC_KEY': 'test-public-key', + 'LANGFUSE_HOST': 'https://test.langfuse.com' + }) + self.env_patcher.start() + + # Create mock objects + self.mock_langfuse_client = MagicMock() + self.mock_langfuse_trace = MagicMock() + self.mock_langfuse_generation = MagicMock() + + # Setup the trace and generation chain + self.mock_langfuse_trace.generation.return_value = self.mock_langfuse_generation + self.mock_langfuse_client.trace.return_value = self.mock_langfuse_trace + + # Mock the langfuse module that's imported locally in methods + self.langfuse_module_patcher = patch.dict('sys.modules', {'langfuse': MagicMock()}) + self.mock_langfuse_module = self.langfuse_module_patcher.start() + + # Create a mock for the langfuse module with version + self.mock_langfuse = MagicMock() + self.mock_langfuse.version = MagicMock() + self.mock_langfuse.version.__version__ = "3.0.0" # Set a version that supports all features + + # Mock the Langfuse class + self.mock_langfuse_class = MagicMock() + self.mock_langfuse_class.return_value = self.mock_langfuse_client + + # Set up the sys.modules['langfuse'] mock + sys.modules['langfuse'] = self.mock_langfuse + sys.modules['langfuse'].Langfuse = self.mock_langfuse_class + + # Mock the Langfuse client + self.mock_langfuse_client = MagicMock() + self.mock_langfuse_trace = MagicMock() + self.mock_langfuse_generation = MagicMock() + + # Setup the trace and generation chain + self.mock_langfuse_trace.generation.return_value = self.mock_langfuse_generation + self.mock_langfuse_client.trace.return_value = self.mock_langfuse_trace + + # Mock the Langfuse class + self.mock_langfuse_class = MagicMock() + self.mock_langfuse_class.return_value = self.mock_langfuse_client + self.mock_langfuse.Langfuse = self.mock_langfuse_class + + # Create the logger + self.logger = LangFuseLogger() + + # Add the log_event_on_langfuse method to the instance + def log_event_on_langfuse(self, kwargs, response_obj, start_time=None, end_time=None, user_id=None, level="DEFAULT", status_message=None): + # This implementation calls _log_langfuse_v2 directly + return self._log_langfuse_v2( + user_id=user_id, + metadata=kwargs.get("litellm_params", {}).get("metadata", {}), + litellm_params=kwargs.get("litellm_params", {}), + output=None, + start_time=start_time, + end_time=end_time, + kwargs=kwargs, + optional_params=kwargs.get("optional_params", {}), + input=None, + response_obj=response_obj, + level=level, + litellm_call_id=kwargs.get("litellm_call_id", None), + print_verbose=True # Add the missing parameter + ) + + # Bind the method to the instance + import types + self.logger.log_event_on_langfuse = types.MethodType(log_event_on_langfuse, self.logger) + + # Make sure _is_langfuse_v2 returns True + def mock_is_langfuse_v2(self): + return True + + self.logger._is_langfuse_v2 = types.MethodType(mock_is_langfuse_v2, self.logger) + + def tearDown(self): + self.env_patcher.stop() + self.langfuse_module_patcher.stop() + + def test_langfuse_usage_details_type(self): + """Test that LangfuseUsageDetails TypedDict is properly defined with the correct fields""" + # Create an instance of LangfuseUsageDetails + usage_details: LangfuseUsageDetails = { + "input": 10, + "output": 20, + "cache_creation_input_tokens": 5, + "cache_read_input_tokens": 3 + } + + # Verify all fields are present + self.assertEqual(usage_details["input"], 10) + self.assertEqual(usage_details["output"], 20) + self.assertEqual(usage_details["cache_creation_input_tokens"], 5) + self.assertEqual(usage_details["cache_read_input_tokens"], 3) + + # Test with all fields (all fields are required in TypedDict by default) + minimal_usage_details: LangfuseUsageDetails = { + "input": 10, + "output": 20, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0 + } + + self.assertEqual(minimal_usage_details["input"], 10) + self.assertEqual(minimal_usage_details["output"], 20) + + def test_log_langfuse_v2_usage_details(self): + """Test that usage_details in _log_langfuse_v2 is correctly typed and assigned""" + # Create a mock response object with usage information + response_obj = MagicMock() + response_obj.usage = MagicMock() + response_obj.usage.prompt_tokens = 15 + response_obj.usage.completion_tokens = 25 + + # Add the cache token attributes using get method + def mock_get(key, default=None): + if key == 'cache_creation_input_tokens': + return 7 + elif key == 'cache_read_input_tokens': + return 4 + return default + + response_obj.usage.get = mock_get + + # Create kwargs for the log_event method + kwargs = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "litellm_params": {"metadata": {}} + } + + # Create start and end times + start_time = datetime.datetime.now() + end_time = start_time + datetime.timedelta(seconds=1) + + # Call the log_event method + with patch.object(self.logger, '_log_langfuse_v2') as mock_log_langfuse_v2: + self.logger.log_event_on_langfuse( + kwargs=kwargs, + response_obj=response_obj, + start_time=start_time, + end_time=end_time + ) + + # Check if _log_langfuse_v2 was called + mock_log_langfuse_v2.assert_called_once() + + # Get the arguments passed to _log_langfuse_v2 + call_args = mock_log_langfuse_v2.call_args[1] + + # Verify response_obj was passed correctly + self.assertEqual(call_args["response_obj"], response_obj) + + def test_langfuse_usage_details_optional_fields(self): + """Test that LangfuseUsageDetails fields are properly defined as Optional""" + # Create an instance with None values for optional fields + usage_details: LangfuseUsageDetails = { + "input": 10, + "output": 20, + "cache_creation_input_tokens": None, + "cache_read_input_tokens": None + } + + # Verify fields can be None + self.assertEqual(usage_details["input"], 10) + self.assertEqual(usage_details["output"], 20) + self.assertIsNone(usage_details["cache_creation_input_tokens"]) + self.assertIsNone(usage_details["cache_read_input_tokens"]) + + def test_langfuse_usage_details_structure(self): + """Test that LangfuseUsageDetails has the correct structure as defined in the commit""" + # This test directly verifies the structure of the TypedDict + # without relying on the LangFuseLogger class + + # Create a dictionary that matches the LangfuseUsageDetails structure + usage_details = { + "input": 15, + "output": 25, + "cache_creation_input_tokens": 7, + "cache_read_input_tokens": 4 + } + + # Verify the structure matches what we expect + self.assertIn("input", usage_details) + self.assertIn("output", usage_details) + self.assertIn("cache_creation_input_tokens", usage_details) + self.assertIn("cache_read_input_tokens", usage_details) + + # Verify the values + self.assertEqual(usage_details["input"], 15) + self.assertEqual(usage_details["output"], 25) + self.assertEqual(usage_details["cache_creation_input_tokens"], 7) + self.assertEqual(usage_details["cache_read_input_tokens"], 4) + + +if __name__ == "__main__": + unittest.main() From 118d15c1c61e26daf0362bc08e13e28a4eafc4f6 Mon Sep 17 00:00:00 2001 From: Fabricio Ceschin Date: Wed, 21 May 2025 21:37:23 -0400 Subject: [PATCH 7/8] Improving langfuse usage details unit test --- tests/litellm/integrations/test_langfuse.py | 76 ++++++++++----------- 1 file changed, 35 insertions(+), 41 deletions(-) diff --git a/tests/litellm/integrations/test_langfuse.py b/tests/litellm/integrations/test_langfuse.py index 26874d9ba2..2aa363e5f9 100644 --- a/tests/litellm/integrations/test_langfuse.py +++ b/tests/litellm/integrations/test_langfuse.py @@ -9,10 +9,8 @@ from litellm.integrations.langfuse.langfuse import LangFuseLogger # Import LangfuseUsageDetails directly from the module where it's defined from litellm.types.integrations.langfuse import * - - class TestLangfuseUsageDetails(unittest.TestCase): - + def setUp(self): # Set up environment variables for testing self.env_patcher = patch.dict('os.environ', { @@ -21,50 +19,50 @@ class TestLangfuseUsageDetails(unittest.TestCase): 'LANGFUSE_HOST': 'https://test.langfuse.com' }) self.env_patcher.start() - + # Create mock objects self.mock_langfuse_client = MagicMock() self.mock_langfuse_trace = MagicMock() self.mock_langfuse_generation = MagicMock() - + # Setup the trace and generation chain self.mock_langfuse_trace.generation.return_value = self.mock_langfuse_generation self.mock_langfuse_client.trace.return_value = self.mock_langfuse_trace - + # Mock the langfuse module that's imported locally in methods self.langfuse_module_patcher = patch.dict('sys.modules', {'langfuse': MagicMock()}) self.mock_langfuse_module = self.langfuse_module_patcher.start() - + # Create a mock for the langfuse module with version self.mock_langfuse = MagicMock() self.mock_langfuse.version = MagicMock() self.mock_langfuse.version.__version__ = "3.0.0" # Set a version that supports all features - + # Mock the Langfuse class self.mock_langfuse_class = MagicMock() self.mock_langfuse_class.return_value = self.mock_langfuse_client - + # Set up the sys.modules['langfuse'] mock sys.modules['langfuse'] = self.mock_langfuse sys.modules['langfuse'].Langfuse = self.mock_langfuse_class - + # Mock the Langfuse client self.mock_langfuse_client = MagicMock() self.mock_langfuse_trace = MagicMock() self.mock_langfuse_generation = MagicMock() - + # Setup the trace and generation chain self.mock_langfuse_trace.generation.return_value = self.mock_langfuse_generation self.mock_langfuse_client.trace.return_value = self.mock_langfuse_trace - + # Mock the Langfuse class self.mock_langfuse_class = MagicMock() self.mock_langfuse_class.return_value = self.mock_langfuse_client self.mock_langfuse.Langfuse = self.mock_langfuse_class - + # Create the logger self.logger = LangFuseLogger() - + # Add the log_event_on_langfuse method to the instance def log_event_on_langfuse(self, kwargs, response_obj, start_time=None, end_time=None, user_id=None, level="DEFAULT", status_message=None): # This implementation calls _log_langfuse_v2 directly @@ -83,21 +81,21 @@ class TestLangfuseUsageDetails(unittest.TestCase): litellm_call_id=kwargs.get("litellm_call_id", None), print_verbose=True # Add the missing parameter ) - + # Bind the method to the instance import types self.logger.log_event_on_langfuse = types.MethodType(log_event_on_langfuse, self.logger) - + # Make sure _is_langfuse_v2 returns True def mock_is_langfuse_v2(self): return True - + self.logger._is_langfuse_v2 = types.MethodType(mock_is_langfuse_v2, self.logger) - + def tearDown(self): self.env_patcher.stop() self.langfuse_module_patcher.stop() - + def test_langfuse_usage_details_type(self): """Test that LangfuseUsageDetails TypedDict is properly defined with the correct fields""" # Create an instance of LangfuseUsageDetails @@ -107,13 +105,13 @@ class TestLangfuseUsageDetails(unittest.TestCase): "cache_creation_input_tokens": 5, "cache_read_input_tokens": 3 } - + # Verify all fields are present self.assertEqual(usage_details["input"], 10) self.assertEqual(usage_details["output"], 20) self.assertEqual(usage_details["cache_creation_input_tokens"], 5) self.assertEqual(usage_details["cache_read_input_tokens"], 3) - + # Test with all fields (all fields are required in TypedDict by default) minimal_usage_details: LangfuseUsageDetails = { "input": 10, @@ -121,10 +119,10 @@ class TestLangfuseUsageDetails(unittest.TestCase): "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0 } - + self.assertEqual(minimal_usage_details["input"], 10) self.assertEqual(minimal_usage_details["output"], 20) - + def test_log_langfuse_v2_usage_details(self): """Test that usage_details in _log_langfuse_v2 is correctly typed and assigned""" # Create a mock response object with usage information @@ -132,7 +130,7 @@ class TestLangfuseUsageDetails(unittest.TestCase): response_obj.usage = MagicMock() response_obj.usage.prompt_tokens = 15 response_obj.usage.completion_tokens = 25 - + # Add the cache token attributes using get method def mock_get(key, default=None): if key == 'cache_creation_input_tokens': @@ -140,20 +138,20 @@ class TestLangfuseUsageDetails(unittest.TestCase): elif key == 'cache_read_input_tokens': return 4 return default - + response_obj.usage.get = mock_get - + # Create kwargs for the log_event method kwargs = { "model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}], "litellm_params": {"metadata": {}} } - + # Create start and end times start_time = datetime.datetime.now() end_time = start_time + datetime.timedelta(seconds=1) - + # Call the log_event method with patch.object(self.logger, '_log_langfuse_v2') as mock_log_langfuse_v2: self.logger.log_event_on_langfuse( @@ -162,16 +160,16 @@ class TestLangfuseUsageDetails(unittest.TestCase): start_time=start_time, end_time=end_time ) - + # Check if _log_langfuse_v2 was called mock_log_langfuse_v2.assert_called_once() - + # Get the arguments passed to _log_langfuse_v2 call_args = mock_log_langfuse_v2.call_args[1] - + # Verify response_obj was passed correctly self.assertEqual(call_args["response_obj"], response_obj) - + def test_langfuse_usage_details_optional_fields(self): """Test that LangfuseUsageDetails fields are properly defined as Optional""" # Create an instance with None values for optional fields @@ -181,18 +179,18 @@ class TestLangfuseUsageDetails(unittest.TestCase): "cache_creation_input_tokens": None, "cache_read_input_tokens": None } - + # Verify fields can be None self.assertEqual(usage_details["input"], 10) self.assertEqual(usage_details["output"], 20) self.assertIsNone(usage_details["cache_creation_input_tokens"]) self.assertIsNone(usage_details["cache_read_input_tokens"]) - + def test_langfuse_usage_details_structure(self): """Test that LangfuseUsageDetails has the correct structure as defined in the commit""" # This test directly verifies the structure of the TypedDict # without relying on the LangFuseLogger class - + # Create a dictionary that matches the LangfuseUsageDetails structure usage_details = { "input": 15, @@ -200,19 +198,15 @@ class TestLangfuseUsageDetails(unittest.TestCase): "cache_creation_input_tokens": 7, "cache_read_input_tokens": 4 } - + # Verify the structure matches what we expect self.assertIn("input", usage_details) self.assertIn("output", usage_details) self.assertIn("cache_creation_input_tokens", usage_details) self.assertIn("cache_read_input_tokens", usage_details) - + # Verify the values self.assertEqual(usage_details["input"], 15) self.assertEqual(usage_details["output"], 25) self.assertEqual(usage_details["cache_creation_input_tokens"], 7) self.assertEqual(usage_details["cache_read_input_tokens"], 4) - - -if __name__ == "__main__": - unittest.main() From 6d7d5ed07805fe40ec3ebdba2eca6fd769de7cdc Mon Sep 17 00:00:00 2001 From: Fabricio Ceschin Date: Mon, 8 Sep 2025 11:37:24 -0400 Subject: [PATCH 8/8] Updating poetry.lock --- poetry.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 771bb6d486..41182d12bf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -9182,4 +9182,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.8.1,<4.0, !=3.9.7" -content-hash = "8ebdb444c0ff253857f90d3ec9926a58d23d4482c644fd158a047f0c0f892bb9" +content-hash = "54374b906931afe92861b6b9b226751579c0d342f650bc8f7ebaf4c5882d6249"