From f0aeedb25d7942c969d420f7a8cc029ffa7c1111 Mon Sep 17 00:00:00 2001 From: Max Deichmann Date: Wed, 31 Jan 2024 13:41:54 +0100 Subject: [PATCH 1/6] add cost tracking --- litellm/integrations/langfuse.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index 722eb198c3..6971b0019e 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -78,6 +78,7 @@ class LangFuseLogger: optional_params, input, response_obj, + print_verbose ) if self._is_langfuse_v2() else self._log_langfuse_v1( user_id, metadata, @@ -88,6 +89,7 @@ class LangFuseLogger: optional_params, input, response_obj, + ) self.Langfuse.flush() @@ -167,11 +169,15 @@ class LangFuseLogger: optional_params, input, response_obj, + print_verbose ): import langfuse tags = [] supports_tags = Version(langfuse.version.__version__) >= Version("2.6.3") + supports_costs = Version(langfuse.version.__version__) >= Version("2.7.0") + + print_verbose(f"Langfuse Layer Logging - logging to langfuse v2 ") trace_params = { "name": metadata.get("generation_name", "litellm-completion"), @@ -188,6 +194,9 @@ class LangFuseLogger: trace_params.update({"tags": tags}) trace = self.Langfuse.trace(**trace_params) + + cost = kwargs["response_cost"] + print_verbose(f"trace: {cost}") trace.generation( name=metadata.get("generation_name", "litellm-completion"), @@ -201,6 +210,8 @@ class LangFuseLogger: usage={ "prompt_tokens": response_obj["usage"]["prompt_tokens"], "completion_tokens": response_obj["usage"]["completion_tokens"], + "input_cost": kwargs["response_cost"] if supports_costs else None, + "output_cost": kwargs["response_cost"] if supports_costs else None, }, metadata=metadata, ) From 926bdae955543fca6def03f67071cab02e8ecb45 Mon Sep 17 00:00:00 2001 From: Max Deichmann Date: Wed, 31 Jan 2024 13:57:48 +0100 Subject: [PATCH 2/6] push --- litellm/integrations/langfuse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index 6971b0019e..4976688ec9 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -175,7 +175,7 @@ class LangFuseLogger: tags = [] supports_tags = Version(langfuse.version.__version__) >= Version("2.6.3") - supports_costs = Version(langfuse.version.__version__) >= Version("2.7.0") + supports_costs = Version(langfuse.version.__version__) >= Version("2.7.3") print_verbose(f"Langfuse Layer Logging - logging to langfuse v2 ") From 01d3eb8634ceed5363cac2dbc132ba794209cd15 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Wed, 31 Jan 2024 06:59:46 -0800 Subject: [PATCH 3/6] (fix) include fastapi-sso==0.10.0 Dockerfile --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 6103091b85..32d57f2199 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ traceloop-sdk==0.5.3 # for open telemetry logging langfuse>=2.6.3 # for langfuse self-hosted logging orjson==3.9.7 # fast /embedding responses apscheduler==3.10.4 # for resetting budget in background +fastapi-sso==0.10.0 ### LITELLM PACKAGE DEPENDENCIES python-dotenv>=0.2.0 # for env tiktoken>=0.4.0 # for calculating usage From e5e625cc5006ae769a4ba94f554b2f8aa9ad2946 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Wed, 31 Jan 2024 07:56:45 -0800 Subject: [PATCH 4/6] (fix) proxy - show value of allow_user_auth --- litellm/proxy/proxy_server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bb7ff0ae7d..03bfa9a18f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -534,6 +534,7 @@ async def user_api_key_auth( and (not is_master_key_valid) and (not general_settings.get("allow_user_auth", False)) ): + # enters this block when allow_user_auth is set to False assert not general_settings.get("allow_user_auth", False) if route == "/key/info": # check if user can access this route @@ -560,8 +561,9 @@ async def user_api_key_auth( # /model/info just shows models user has access to pass else: + allow_user_auth = general_settings.get("allow_user_auth", False) raise Exception( - f"only master key can be used to generate, delete, update or get info for new keys/users." + f"Only master key can be used to generate, delete, update or get info for new keys/users. Value of allow_user_auth={allow_user_auth}" ) return UserAPIKeyAuth(api_key=api_key, **valid_token_dict) From 18c58f1a9bc6b5b0fb8bae349342f55ad5f564fd Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 31 Jan 2024 08:33:10 -0800 Subject: [PATCH 5/6] test(test_amazing_vertex_completion.py): fix test to handle api instability --- litellm/tests/test_amazing_vertex_completion.py | 7 +++++-- litellm/utils.py | 7 ++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/litellm/tests/test_amazing_vertex_completion.py b/litellm/tests/test_amazing_vertex_completion.py index 85c1cb9337..62e6bc6574 100644 --- a/litellm/tests/test_amazing_vertex_completion.py +++ b/litellm/tests/test_amazing_vertex_completion.py @@ -277,7 +277,7 @@ def test_gemini_pro_vision(): try: load_vertex_ai_credentials() litellm.set_verbose = True - litellm.num_retries = 0 + litellm.num_retries = 3 resp = litellm.completion( model="vertex_ai/gemini-pro-vision", messages=[ @@ -304,7 +304,10 @@ def test_gemini_pro_vision(): assert prompt_tokens == 263 # the gemini api returns 263 to us except Exception as e: - pytest.fail(f"An exception occurred - {str(e)}") + if "500 Internal error encountered.'" in str(e): + pass + else: + pytest.fail(f"An exception occurred - {str(e)}") # test_gemini_pro_vision() diff --git a/litellm/utils.py b/litellm/utils.py index ed7d52125f..d102476f31 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2248,7 +2248,12 @@ def client(original_function): ) if num_retries: - if isinstance(e, openai.APIError) or isinstance(e, openai.Timeout): + if ( + isinstance(e, openai.APIError) + or isinstance(e, openai.Timeout) + or isinstance(openai.APIConnectionError) + ): + print_verbose(f"RETRY TRIGGERED!") kwargs["num_retries"] = num_retries return litellm.completion_with_retries(*args, **kwargs) elif ( From 345d0029afd6554da44e625d0632eb9f74b2156d Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 31 Jan 2024 16:04:56 -0800 Subject: [PATCH 6/6] fix(langfuse.py): use total_cost --- litellm/integrations/langfuse.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index 4976688ec9..7e5a61a3a6 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -78,7 +78,7 @@ class LangFuseLogger: optional_params, input, response_obj, - print_verbose + print_verbose, ) if self._is_langfuse_v2() else self._log_langfuse_v1( user_id, metadata, @@ -89,7 +89,6 @@ class LangFuseLogger: optional_params, input, response_obj, - ) self.Langfuse.flush() @@ -169,7 +168,7 @@ class LangFuseLogger: optional_params, input, response_obj, - print_verbose + print_verbose, ): import langfuse @@ -186,6 +185,8 @@ class LangFuseLogger: "user_id": metadata.get("trace_user_id", user_id), "id": metadata.get("trace_id", None), } + cost = kwargs["response_cost"] + print_verbose(f"trace: {cost}") if supports_tags: for key, value in metadata.items(): tags.append(f"{key}:{value}") @@ -194,10 +195,6 @@ class LangFuseLogger: trace_params.update({"tags": tags}) trace = self.Langfuse.trace(**trace_params) - - cost = kwargs["response_cost"] - print_verbose(f"trace: {cost}") - trace.generation( name=metadata.get("generation_name", "litellm-completion"), id=metadata.get("generation_id", None), @@ -210,8 +207,7 @@ class LangFuseLogger: usage={ "prompt_tokens": response_obj["usage"]["prompt_tokens"], "completion_tokens": response_obj["usage"]["completion_tokens"], - "input_cost": kwargs["response_cost"] if supports_costs else None, - "output_cost": kwargs["response_cost"] if supports_costs else None, + "total_cost": cost if supports_costs else None, }, metadata=metadata, )