From e1786848cbfa685948c0b6d869af58771283853d Mon Sep 17 00:00:00 2001 From: CyanideByte Date: Sat, 27 Apr 2024 13:08:45 -0700 Subject: [PATCH 01/52] protected_namespaces fixed for model_info --- litellm/tests/test_config.py | 3 +++ litellm/types/router.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/litellm/tests/test_config.py b/litellm/tests/test_config.py index ed68694039..800f0693e6 100644 --- a/litellm/tests/test_config.py +++ b/litellm/tests/test_config.py @@ -26,6 +26,9 @@ class DBModel(BaseModel): model_info: dict litellm_params: dict + class Config: + protected_namespaces = () + @pytest.mark.asyncio async def test_delete_deployment(): diff --git a/litellm/types/router.py b/litellm/types/router.py index 09965bb8a9..87608a71cc 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -202,12 +202,18 @@ class updateDeployment(BaseModel): litellm_params: Optional[updateLiteLLMParams] = None model_info: Optional[ModelInfo] = None + class Config: + protected_namespaces = () + class Deployment(BaseModel): model_name: str litellm_params: LiteLLM_Params model_info: ModelInfo + class Config: + protected_namespaces = () + def __init__( self, model_name: str, From a4c7d933a9230da2a32a9be97bf8760a14a928bd Mon Sep 17 00:00:00 2001 From: CyanideByte Date: Sat, 27 Apr 2024 15:44:40 -0700 Subject: [PATCH 02/52] Added pytest for pydantic protected namespace warning --- litellm/tests/test_pydantic_namespaces.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 litellm/tests/test_pydantic_namespaces.py diff --git a/litellm/tests/test_pydantic_namespaces.py b/litellm/tests/test_pydantic_namespaces.py new file mode 100644 index 0000000000..8314216e1a --- /dev/null +++ b/litellm/tests/test_pydantic_namespaces.py @@ -0,0 +1,10 @@ +import warnings +import pytest + +def test_namespace_conflict_warning(): + with warnings.catch_warnings(record=True) as recorded_warnings: + warnings.simplefilter("always") # Capture all warnings + import litellm + + # Check that no warning with the specific message was raised + assert not any("conflict with protected namespace" in str(w.message) for w in recorded_warnings), "Test failed: 'conflict with protected namespace' warning was encountered!" From 03a43b99a5ff8b57a117edab27b2d7f322c2fed1 Mon Sep 17 00:00:00 2001 From: CyanideByte Date: Sat, 27 Apr 2024 20:42:54 -0700 Subject: [PATCH 03/52] Added _types.py cases from edwinjosegeorge PR#3340 --- litellm/proxy/_types.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 293d06023a..fbe914a26f 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -422,6 +422,9 @@ class LiteLLM_ModelTable(LiteLLMBase): created_by: str updated_by: str + class Config: + protected_namespaces = () + class NewUserRequest(GenerateKeyRequest): max_budget: Optional[float] = None @@ -485,6 +488,9 @@ class TeamBase(LiteLLMBase): class NewTeamRequest(TeamBase): model_aliases: Optional[dict] = None + class Config: + protected_namespaces = () + class GlobalEndUsersSpend(LiteLLMBase): api_key: Optional[str] = None @@ -534,6 +540,9 @@ class LiteLLM_TeamTable(TeamBase): budget_reset_at: Optional[datetime] = None model_id: Optional[int] = None + class Config: + protected_namespaces = () + @root_validator(pre=True) def set_model_info(cls, values): dict_fields = [ @@ -570,6 +579,9 @@ class LiteLLM_BudgetTable(LiteLLMBase): model_max_budget: Optional[dict] = None budget_duration: Optional[str] = None + class Config: + protected_namespaces = () + class NewOrganizationRequest(LiteLLM_BudgetTable): organization_id: Optional[str] = None From 0db7fa3fd8d5ec4140b5e34154b5819280737c9c Mon Sep 17 00:00:00 2001 From: alisalim17 Date: Mon, 29 Apr 2024 14:20:24 +0400 Subject: [PATCH 04/52] fix: cohere tool results --- litellm/llms/prompt_templates/factory.py | 80 ++++++++++++++++++------ 1 file changed, 61 insertions(+), 19 deletions(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index c51dc89be5..e1fa354c63 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -3,8 +3,14 @@ import requests, traceback import json, re, xml.etree.ElementTree as ET from jinja2 import Template, exceptions, meta, BaseLoader from jinja2.sandbox import ImmutableSandboxedEnvironment -from typing import Optional, Any -from typing import List +from typing import ( + Any, + List, + Mapping, + MutableMapping, + Optional, + Sequence, +) import litellm @@ -430,8 +436,10 @@ def format_prompt_togetherai(messages, prompt_format, chat_template): prompt = default_pt(messages) return prompt + ### IBM Granite + def ibm_granite_pt(messages: list): """ IBM's Granite models uses the template: @@ -440,23 +448,24 @@ def ibm_granite_pt(messages: list): See: https://www.ibm.com/docs/en/watsonx-as-a-service?topic=solutions-supported-foundation-models """ return custom_prompt( - messages=messages, + messages=messages, role_dict={ - 'system': { - 'pre_message': '<|system|>\n', - 'post_message': '\n', + "system": { + "pre_message": "<|system|>\n", + "post_message": "\n", }, - 'user': { - 'pre_message': '<|user|>\n', - 'post_message': '\n', + "user": { + "pre_message": "<|user|>\n", + "post_message": "\n", }, - 'assistant': { - 'pre_message': '<|assistant|>\n', - 'post_message': '\n', - } - } + "assistant": { + "pre_message": "<|assistant|>\n", + "post_message": "\n", + }, + }, ).strip() + ### ANTHROPIC ### @@ -1043,6 +1052,30 @@ def get_system_prompt(messages): return system_prompt, messages +def convert_to_documents( + observations: Any, +) -> List[MutableMapping]: + """Converts observations into a 'document' dict""" + documents: List[MutableMapping] = [] + if isinstance(observations, str): + # strings are turned into a key/value pair and a key of 'output' is added. + observations = [{"output": observations}] + elif isinstance(observations, Mapping): + # single mappings are transformed into a list to simplify the rest of the code. + observations = [observations] + elif not isinstance(observations, Sequence): + # all other types are turned into a key/value pair within a list + observations = [{"output": observations}] + + for doc in observations: + if not isinstance(doc, Mapping): + # types that aren't Mapping are turned into a key/value pair. + doc = {"output": doc} + documents.append(doc) + + return documents + + def convert_openai_message_to_cohere_tool_result(message): """ OpenAI message with a tool result looks like: @@ -1084,7 +1117,7 @@ def convert_openai_message_to_cohere_tool_result(message): "parameters": {"location": "San Francisco, CA"}, "generation_id": tool_call_id, }, - "outputs": [content], + "outputs": convert_to_documents(content), } return cohere_tool_result @@ -1097,7 +1130,7 @@ def cohere_message_pt(messages: list): if message["role"] == "tool": tool_result = convert_openai_message_to_cohere_tool_result(message) tool_results.append(tool_result) - else: + elif message.get("content"): prompt += message["content"] + "\n\n" prompt = prompt.rstrip() return prompt, tool_results @@ -1396,9 +1429,18 @@ def prompt_factory( # https://llama.meta.com/docs/model-cards-and-prompt-formats/meta-llama-3/ return custom_prompt( role_dict={ - "system": {"pre_message": "<|start_header_id|>system<|end_header_id|>\n", "post_message": "<|eot_id|>"}, - "user": {"pre_message": "<|start_header_id|>user<|end_header_id|>\n", "post_message": "<|eot_id|>"}, - "assistant": {"pre_message": "<|start_header_id|>assistant<|end_header_id|>\n", "post_message": "<|eot_id|>"}, + "system": { + "pre_message": "<|start_header_id|>system<|end_header_id|>\n", + "post_message": "<|eot_id|>", + }, + "user": { + "pre_message": "<|start_header_id|>user<|end_header_id|>\n", + "post_message": "<|eot_id|>", + }, + "assistant": { + "pre_message": "<|start_header_id|>assistant<|end_header_id|>\n", + "post_message": "<|eot_id|>", + }, }, messages=messages, initial_prompt_value="<|begin_of_text|>", From 7b617e666decde16b86473875c5cc570ca69bb07 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 07:23:10 -0700 Subject: [PATCH 05/52] fix(proxy_server.py): return more detailed auth error message. --- litellm/proxy/proxy_server.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f186b3833c..29f3c41dba 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1059,8 +1059,18 @@ async def user_api_key_auth( ): pass else: + user_role = "unknown" + user_id = "unknown" + if user_id_information is not None and isinstance( + user_id_information, list + ): + _user = user_id_information[0] + user_role = _user.get("user_role", {}).get( + "user_role", "unknown" + ) + user_id = _user.get("user_id", "unknown") raise Exception( - f"Only master key can be used to generate, delete, update info for new keys/users/teams. Route={route}" + f"Only proxy admin can be used to generate, delete, update info for new keys/users/teams. Route={route}. Your role={user_role}. Your user_id={user_id}" ) # check if token is from litellm-ui, litellm ui makes keys to allow users to login with sso. These keys can only be used for LiteLLM UI functions From 0a6b6302f1d95e2af4985ba172fd3c132eac0d01 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 07:25:39 -0700 Subject: [PATCH 06/52] fix(router.py): fix typing error --- litellm/types/router.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/litellm/types/router.py b/litellm/types/router.py index 87608a71cc..042d9f277c 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -211,9 +211,6 @@ class Deployment(BaseModel): litellm_params: LiteLLM_Params model_info: ModelInfo - class Config: - protected_namespaces = () - def __init__( self, model_name: str, From 0aa8b94ff5e7c1e1cbcf963de492d4c887f6b3ef Mon Sep 17 00:00:00 2001 From: alisalim17 Date: Mon, 29 Apr 2024 18:38:12 +0400 Subject: [PATCH 07/52] test: completion with Cohere command-r-plus model --- litellm/tests/test_completion.py | 70 ++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index fe4aa9c1c8..0174cdaac5 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -231,6 +231,76 @@ def test_completion_claude_3_function_call(): pytest.fail(f"Error occurred: {e}") +def test_completion_cohere_command_r_plus_function_call(): + litellm.set_verbose = True + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, + } + ] + messages = [ + { + "role": "user", + "content": "What's the weather like in Boston today in Fahrenheit?", + } + ] + try: + # test without max tokens + response = completion( + model="command-r-plus", + messages=messages, + tools=tools, + tool_choice="auto", + ) + # Add any assertions, here to check response args + print(response) + assert isinstance(response.choices[0].message.tool_calls[0].function.name, str) + assert isinstance( + response.choices[0].message.tool_calls[0].function.arguments, str + ) + + messages.append( + response.choices[0].message.model_dump() + ) # Add assistant tool invokes + tool_result = ( + '{"location": "Boston", "temperature": "72", "unit": "fahrenheit"}' + ) + # Add user submitted tool results in the OpenAI format + messages.append( + { + "tool_call_id": response.choices[0].message.tool_calls[0].id, + "role": "tool", + "name": response.choices[0].message.tool_calls[0].function.name, + "content": tool_result, + } + ) + # In the second response, Cohere should deduce answer from tool results + second_response = completion( + model="command-r-plus", + messages=messages, + tools=tools, + tool_choice="auto", + ) + print(second_response) + except Exception as e: + pytest.fail(f"Error occurred: {e}") + + def test_parse_xml_params(): from litellm.llms.prompt_templates.factory import parse_xml_params From 2cfb97141d4087a2916c04dfb2342a3bf2d25a03 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 08:06:15 -0700 Subject: [PATCH 08/52] fix(utils.py): replicate now also has token based pricing for some models --- litellm/tests/test_completion_cost.py | 53 +++++++++++++++++++++++++++ litellm/utils.py | 4 +- 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/litellm/tests/test_completion_cost.py b/litellm/tests/test_completion_cost.py index f17d5a4644..fecd53e193 100644 --- a/litellm/tests/test_completion_cost.py +++ b/litellm/tests/test_completion_cost.py @@ -328,3 +328,56 @@ def test_dalle_3_azure_cost_tracking(): completion_response=response, call_type="image_generation" ) assert cost > 0 + + +def test_replicate_llama3_cost_tracking(): + litellm.set_verbose = True + model = "replicate/meta/meta-llama-3-8b-instruct" + litellm.register_model( + { + "replicate/meta/meta-llama-3-8b-instruct": { + "input_cost_per_token": 0.00000005, + "output_cost_per_token": 0.00000025, + "litellm_provider": "replicate", + } + } + ) + response = litellm.ModelResponse( + id="chatcmpl-cad7282f-7f68-41e7-a5ab-9eb33ae301dc", + choices=[ + litellm.utils.Choices( + finish_reason="stop", + index=0, + message=litellm.utils.Message( + content="I'm doing well, thanks for asking! I'm here to help you with any questions or tasks you may have. How can I assist you today?", + role="assistant", + ), + ) + ], + created=1714401369, + model="replicate/meta/meta-llama-3-8b-instruct", + object="chat.completion", + system_fingerprint=None, + usage=litellm.utils.Usage( + prompt_tokens=48, completion_tokens=31, total_tokens=79 + ), + ) + cost = litellm.completion_cost( + completion_response=response, + messages=[{"role": "user", "content": "Hey, how's it going?"}], + ) + + print(f"cost: {cost}") + cost = round(cost, 5) + expected_cost = round( + litellm.model_cost["replicate/meta/meta-llama-3-8b-instruct"][ + "input_cost_per_token" + ] + * 48 + + litellm.model_cost["replicate/meta/meta-llama-3-8b-instruct"][ + "output_cost_per_token" + ] + * 31, + 5, + ) + assert cost == expected_cost diff --git a/litellm/utils.py b/litellm/utils.py index 6e62b64c9f..e1d80c2d2e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4269,8 +4269,8 @@ def completion_cost( model = get_model_params_and_category(model) # replicate llms are calculate based on time for request running # see https://replicate.com/pricing - elif model in litellm.replicate_models or "replicate" in model: - return get_replicate_completion_pricing(completion_response, total_time) + # elif model in litellm.replicate_models or "replicate" in model: + # return get_replicate_completion_pricing(completion_response, total_time) ( prompt_tokens_cost_usd_dollar, From ab954243e814c92a1807d22ca65930c9df44d6c0 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 08:09:59 -0700 Subject: [PATCH 09/52] fix(utils.py): fix watson streaming --- litellm/utils.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 6e62b64c9f..bb8df09b0d 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10154,21 +10154,6 @@ class CustomStreamWrapper: elif self.custom_llm_provider == "watsonx": response_obj = self.handle_watsonx_stream(chunk) completion_obj["content"] = response_obj["text"] - print_verbose(f"completion obj content: {completion_obj['content']}") - if response_obj.get("prompt_tokens") is not None: - prompt_token_count = getattr( - model_response.usage, "prompt_tokens", 0 - ) - model_response.usage.prompt_tokens = ( - prompt_token_count + response_obj["prompt_tokens"] - ) - if response_obj.get("completion_tokens") is not None: - model_response.usage.completion_tokens = response_obj[ - "completion_tokens" - ] - model_response.usage.total_tokens = getattr( - model_response.usage, "prompt_tokens", 0 - ) + getattr(model_response.usage, "completion_tokens", 0) if response_obj["is_finished"]: self.received_finish_reason = response_obj["finish_reason"] elif self.custom_llm_provider == "text-completion-openai": From dc5c17540600717de839112235f381467dee0e83 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 08:20:44 -0700 Subject: [PATCH 10/52] build(model_prices_and_context_window.json): add token-based replicate costs to model cost map --- ...odel_prices_and_context_window_backup.json | 117 ++++++++++++++++++ model_prices_and_context_window.json | 117 ++++++++++++++++++ 2 files changed, 234 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b12edc262a..b695d80866 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1418,6 +1418,123 @@ "litellm_provider": "replicate", "mode": "chat" }, + "replicate/meta/llama-2-13b": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000001, + "output_cost_per_token": 0.0000005, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/meta/llama-2-13b-chat": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000001, + "output_cost_per_token": 0.0000005, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/meta/llama-2-70b": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000065, + "output_cost_per_token": 0.00000275, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/meta/llama-2-70b-chat": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000065, + "output_cost_per_token": 0.00000275, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/meta/llama-2-7b": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000005, + "output_cost_per_token": 0.00000025, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/meta/llama-2-7b-chat": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000005, + "output_cost_per_token": 0.00000025, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/meta/llama-3-70b": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000065, + "output_cost_per_token": 0.00000275, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/meta/llama-3-70b-instruct": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000065, + "output_cost_per_token": 0.00000275, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/meta/llama-3-8b": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000005, + "output_cost_per_token": 0.00000025, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/meta/llama-3-8b-instruct": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000005, + "output_cost_per_token": 0.00000025, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/mistralai/mistral-7b-v0.1": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000005, + "output_cost_per_token": 0.00000025, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/mistralai/mistral-7b-instruct-v0.2": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000005, + "output_cost_per_token": 0.00000025, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/mistralai/mixtral-8x7b-instruct-v0.1": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000003, + "output_cost_per_token": 0.000001, + "litellm_provider": "replicate", + "mode": "chat" + }, "openrouter/openai/gpt-3.5-turbo": { "max_tokens": 4095, "input_cost_per_token": 0.0000015, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b12edc262a..b695d80866 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1418,6 +1418,123 @@ "litellm_provider": "replicate", "mode": "chat" }, + "replicate/meta/llama-2-13b": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000001, + "output_cost_per_token": 0.0000005, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/meta/llama-2-13b-chat": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000001, + "output_cost_per_token": 0.0000005, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/meta/llama-2-70b": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000065, + "output_cost_per_token": 0.00000275, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/meta/llama-2-70b-chat": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000065, + "output_cost_per_token": 0.00000275, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/meta/llama-2-7b": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000005, + "output_cost_per_token": 0.00000025, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/meta/llama-2-7b-chat": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000005, + "output_cost_per_token": 0.00000025, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/meta/llama-3-70b": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000065, + "output_cost_per_token": 0.00000275, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/meta/llama-3-70b-instruct": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000065, + "output_cost_per_token": 0.00000275, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/meta/llama-3-8b": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000005, + "output_cost_per_token": 0.00000025, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/meta/llama-3-8b-instruct": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000005, + "output_cost_per_token": 0.00000025, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/mistralai/mistral-7b-v0.1": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000005, + "output_cost_per_token": 0.00000025, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/mistralai/mistral-7b-instruct-v0.2": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000005, + "output_cost_per_token": 0.00000025, + "litellm_provider": "replicate", + "mode": "chat" + }, + "replicate/mistralai/mixtral-8x7b-instruct-v0.1": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000003, + "output_cost_per_token": 0.000001, + "litellm_provider": "replicate", + "mode": "chat" + }, "openrouter/openai/gpt-3.5-turbo": { "max_tokens": 4095, "input_cost_per_token": 0.0000015, From a18844b2309d67bce975302cfff41e8f3d1fb0e6 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 08:28:31 -0700 Subject: [PATCH 11/52] fix(utils.py): use llama tokenizer for replicate models --- litellm/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index e1d80c2d2e..45bfba3031 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3668,7 +3668,7 @@ def _select_tokenizer(model: str): tokenizer = Tokenizer.from_str(json_str) return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} # llama2 - elif "llama-2" in model.lower(): + elif "llama-2" in model.lower() or "replicate" in model.lower(): tokenizer = Tokenizer.from_pretrained("hf-internal-testing/llama-tokenizer") return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} # default - tiktoken From 3725732c4d5db8a7ae58aea81adaeb09f8d3f128 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 08:36:01 -0700 Subject: [PATCH 12/52] fix(utils.py): default to time-based tracking for unmapped replicate models. fix time-based cost calc for replicate --- litellm/utils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 45bfba3031..d03443dfda 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3641,12 +3641,12 @@ def get_replicate_completion_pricing(completion_response=None, total_time=0.0): a100_80gb_price_per_second_public = ( 0.001400 # assume all calls sent to A100 80GB for now ) - if total_time == 0.0: + if total_time == 0.0: # total time is in ms start_time = completion_response["created"] end_time = completion_response["ended"] total_time = end_time - start_time - return a100_80gb_price_per_second_public * total_time + return a100_80gb_price_per_second_public * total_time / 1000 def _select_tokenizer(model: str): @@ -4269,8 +4269,11 @@ def completion_cost( model = get_model_params_and_category(model) # replicate llms are calculate based on time for request running # see https://replicate.com/pricing - # elif model in litellm.replicate_models or "replicate" in model: - # return get_replicate_completion_pricing(completion_response, total_time) + elif ( + model in litellm.replicate_models or "replicate" in model + ) and model not in litellm.model_cost: + # for unmapped replicate model, default to replicate's time tracking logic + return get_replicate_completion_pricing(completion_response, total_time) ( prompt_tokens_cost_usd_dollar, From 4b04b017df697e3ece40fb22b8c463fd7c8d65d6 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 09:16:44 -0700 Subject: [PATCH 13/52] =?UTF-8?q?bump:=20version=201.35.31=20=E2=86=92=201?= =?UTF-8?q?.35.32?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ae09ad3cbe..837717a97f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.35.31" +version = "1.35.32" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -80,7 +80,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.35.31" +version = "1.35.32" version_files = [ "pyproject.toml:^version" ] From 89e655c79ec7436406c71f011be3513e48ec7501 Mon Sep 17 00:00:00 2001 From: sumanth Date: Tue, 30 Apr 2024 00:29:38 +0530 Subject: [PATCH 14/52] usage based routing RPM count fix --- litellm/router_strategy/lowest_tpm_rpm.py | 2 +- litellm/router_strategy/lowest_tpm_rpm_v2.py | 2 +- litellm/tests/test_router_caching.py | 29 ++++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/litellm/router_strategy/lowest_tpm_rpm.py b/litellm/router_strategy/lowest_tpm_rpm.py index 0437c2affc..0a7773a84b 100644 --- a/litellm/router_strategy/lowest_tpm_rpm.py +++ b/litellm/router_strategy/lowest_tpm_rpm.py @@ -206,7 +206,7 @@ class LowestTPMLoggingHandler(CustomLogger): if item_tpm + input_tokens > _deployment_tpm: continue elif (rpm_dict is not None and item in rpm_dict) and ( - rpm_dict[item] + 1 > _deployment_rpm + rpm_dict[item] + 1 >= _deployment_rpm ): continue elif item_tpm < lowest_tpm: diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py index 39dbcd9d05..f61484e08c 100644 --- a/litellm/router_strategy/lowest_tpm_rpm_v2.py +++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py @@ -366,7 +366,7 @@ class LowestTPMLoggingHandler_v2(CustomLogger): if item_tpm + input_tokens > _deployment_tpm: continue elif (rpm_dict is not None and item in rpm_dict) and ( - rpm_dict[item] + 1 > _deployment_rpm + rpm_dict[item] + 1 >= _deployment_rpm ): continue elif item_tpm < lowest_tpm: diff --git a/litellm/tests/test_router_caching.py b/litellm/tests/test_router_caching.py index ebace161c9..ce03498f92 100644 --- a/litellm/tests/test_router_caching.py +++ b/litellm/tests/test_router_caching.py @@ -264,3 +264,32 @@ async def test_acompletion_caching_on_router_caching_groups(): except Exception as e: traceback.print_exc() pytest.fail(f"Error occurred: {e}") + +def test_rpm_limiting(): + try: + litellm.set_verbose = True + model_list = [ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": os.getenv("OPENAI_API_KEY"), + }, + "tpm": 10000, + "rpm": 3, + }, + ] + + router = Router( + model_list = model_list, + routing_strategy = "usage-based-routing", + ) + failedCount = 0 + for i in range(10): + try: + response = router.completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": ""}]) + except ValueError as e: + failedCount += 1 + assert failedCount == 7 + except Exception as e: + pytest.fail(f"An exception occurred - {str(e)}") \ No newline at end of file From 8d26030b9912c858382a855d0c7b69d7d5aabf6f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Apr 2024 13:15:08 -0700 Subject: [PATCH 15/52] docs - track cost custom callbacks --- .../docs/observability/custom_callback.md | 36 ++++--------------- 1 file changed, 6 insertions(+), 30 deletions(-) diff --git a/docs/my-website/docs/observability/custom_callback.md b/docs/my-website/docs/observability/custom_callback.md index 7cc38168bc..3168222273 100644 --- a/docs/my-website/docs/observability/custom_callback.md +++ b/docs/my-website/docs/observability/custom_callback.md @@ -331,49 +331,25 @@ response = litellm.completion(model="gpt-3.5-turbo", messages=messages, metadata ## Examples ### Custom Callback to track costs for Streaming + Non-Streaming +By default, the response cost is accessible in the logging object via `kwargs["response_cost"]` on success (sync + async) ```python +# Step 1. Write your custom callback function def track_cost_callback( kwargs, # kwargs to completion completion_response, # response from completion start_time, end_time # start/end time ): try: - # init logging config - logging.basicConfig( - filename='cost.log', - level=logging.INFO, - format='%(asctime)s - %(message)s', - datefmt='%Y-%m-%d %H:%M:%S' - ) - - # check if it has collected an entire stream response - if "complete_streaming_response" in kwargs: - # for tracking streaming cost we pass the "messages" and the output_text to litellm.completion_cost - completion_response=kwargs["complete_streaming_response"] - input_text = kwargs["messages"] - output_text = completion_response["choices"][0]["message"]["content"] - response_cost = litellm.completion_cost( - model = kwargs["model"], - messages = input_text, - completion=output_text - ) - print("streaming response_cost", response_cost) - logging.info(f"Model {kwargs['model']} Cost: ${response_cost:.8f}") - - # for non streaming responses - else: - # we pass the completion_response obj - if kwargs["stream"] != True: - response_cost = litellm.completion_cost(completion_response=completion_response) - print("regular response_cost", response_cost) - logging.info(f"Model {completion_response.model} Cost: ${response_cost:.8f}") + response_cost = kwargs["response_cost"] # litellm calculates response cost for you + print("regular response_cost", response_cost) except: pass -# Assign the custom callback function +# Step 2. Assign the custom callback function litellm.success_callback = [track_cost_callback] +# Step 3. Make litellm.completion call response = completion( model="gpt-3.5-turbo", messages=[ From f10a066d360ba460394f1792289cd17d38866a29 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 15:04:22 -0700 Subject: [PATCH 16/52] fix(lowest_tpm_rpm_v2.py): add more detail to 'No deployments available' error message --- litellm/router.py | 6 +- litellm/router_strategy/lowest_tpm_rpm_v2.py | 115 ++++++++++++++++++- litellm/types/router.py | 1 + 3 files changed, 117 insertions(+), 5 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index df4c2e046a..b66c29533e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2807,7 +2807,7 @@ class Router: if _rate_limit_error == True: # allow generic fallback logic to take place raise ValueError( - f"No deployments available for selected model, passed model={model}" + f"{RouterErrors.no_deployments_available.value}, passed model={model}" ) elif _context_window_error == True: raise litellm.ContextWindowExceededError( @@ -3000,7 +3000,7 @@ class Router: f"get_available_deployment for model: {model}, No deployment available" ) raise ValueError( - f"No deployments available for selected model, passed model={model}" + f"{RouterErrors.no_deployments_available.value}, passed model={model}" ) verbose_router_logger.info( f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment)} for model: {model}" @@ -3130,7 +3130,7 @@ class Router: f"get_available_deployment for model: {model}, No deployment available" ) raise ValueError( - f"No deployments available for selected model, passed model={model}" + f"{RouterErrors.no_deployments_available.value}, passed model={model}" ) verbose_router_logger.info( f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment)} for model: {model}" diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py index 39dbcd9d05..a11c6d8723 100644 --- a/litellm/router_strategy/lowest_tpm_rpm_v2.py +++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py @@ -394,6 +394,7 @@ class LowestTPMLoggingHandler_v2(CustomLogger): dt = get_utc_datetime() current_minute = dt.strftime("%H-%M") + tpm_keys = [] rpm_keys = [] for m in healthy_deployments: @@ -416,7 +417,7 @@ class LowestTPMLoggingHandler_v2(CustomLogger): tpm_values = combined_tpm_rpm_values[: len(tpm_keys)] rpm_values = combined_tpm_rpm_values[len(tpm_keys) :] - return self._common_checks_available_deployment( + deployment = self._common_checks_available_deployment( model_group=model_group, healthy_deployments=healthy_deployments, tpm_keys=tpm_keys, @@ -427,6 +428,61 @@ class LowestTPMLoggingHandler_v2(CustomLogger): input=input, ) + try: + assert deployment is not None + return deployment + except Exception as e: + ### GET THE DICT OF TPM / RPM + LIMITS PER DEPLOYMENT ### + deployment_dict = {} + for index, _deployment in enumerate(healthy_deployments): + if isinstance(_deployment, dict): + id = _deployment.get("model_info", {}).get("id") + ### GET DEPLOYMENT TPM LIMIT ### + _deployment_tpm = None + if _deployment_tpm is None: + _deployment_tpm = _deployment.get("tpm", None) + if _deployment_tpm is None: + _deployment_tpm = _deployment.get("litellm_params", {}).get( + "tpm", None + ) + if _deployment_tpm is None: + _deployment_tpm = _deployment.get("model_info", {}).get( + "tpm", None + ) + if _deployment_tpm is None: + _deployment_tpm = float("inf") + + ### GET CURRENT TPM ### + current_tpm = tpm_values[index] + + ### GET DEPLOYMENT TPM LIMIT ### + _deployment_rpm = None + if _deployment_rpm is None: + _deployment_rpm = _deployment.get("rpm", None) + if _deployment_rpm is None: + _deployment_rpm = _deployment.get("litellm_params", {}).get( + "rpm", None + ) + if _deployment_rpm is None: + _deployment_rpm = _deployment.get("model_info", {}).get( + "rpm", None + ) + if _deployment_rpm is None: + _deployment_rpm = float("inf") + + ### GET CURRENT RPM ### + current_rpm = rpm_values[index] + + deployment_dict[id] = { + "current_tpm": current_tpm, + "tpm_limit": _deployment_tpm, + "current_rpm": current_rpm, + "rpm_limit": _deployment_rpm, + } + raise ValueError( + f"{RouterErrors.no_deployments_available.value}. Passed model={model_group}. Deployments={deployment_dict}" + ) + def get_available_deployments( self, model_group: str, @@ -464,7 +520,7 @@ class LowestTPMLoggingHandler_v2(CustomLogger): keys=rpm_keys ) # [1, 2, None, ..] - return self._common_checks_available_deployment( + deployment = self._common_checks_available_deployment( model_group=model_group, healthy_deployments=healthy_deployments, tpm_keys=tpm_keys, @@ -474,3 +530,58 @@ class LowestTPMLoggingHandler_v2(CustomLogger): messages=messages, input=input, ) + + try: + assert deployment is not None + return deployment + except Exception as e: + ### GET THE DICT OF TPM / RPM + LIMITS PER DEPLOYMENT ### + deployment_dict = {} + for index, _deployment in enumerate(healthy_deployments): + if isinstance(_deployment, dict): + id = _deployment.get("model_info", {}).get("id") + ### GET DEPLOYMENT TPM LIMIT ### + _deployment_tpm = None + if _deployment_tpm is None: + _deployment_tpm = _deployment.get("tpm", None) + if _deployment_tpm is None: + _deployment_tpm = _deployment.get("litellm_params", {}).get( + "tpm", None + ) + if _deployment_tpm is None: + _deployment_tpm = _deployment.get("model_info", {}).get( + "tpm", None + ) + if _deployment_tpm is None: + _deployment_tpm = float("inf") + + ### GET CURRENT TPM ### + current_tpm = tpm_values[index] + + ### GET DEPLOYMENT TPM LIMIT ### + _deployment_rpm = None + if _deployment_rpm is None: + _deployment_rpm = _deployment.get("rpm", None) + if _deployment_rpm is None: + _deployment_rpm = _deployment.get("litellm_params", {}).get( + "rpm", None + ) + if _deployment_rpm is None: + _deployment_rpm = _deployment.get("model_info", {}).get( + "rpm", None + ) + if _deployment_rpm is None: + _deployment_rpm = float("inf") + + ### GET CURRENT RPM ### + current_rpm = rpm_values[index] + + deployment_dict[id] = { + "current_tpm": current_tpm, + "tpm_limit": _deployment_tpm, + "current_rpm": current_rpm, + "rpm_limit": _deployment_rpm, + } + raise ValueError( + f"{RouterErrors.no_deployments_available.value}. Passed model={model_group}. Deployments={deployment_dict}" + ) diff --git a/litellm/types/router.py b/litellm/types/router.py index 042d9f277c..64b71b999e 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -263,3 +263,4 @@ class RouterErrors(enum.Enum): """ user_defined_ratelimit_error = "Deployment over user-defined ratelimit." + no_deployments_available = "No deployments available for selected model" From a978f2d8813c04dad34802cb95e0a0e35a3324bc Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 15:23:47 -0700 Subject: [PATCH 17/52] fix(lowest_tpm_rpm_v2.py): shuffle deployments with same tpm values --- litellm/router_strategy/lowest_tpm_rpm_v2.py | 12 +++- litellm/tests/test_tpm_rpm_routing_v2.py | 58 ++++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py index a11c6d8723..3300e56ff7 100644 --- a/litellm/router_strategy/lowest_tpm_rpm_v2.py +++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py @@ -333,7 +333,7 @@ class LowestTPMLoggingHandler_v2(CustomLogger): tpm_dict[tpm_key] = 0 all_deployments = tpm_dict - deployment = None + potential_deployments = [] # if multiple deployments have the same low value for item, item_tpm in all_deployments.items(): ## get the item from model list _deployment = None @@ -369,11 +369,17 @@ class LowestTPMLoggingHandler_v2(CustomLogger): rpm_dict[item] + 1 > _deployment_rpm ): continue + elif item_tpm == lowest_tpm: + potential_deployments.append(_deployment) elif item_tpm < lowest_tpm: lowest_tpm = item_tpm - deployment = _deployment + potential_deployments = [_deployment] print_verbose("returning picked lowest tpm/rpm deployment.") - return deployment + + if len(potential_deployments) > 0: + return random.choice(potential_deployments) + else: + return None async def async_get_available_deployments( self, diff --git a/litellm/tests/test_tpm_rpm_routing_v2.py b/litellm/tests/test_tpm_rpm_routing_v2.py index 9a43ae3ca1..78620a6e55 100644 --- a/litellm/tests/test_tpm_rpm_routing_v2.py +++ b/litellm/tests/test_tpm_rpm_routing_v2.py @@ -282,6 +282,64 @@ def test_router_skip_rate_limited_deployments(): print(f"An exception occurred! {str(e)}") +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_multiple_potential_deployments(sync_mode): + """ + If multiple deployments have the same tpm value + + call 5 times, test if deployments are shuffled. + + -> prevents single deployment from being overloaded in high-concurrency scenario + """ + + model_list = [ + { + "model_name": "azure-model", + "litellm_params": { + "model": "azure/gpt-turbo", + "api_key": "os.environ/AZURE_FRANCE_API_KEY", + "api_base": "https://openai-france-1234.openai.azure.com", + "tpm": 1440, + }, + }, + { + "model_name": "azure-model", + "litellm_params": { + "model": "azure/gpt-turbo-2", + "api_key": "os.environ/AZURE_FRANCE_API_KEY", + "api_base": "https://openai-france-1234.openai.azure.com", + "tpm": 1440, + }, + }, + ] + router = Router( + model_list=model_list, + routing_strategy="usage-based-routing-v2", + set_verbose=False, + num_retries=3, + ) # type: ignore + + model_ids = set() + for _ in range(5): + if sync_mode: + deployment = router.get_available_deployment( + model="azure-model", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + ) + else: + deployment = await router.async_get_available_deployment( + model="azure-model", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + ) + + ## get id ## + id = deployment.get("model_info", {}).get("id") + model_ids.add(id) + + assert len(model_ids) == 2 + + def test_single_deployment_tpm_zero(): import litellm import os From cef2d95bb40eae61c12fec1a4127fed8642b55b5 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 15:37:48 -0700 Subject: [PATCH 18/52] docs(routing.md): add max parallel requests to router docs --- docs/my-website/docs/routing.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index 76d3acb7b1..028b40b6fd 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -443,6 +443,35 @@ asyncio.run(router_acompletion()) ## Basic Reliability +### Max Parallel Requests (ASYNC) + +Used in semaphore for async requests on router. Limit the max concurrent calls made to a deployment. Useful in high-traffic scenarios. + +If tpm/rpm is set, and no max parallel request limit given, we use the RPM or calculated RPM (tpm/1000/6) as the max parallel request limit. + + +```python +from litellm import Router + +model_list = [{ + "model_name": "gpt-4", + "litellm_params": { + "model": "azure/gpt-4", + ... + "max_parallel_requests": 10 # 👈 SET PER DEPLOYMENT + } +}] + +### OR ### + +router = Router(model_list=model_list, default_max_parallel_requests=20) # 👈 SET DEFAULT MAX PARALLEL REQUESTS + + +# deployment max parallel requests > default max parallel requests +``` + +[**See Code**](https://github.com/BerriAI/litellm/blob/a978f2d8813c04dad34802cb95e0a0e35a3324bc/litellm/utils.py#L5605) + ### Timeouts The timeout set in router is for the entire length of the call, and is passed down to the completion() call level as well. From 5247d7b6a5de50df28a5e091c111d4d45e4d2023 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Apr 2024 15:51:01 -0700 Subject: [PATCH 19/52] test - lowest latency router --- litellm/tests/test_lowest_latency_routing.py | 76 ++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/litellm/tests/test_lowest_latency_routing.py b/litellm/tests/test_lowest_latency_routing.py index 4b93853f41..24e6bb4c5d 100644 --- a/litellm/tests/test_lowest_latency_routing.py +++ b/litellm/tests/test_lowest_latency_routing.py @@ -555,3 +555,79 @@ async def test_lowest_latency_routing_with_timeouts(): # ALL the Requests should have been routed to the fast-endpoint assert deployments["fast-endpoint"] == 10 + + +@pytest.mark.asyncio +async def test_lowest_latency_routing_first_pick(): + """ + PROD Test: + - When all deployments are latency=0, it should randomly pick a deployment + - IT SHOULD NEVER PICK THE Very First deployment everytime all deployment latencies are 0 + - This ensures that after the ttl window resets it randomly picks a deployment + """ + import litellm + + litellm.set_verbose = True + + router = Router( + model_list=[ + { + "model_name": "azure-model", + "litellm_params": { + "model": "openai/fast-endpoint", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "api_key": "fake-key", + }, + "model_info": {"id": "fast-endpoint"}, + }, + { + "model_name": "azure-model", + "litellm_params": { + "model": "openai/fast-endpoint-2", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "api_key": "fake-key", + }, + "model_info": {"id": "fast-endpoint-2"}, + }, + { + "model_name": "azure-model", + "litellm_params": { + "model": "openai/fast-endpoint-2", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "api_key": "fake-key", + }, + "model_info": {"id": "fast-endpoint-3"}, + }, + { + "model_name": "azure-model", + "litellm_params": { + "model": "openai/fast-endpoint-2", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "api_key": "fake-key", + }, + "model_info": {"id": "fast-endpoint-4"}, + }, + ], + routing_strategy="latency-based-routing", + routing_strategy_args={"ttl": 0.0000000001}, + set_verbose=True, + debug_level="DEBUG", + ) # type: ignore + + deployments = {} + for _ in range(5): + response = await router.acompletion( + model="azure-model", messages=[{"role": "user", "content": "hello"}] + ) + print(response) + _picked_model_id = response._hidden_params["model_id"] + if _picked_model_id not in deployments: + deployments[_picked_model_id] = 1 + else: + deployments[_picked_model_id] += 1 + await asyncio.sleep(0.000000000005) + + print("deployments", deployments) + + # assert that len(deployments) >1 + assert len(deployments) > 1 From 3b0aa0537854dbb53dffe402569b4fe3722ce42c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Apr 2024 15:51:52 -0700 Subject: [PATCH 20/52] fix lowest latency - routing --- litellm/router_strategy/lowest_latency.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index 19780f708d..cd5cea2e94 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -312,6 +312,8 @@ class LowestLatencyLoggingHandler(CustomLogger): except: input_tokens = 0 + all_deployments = random.sample(all_deployments.items(), len(all_deployments)) + all_deployments = dict(all_deployments) for item, item_map in all_deployments.items(): ## get the item from model list _deployment = None From 5fe0f38558f69b3ab58c34580513c8cdea6bff72 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 15:58:14 -0700 Subject: [PATCH 21/52] docs(load_test.md): load test multiple instances of the proxy w/ tpm/rpm limits on deployments --- docs/my-website/docs/load_test.md | 236 ++++++++++++++++++++++++ litellm/proxy/_super_secret_config.yaml | 20 +- 2 files changed, 244 insertions(+), 12 deletions(-) diff --git a/docs/my-website/docs/load_test.md b/docs/my-website/docs/load_test.md index 5eb6a06109..08282d0636 100644 --- a/docs/my-website/docs/load_test.md +++ b/docs/my-website/docs/load_test.md @@ -213,3 +213,239 @@ asyncio.run(loadtest_fn()) ``` +## Multi-Instance TPM/RPM Load Test + +Test if your defined tpm/rpm limits are respected across multiple instances. + +The quickest way to do this is by testing the [proxy](./proxy/quick_start.md). The proxy uses the [router](./routing.md) under the hood, so if you're using either of them, this test should work for you. + +In our test: +- Max RPM per deployment is 100 requests per minute +- Max Throughput / min on proxy = 200 requests per minute (2 deployments) +- Load we'll send to proxy = 600 requests per minute + + +So we'll send 600 requests per minute, but expect only 200 requests per minute to succeed. + +### 0. Setup Fake OpenAI Server + +Let's setup a fake openai server with a RPM limit of 100. + +Let's call our file `fake_openai_server.py`. + +``` +# import sys, os +# sys.path.insert( +# 0, os.path.abspath("../") +# ) # Adds the parent directory to the system path +from fastapi import FastAPI, Request, status, HTTPException, Depends +from fastapi.responses import StreamingResponse +from fastapi.security import OAuth2PasswordBearer +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse +from fastapi import FastAPI, Request, HTTPException, UploadFile, File +import httpx, os, json +from openai import AsyncOpenAI +from typing import Optional +from slowapi import Limiter +from slowapi.util import get_remote_address +from slowapi.errors import RateLimitExceeded +from fastapi import FastAPI, Request, HTTPException +from fastapi.responses import PlainTextResponse + + +class ProxyException(Exception): + # NOTE: DO NOT MODIFY THIS + # This is used to map exactly to OPENAI Exceptions + def __init__( + self, + message: str, + type: str, + param: Optional[str], + code: Optional[int], + ): + self.message = message + self.type = type + self.param = param + self.code = code + + def to_dict(self) -> dict: + """Converts the ProxyException instance to a dictionary.""" + return { + "message": self.message, + "type": self.type, + "param": self.param, + "code": self.code, + } + + +limiter = Limiter(key_func=get_remote_address) +app = FastAPI() +app.state.limiter = limiter + +@app.exception_handler(RateLimitExceeded) +async def _rate_limit_exceeded_handler(request: Request, exc: RateLimitExceeded): + return JSONResponse(status_code=429, + content={"detail": "Rate Limited!"}) + +app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +# for completion +@app.post("/chat/completions") +@app.post("/v1/chat/completions") +@limiter.limit("100/minute") +async def completion(request: Request): + # raise HTTPException(status_code=429, detail="Rate Limited!") + return { + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": None, + "system_fingerprint": "fp_44709d6fcb", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "\n\nHello there, how may I assist you today?", + }, + "logprobs": None, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21 + } + } + +if __name__ == "__main__": + import socket + import uvicorn + port = 8080 + while True: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = sock.connect_ex(('0.0.0.0', port)) + if result != 0: + print(f"Port {port} is available, starting server...") + break + else: + port += 1 + + uvicorn.run(app, host="0.0.0.0", port=port) +``` + +```bash +python3 fake_openai_server.py +``` + +### 1. Setup config + +```yaml +model_list: +- litellm_params: + api_base: http://0.0.0.0:8080 + api_key: my-fake-key + model: openai/my-fake-model + rpm: 100 + model_name: fake-openai-endpoint +- litellm_params: + api_base: http://0.0.0.0:8081 + api_key: my-fake-key + model: openai/my-fake-model-2 + rpm: 100 + model_name: fake-openai-endpoint +router_settings: + num_retries: 0 + enable_pre_call_checks: true + redis_host: os.environ/REDIS_HOST ## 👈 IMPORTANT! Setup the proxy w/ redis + redis_password: os.environ/REDIS_PASSWORD + redis_port: os.environ/REDIS_PORT +``` + +### 2. Start proxy 2 instances + +**Instance 1** +```bash +litellm --config /path/to/config.yaml --port 4000 + +## RUNNING on http://0.0.0.0:4000 +``` + +**Instance 2** +```bash +litellm --config /path/to/config.yaml --port 4001 + +## RUNNING on http://0.0.0.0:4001 +``` + +### 3. Run Test + +Let's hit the proxy with 600 requests per minute. + +```python +from openai import AsyncOpenAI, AsyncAzureOpenAI +import random, uuid +import time, asyncio, litellm +# import logging +# logging.basicConfig(level=logging.DEBUG) +#### LITELLM PROXY #### +litellm_client = AsyncOpenAI( + api_key="sk-1234", # [CHANGE THIS] + base_url="http://0.0.0.0:4000" +) +litellm_client_2 = AsyncOpenAI( + api_key="sk-1234", # [CHANGE THIS] + base_url="http://0.0.0.0:4001" +) + +async def proxy_completion_non_streaming(): + try: + client = random.sample([litellm_client, litellm_client_2], 1)[0] # randomly pick b/w clients + # print(f"client={client}") + response = await client.chat.completions.create( + model="fake-openai-endpoint", # [CHANGE THIS] (if you call it something else on your proxy) + messages=[{"role": "user", "content": f"This is a test: {uuid.uuid4()}"}], + ) + return response + except Exception as e: + # print(e) + return None + +async def loadtest_fn(): + start = time.time() + n = 500 # Number of concurrent tasks + tasks = [proxy_completion_non_streaming() for _ in range(n)] + chat_completions = await asyncio.gather(*tasks) + successful_completions = [c for c in chat_completions if c is not None] + print(n, time.time() - start, len(successful_completions)) + +def get_utc_datetime(): + import datetime as dt + from datetime import datetime + + if hasattr(dt, "UTC"): + return datetime.now(dt.UTC) # type: ignore + else: + return datetime.utcnow() # type: ignore + + +# Run the event loop to execute the async function +async def parent_fn(): + for _ in range(10): + dt = get_utc_datetime() + current_minute = dt.strftime("%H-%M") + print(f"triggered new batch - {current_minute}") + await loadtest_fn() + await asyncio.sleep(10) + +asyncio.run(parent_fn()) + +``` \ No newline at end of file diff --git a/litellm/proxy/_super_secret_config.yaml b/litellm/proxy/_super_secret_config.yaml index 0e1b4b2e13..9db128d0e2 100644 --- a/litellm/proxy/_super_secret_config.yaml +++ b/litellm/proxy/_super_secret_config.yaml @@ -3,21 +3,17 @@ model_list: api_base: http://0.0.0.0:8080 api_key: my-fake-key model: openai/my-fake-model + rpm: 100 model_name: fake-openai-endpoint - litellm_params: - api_base: http://0.0.0.0:8080 + api_base: http://0.0.0.0:8081 api_key: my-fake-key model: openai/my-fake-model-2 - model_name: fake-openai-endpoint -- litellm_params: - api_base: http://0.0.0.0:8080 - api_key: my-fake-key - model: openai/my-fake-model-3 - model_name: fake-openai-endpoint -- litellm_params: - api_base: http://0.0.0.0:8080 - api_key: my-fake-key - model: openai/my-fake-model-4 + rpm: 100 model_name: fake-openai-endpoint router_settings: - num_retries: 0 \ No newline at end of file + num_retries: 0 + enable_pre_call_checks: true + redis_host: os.environ/REDIS_HOST + redis_password: os.environ/REDIS_PASSWORD + redis_port: os.environ/REDIS_PORT \ No newline at end of file From fcb83781ec0ca05727584351fc3e3f8cfca28f7c Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 15:58:41 -0700 Subject: [PATCH 22/52] docs(load_test.md): formatting --- docs/my-website/docs/load_test.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/load_test.md b/docs/my-website/docs/load_test.md index 08282d0636..fb737c0f2e 100644 --- a/docs/my-website/docs/load_test.md +++ b/docs/my-website/docs/load_test.md @@ -220,7 +220,7 @@ Test if your defined tpm/rpm limits are respected across multiple instances. The quickest way to do this is by testing the [proxy](./proxy/quick_start.md). The proxy uses the [router](./routing.md) under the hood, so if you're using either of them, this test should work for you. In our test: -- Max RPM per deployment is 100 requests per minute +- Max RPM per deployment is = 100 requests per minute - Max Throughput / min on proxy = 200 requests per minute (2 deployments) - Load we'll send to proxy = 600 requests per minute From 8f830bd94864eb5c81346d658152671ce5cf67fb Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 16:00:02 -0700 Subject: [PATCH 23/52] docs(load_test.md): simplify doc --- docs/my-website/docs/load_test.md | 217 +++++++++++++++--------------- 1 file changed, 112 insertions(+), 105 deletions(-) diff --git a/docs/my-website/docs/load_test.md b/docs/my-website/docs/load_test.md index fb737c0f2e..bd97c62290 100644 --- a/docs/my-website/docs/load_test.md +++ b/docs/my-website/docs/load_test.md @@ -227,7 +227,118 @@ In our test: So we'll send 600 requests per minute, but expect only 200 requests per minute to succeed. -### 0. Setup Fake OpenAI Server +:::info + +If you don't want to call a real LLM API endpoint, you can setup a fake openai server. [See code](#extra---setup-fake-openai-server) + +::: + +### 1. Setup config + +```yaml +model_list: +- litellm_params: + api_base: http://0.0.0.0:8080 + api_key: my-fake-key + model: openai/my-fake-model + rpm: 100 + model_name: fake-openai-endpoint +- litellm_params: + api_base: http://0.0.0.0:8081 + api_key: my-fake-key + model: openai/my-fake-model-2 + rpm: 100 + model_name: fake-openai-endpoint +router_settings: + num_retries: 0 + enable_pre_call_checks: true + redis_host: os.environ/REDIS_HOST ## 👈 IMPORTANT! Setup the proxy w/ redis + redis_password: os.environ/REDIS_PASSWORD + redis_port: os.environ/REDIS_PORT +``` + +### 2. Start proxy 2 instances + +**Instance 1** +```bash +litellm --config /path/to/config.yaml --port 4000 + +## RUNNING on http://0.0.0.0:4000 +``` + +**Instance 2** +```bash +litellm --config /path/to/config.yaml --port 4001 + +## RUNNING on http://0.0.0.0:4001 +``` + +### 3. Run Test + +Let's hit the proxy with 600 requests per minute. + +```python +from openai import AsyncOpenAI, AsyncAzureOpenAI +import random, uuid +import time, asyncio, litellm +# import logging +# logging.basicConfig(level=logging.DEBUG) +#### LITELLM PROXY #### +litellm_client = AsyncOpenAI( + api_key="sk-1234", # [CHANGE THIS] + base_url="http://0.0.0.0:4000" +) +litellm_client_2 = AsyncOpenAI( + api_key="sk-1234", # [CHANGE THIS] + base_url="http://0.0.0.0:4001" +) + +async def proxy_completion_non_streaming(): + try: + client = random.sample([litellm_client, litellm_client_2], 1)[0] # randomly pick b/w clients + # print(f"client={client}") + response = await client.chat.completions.create( + model="fake-openai-endpoint", # [CHANGE THIS] (if you call it something else on your proxy) + messages=[{"role": "user", "content": f"This is a test: {uuid.uuid4()}"}], + ) + return response + except Exception as e: + # print(e) + return None + +async def loadtest_fn(): + start = time.time() + n = 500 # Number of concurrent tasks + tasks = [proxy_completion_non_streaming() for _ in range(n)] + chat_completions = await asyncio.gather(*tasks) + successful_completions = [c for c in chat_completions if c is not None] + print(n, time.time() - start, len(successful_completions)) + +def get_utc_datetime(): + import datetime as dt + from datetime import datetime + + if hasattr(dt, "UTC"): + return datetime.now(dt.UTC) # type: ignore + else: + return datetime.utcnow() # type: ignore + + +# Run the event loop to execute the async function +async def parent_fn(): + for _ in range(10): + dt = get_utc_datetime() + current_minute = dt.strftime("%H-%M") + print(f"triggered new batch - {current_minute}") + await loadtest_fn() + await asyncio.sleep(10) + +asyncio.run(parent_fn()) + +``` + + +### Extra - Setup Fake OpenAI Server Let's setup a fake openai server with a RPM limit of 100. @@ -345,107 +456,3 @@ if __name__ == "__main__": ```bash python3 fake_openai_server.py ``` - -### 1. Setup config - -```yaml -model_list: -- litellm_params: - api_base: http://0.0.0.0:8080 - api_key: my-fake-key - model: openai/my-fake-model - rpm: 100 - model_name: fake-openai-endpoint -- litellm_params: - api_base: http://0.0.0.0:8081 - api_key: my-fake-key - model: openai/my-fake-model-2 - rpm: 100 - model_name: fake-openai-endpoint -router_settings: - num_retries: 0 - enable_pre_call_checks: true - redis_host: os.environ/REDIS_HOST ## 👈 IMPORTANT! Setup the proxy w/ redis - redis_password: os.environ/REDIS_PASSWORD - redis_port: os.environ/REDIS_PORT -``` - -### 2. Start proxy 2 instances - -**Instance 1** -```bash -litellm --config /path/to/config.yaml --port 4000 - -## RUNNING on http://0.0.0.0:4000 -``` - -**Instance 2** -```bash -litellm --config /path/to/config.yaml --port 4001 - -## RUNNING on http://0.0.0.0:4001 -``` - -### 3. Run Test - -Let's hit the proxy with 600 requests per minute. - -```python -from openai import AsyncOpenAI, AsyncAzureOpenAI -import random, uuid -import time, asyncio, litellm -# import logging -# logging.basicConfig(level=logging.DEBUG) -#### LITELLM PROXY #### -litellm_client = AsyncOpenAI( - api_key="sk-1234", # [CHANGE THIS] - base_url="http://0.0.0.0:4000" -) -litellm_client_2 = AsyncOpenAI( - api_key="sk-1234", # [CHANGE THIS] - base_url="http://0.0.0.0:4001" -) - -async def proxy_completion_non_streaming(): - try: - client = random.sample([litellm_client, litellm_client_2], 1)[0] # randomly pick b/w clients - # print(f"client={client}") - response = await client.chat.completions.create( - model="fake-openai-endpoint", # [CHANGE THIS] (if you call it something else on your proxy) - messages=[{"role": "user", "content": f"This is a test: {uuid.uuid4()}"}], - ) - return response - except Exception as e: - # print(e) - return None - -async def loadtest_fn(): - start = time.time() - n = 500 # Number of concurrent tasks - tasks = [proxy_completion_non_streaming() for _ in range(n)] - chat_completions = await asyncio.gather(*tasks) - successful_completions = [c for c in chat_completions if c is not None] - print(n, time.time() - start, len(successful_completions)) - -def get_utc_datetime(): - import datetime as dt - from datetime import datetime - - if hasattr(dt, "UTC"): - return datetime.now(dt.UTC) # type: ignore - else: - return datetime.utcnow() # type: ignore - - -# Run the event loop to execute the async function -async def parent_fn(): - for _ in range(10): - dt = get_utc_datetime() - current_minute = dt.strftime("%H-%M") - print(f"triggered new batch - {current_minute}") - await loadtest_fn() - await asyncio.sleep(10) - -asyncio.run(parent_fn()) - -``` \ No newline at end of file From 4cb4a7f06def393849f88d90d42adc02ed09ff61 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Apr 2024 16:02:57 -0700 Subject: [PATCH 24/52] fix - lowest latency routing --- litellm/router_strategy/lowest_latency.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index cd5cea2e94..eecf5578ce 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -312,7 +312,9 @@ class LowestLatencyLoggingHandler(CustomLogger): except: input_tokens = 0 - all_deployments = random.sample(all_deployments.items(), len(all_deployments)) + # randomly sample from all_deployments, incase all deployments have latency=0.0 + _items = all_deployments.items() + all_deployments = random.sample(list(_items), len(_items)) all_deployments = dict(all_deployments) for item, item_map in all_deployments.items(): ## get the item from model list From af6a21f27ceb9759a9521558df7108a6d881d50e Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 16:25:56 -0700 Subject: [PATCH 25/52] docs(load_test.md): add multi-instance router load test to docs --- docs/my-website/docs/load_test.md | 190 +++++++++++++++++++++++++++++- 1 file changed, 188 insertions(+), 2 deletions(-) diff --git a/docs/my-website/docs/load_test.md b/docs/my-website/docs/load_test.md index bd97c62290..c1fac60fcb 100644 --- a/docs/my-website/docs/load_test.md +++ b/docs/my-website/docs/load_test.md @@ -213,7 +213,191 @@ asyncio.run(loadtest_fn()) ``` -## Multi-Instance TPM/RPM Load Test +## Multi-Instance TPM/RPM Load Test (Router) + +Test if your defined tpm/rpm limits are respected across multiple instances of the Router object. + +In our test: +- Max RPM per deployment is = 100 requests per minute +- Max Throughput / min on router = 200 requests per minute (2 deployments) +- Load we'll send through router = 600 requests per minute + +:::info + +If you don't want to call a real LLM API endpoint, you can setup a fake openai server. [See code](#extra---setup-fake-openai-server) + +::: + +### 1. Setup Router + +```python +from litellm import Router +import litellm +litellm.suppress_debug_info = True +litellm.set_verbose = False +import logging +logging.basicConfig(level=logging.CRITICAL) +import os, random, uuid, time, asyncio + +os.environ["REDIS_SSL"] = "True" + +# Model list for OpenAI and Anthropic models +model_list = [ + { + "model_name": "fake-openai-endpoint", + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": "my-fake-key", + "api_base": "http://0.0.0.0:8080", + "rpm": 100 + }, + }, + { + "model_name": "fake-openai-endpoint", + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": "my-fake-key", + "api_base": "http://0.0.0.0:8081", + "rpm": 100 + }, + }, +] + +router_1 = Router(model_list=model_list, num_retries=0, enable_pre_call_checks=True, redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD")) +router_2 = Router(model_list=model_list, num_retries=0, enable_pre_call_checks=True, redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD")) + + + +async def router_completion_non_streaming(): + try: + client: Router = random.sample([router_1, router_2], 1)[0] # randomly pick b/w clients + # print(f"client={client}") + response = await client.acompletion( + model="fake-openai-endpoint", # [CHANGE THIS] (if you call it something else on your proxy) + messages=[{"role": "user", "content": f"This is a test: {uuid.uuid4()}"}], + ) + return response + except Exception as e: + # print(e) + return None + +async def loadtest_fn(): + start = time.time() + n = 500 # Number of concurrent tasks + tasks = [router_completion_non_streaming() for _ in range(n)] + chat_completions = await asyncio.gather(*tasks) + successful_completions = [c for c in chat_completions if c is not None] + print(n, time.time() - start, len(successful_completions)) + +def get_utc_datetime(): + import datetime as dt + from datetime import datetime + + if hasattr(dt, "UTC"): + return datetime.now(dt.UTC) # type: ignore + else: + return datetime.utcnow() # type: ignore + + +# Run the event loop to execute the async function +async def parent_fn(): + for _ in range(10): + dt = get_utc_datetime() + current_minute = dt.strftime("%H-%M") + print(f"triggered new batch - {current_minute}") + await loadtest_fn() + await asyncio.sleep(10) + +asyncio.run(parent_fn()) + +``` + +### 2. Test it! + +Let's hit the router with 600 requests per minute. + +Copy this script 👇. Save it as `test_loadtest_router.py` AND run it with `python3 test_loadtest_router.py` + + +```python +from litellm import Router +import litellm +litellm.suppress_debug_info = True +litellm.set_verbose = False +import logging +logging.basicConfig(level=logging.CRITICAL) +import os, random, uuid, time, asyncio + +# Model list for OpenAI and Anthropic models +model_list = [ + { + "model_name": "fake-openai-endpoint", + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": "my-fake-key", + "api_base": "http://0.0.0.0:8080", + "rpm": 100 + }, + }, + { + "model_name": "fake-openai-endpoint", + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": "my-fake-key", + "api_base": "http://0.0.0.0:8081", + "rpm": 100 + }, + }, +] + +router_1 = Router(model_list=model_list, num_retries=0, enable_pre_call_checks=True, redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD")) +router_2 = Router(model_list=model_list, num_retries=0, enable_pre_call_checks=True, redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD")) + + + +async def router_completion_non_streaming(): + try: + client: Router = random.sample([router_1, router_2], 1)[0] # randomly pick b/w clients + # print(f"client={client}") + response = await client.acompletion( + model="fake-openai-endpoint", # [CHANGE THIS] (if you call it something else on your proxy) + messages=[{"role": "user", "content": f"This is a test: {uuid.uuid4()}"}], + ) + return response + except Exception as e: + # print(e) + return None + +async def loadtest_fn(): + start = time.time() + n = 600 # Number of concurrent tasks + tasks = [router_completion_non_streaming() for _ in range(n)] + chat_completions = await asyncio.gather(*tasks) + successful_completions = [c for c in chat_completions if c is not None] + print(n, time.time() - start, len(successful_completions)) + +def get_utc_datetime(): + import datetime as dt + from datetime import datetime + + if hasattr(dt, "UTC"): + return datetime.now(dt.UTC) # type: ignore + else: + return datetime.utcnow() # type: ignore + + +# Run the event loop to execute the async function +async def parent_fn(): + for _ in range(10): + dt = get_utc_datetime() + current_minute = dt.strftime("%H-%M") + print(f"triggered new batch - {current_minute}") + await loadtest_fn() + await asyncio.sleep(10) + +asyncio.run(parent_fn()) +``` +## Multi-Instance TPM/RPM Load Test (Proxy) Test if your defined tpm/rpm limits are respected across multiple instances. @@ -277,6 +461,8 @@ litellm --config /path/to/config.yaml --port 4001 Let's hit the proxy with 600 requests per minute. +Copy this script 👇. Save it as `test_loadtest_proxy.py` AND run it with `python3 test_loadtest_proxy.py` + ```python from openai import AsyncOpenAI, AsyncAzureOpenAI import random, uuid @@ -308,7 +494,7 @@ async def proxy_completion_non_streaming(): async def loadtest_fn(): start = time.time() - n = 500 # Number of concurrent tasks + n = 600 # Number of concurrent tasks tasks = [proxy_completion_non_streaming() for _ in range(n)] chat_completions = await asyncio.gather(*tasks) successful_completions = [c for c in chat_completions if c is not None] From 77f155d158339b623de8e5184e6c30d3aeaa5e2a Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 16:27:58 -0700 Subject: [PATCH 26/52] docs(load_test.md): cleanup docs --- docs/my-website/docs/load_test.md | 91 ++----------------------------- 1 file changed, 4 insertions(+), 87 deletions(-) diff --git a/docs/my-website/docs/load_test.md b/docs/my-website/docs/load_test.md index c1fac60fcb..754db4b8f1 100644 --- a/docs/my-website/docs/load_test.md +++ b/docs/my-website/docs/load_test.md @@ -228,91 +228,7 @@ If you don't want to call a real LLM API endpoint, you can setup a fake openai s ::: -### 1. Setup Router - -```python -from litellm import Router -import litellm -litellm.suppress_debug_info = True -litellm.set_verbose = False -import logging -logging.basicConfig(level=logging.CRITICAL) -import os, random, uuid, time, asyncio - -os.environ["REDIS_SSL"] = "True" - -# Model list for OpenAI and Anthropic models -model_list = [ - { - "model_name": "fake-openai-endpoint", - "litellm_params": { - "model": "gpt-3.5-turbo", - "api_key": "my-fake-key", - "api_base": "http://0.0.0.0:8080", - "rpm": 100 - }, - }, - { - "model_name": "fake-openai-endpoint", - "litellm_params": { - "model": "gpt-3.5-turbo", - "api_key": "my-fake-key", - "api_base": "http://0.0.0.0:8081", - "rpm": 100 - }, - }, -] - -router_1 = Router(model_list=model_list, num_retries=0, enable_pre_call_checks=True, redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD")) -router_2 = Router(model_list=model_list, num_retries=0, enable_pre_call_checks=True, redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD")) - - - -async def router_completion_non_streaming(): - try: - client: Router = random.sample([router_1, router_2], 1)[0] # randomly pick b/w clients - # print(f"client={client}") - response = await client.acompletion( - model="fake-openai-endpoint", # [CHANGE THIS] (if you call it something else on your proxy) - messages=[{"role": "user", "content": f"This is a test: {uuid.uuid4()}"}], - ) - return response - except Exception as e: - # print(e) - return None - -async def loadtest_fn(): - start = time.time() - n = 500 # Number of concurrent tasks - tasks = [router_completion_non_streaming() for _ in range(n)] - chat_completions = await asyncio.gather(*tasks) - successful_completions = [c for c in chat_completions if c is not None] - print(n, time.time() - start, len(successful_completions)) - -def get_utc_datetime(): - import datetime as dt - from datetime import datetime - - if hasattr(dt, "UTC"): - return datetime.now(dt.UTC) # type: ignore - else: - return datetime.utcnow() # type: ignore - - -# Run the event loop to execute the async function -async def parent_fn(): - for _ in range(10): - dt = get_utc_datetime() - current_minute = dt.strftime("%H-%M") - print(f"triggered new batch - {current_minute}") - await loadtest_fn() - await asyncio.sleep(10) - -asyncio.run(parent_fn()) - -``` - -### 2. Test it! +### Code Let's hit the router with 600 requests per minute. @@ -350,8 +266,8 @@ model_list = [ }, ] -router_1 = Router(model_list=model_list, num_retries=0, enable_pre_call_checks=True, redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD")) -router_2 = Router(model_list=model_list, num_retries=0, enable_pre_call_checks=True, redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD")) +router_1 = Router(model_list=model_list, num_retries=0, enable_pre_call_checks=True, routing_strategy="usage-based-routing-v2", redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD")) +router_2 = Router(model_list=model_list, num_retries=0, routing_strategy="usage-based-routing-v2", enable_pre_call_checks=True, redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD")) @@ -439,6 +355,7 @@ router_settings: redis_host: os.environ/REDIS_HOST ## 👈 IMPORTANT! Setup the proxy w/ redis redis_password: os.environ/REDIS_PASSWORD redis_port: os.environ/REDIS_PORT + routing_strategy: usage-based-routing-v2 ``` ### 2. Start proxy 2 instances From 2cf069befbdef8d91912be59b129c2f9779d950b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 16:33:21 -0700 Subject: [PATCH 27/52] fix(langfuse.py): don't set default trace_name if trace_id given --- litellm/integrations/langfuse.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index c2612feb80..063b675d42 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -265,15 +265,17 @@ class LangFuseLogger: tags = metadata_tags trace_name = metadata.get("trace_name", None) - if trace_name is None: + trace_id = metadata.get("trace_id", None) + if trace_name is None and trace_id is None: # just log `litellm-{call_type}` as the trace name + ## DO NOT SET TRACE_NAME if trace-id set. this can lead to overwriting of past traces. trace_name = f"litellm-{kwargs.get('call_type', 'completion')}" trace_params = { "name": trace_name, "input": input, "user_id": metadata.get("trace_user_id", user_id), - "id": metadata.get("trace_id", None), + "id": trace_id, "session_id": metadata.get("session_id", None), } From 853b70aba98f59433b7095ef71e07cd3df8bb060 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 16:39:01 -0700 Subject: [PATCH 28/52] fix(langfuse.py): support 'existing_trace_id' param allow user to call out a trace as pre-existing, this prevents creating a default trace name, and potentially overwriting past traces --- litellm/integrations/langfuse.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index 063b675d42..b1c0e4b097 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -266,7 +266,8 @@ class LangFuseLogger: trace_name = metadata.get("trace_name", None) trace_id = metadata.get("trace_id", None) - if trace_name is None and trace_id is None: + existing_trace_id = metadata.get("existing_trace_id", None) + if trace_name is None and existing_trace_id is None: # just log `litellm-{call_type}` as the trace name ## DO NOT SET TRACE_NAME if trace-id set. this can lead to overwriting of past traces. trace_name = f"litellm-{kwargs.get('call_type', 'completion')}" @@ -275,7 +276,7 @@ class LangFuseLogger: "name": trace_name, "input": input, "user_id": metadata.get("trace_user_id", user_id), - "id": trace_id, + "id": trace_id or existing_trace_id, "session_id": metadata.get("session_id", None), } From bd79e8b516bca8102a07f17ba6b702bf04a99d70 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 16:40:38 -0700 Subject: [PATCH 29/52] docs(langfuse_integration.md): add 'existing_trace_id' to langfuse docs --- docs/my-website/docs/observability/langfuse_integration.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/observability/langfuse_integration.md b/docs/my-website/docs/observability/langfuse_integration.md index fe210e6b71..bf62ee9bc8 100644 --- a/docs/my-website/docs/observability/langfuse_integration.md +++ b/docs/my-website/docs/observability/langfuse_integration.md @@ -121,10 +121,12 @@ response = completion( metadata={ "generation_name": "ishaan-test-generation", # set langfuse Generation Name "generation_id": "gen-id22", # set langfuse Generation ID - "trace_id": "trace-id22", # set langfuse Trace ID "trace_user_id": "user-id2", # set langfuse Trace User ID "session_id": "session-1", # set langfuse Session ID "tags": ["tag1", "tag2"] # set langfuse Tags + "trace_id": "trace-id22", # set langfuse Trace ID + ### OR ### + "existing_trace_id": "trace-id22", # if generation is continuation of past trace. This prevents default behaviour of setting a trace name }, ) From e7b4882e9726c1d28d18246aecbc3a6de7f62176 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 16:47:30 -0700 Subject: [PATCH 30/52] fix(router.py): fix high-traffic bug for usage-based-routing-v2 --- litellm/router.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index b66c29533e..46625b3c65 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2779,7 +2779,10 @@ class Router: self.cache.get_cache(key=model_id, local_only=True) or 0 ) ### get usage based cache ### - if isinstance(model_group_cache, dict): + if ( + isinstance(model_group_cache, dict) + and self.routing_strategy != "usage-based-routing-v2" + ): model_group_cache[model_id] = model_group_cache.get(model_id, 0) current_request = max( From 1e53c060646b0a34cc14f8eb52e1b199c37e6574 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 18:37:31 -0700 Subject: [PATCH 31/52] test(test_router_caching.py): remove unstable test test would fail due to timing issues --- litellm/tests/test_router_caching.py | 29 ---------------------------- 1 file changed, 29 deletions(-) diff --git a/litellm/tests/test_router_caching.py b/litellm/tests/test_router_caching.py index ce03498f92..ebace161c9 100644 --- a/litellm/tests/test_router_caching.py +++ b/litellm/tests/test_router_caching.py @@ -264,32 +264,3 @@ async def test_acompletion_caching_on_router_caching_groups(): except Exception as e: traceback.print_exc() pytest.fail(f"Error occurred: {e}") - -def test_rpm_limiting(): - try: - litellm.set_verbose = True - model_list = [ - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { - "model": "gpt-3.5-turbo", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - "tpm": 10000, - "rpm": 3, - }, - ] - - router = Router( - model_list = model_list, - routing_strategy = "usage-based-routing", - ) - failedCount = 0 - for i in range(10): - try: - response = router.completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": ""}]) - except ValueError as e: - failedCount += 1 - assert failedCount == 7 - except Exception as e: - pytest.fail(f"An exception occurred - {str(e)}") \ No newline at end of file From f0e48cdd53be4d8591b3e91ea5776b2d84163da7 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 18:48:04 -0700 Subject: [PATCH 32/52] fix(router.py): raise better exception when no deployments are available Fixes https://github.com/BerriAI/litellm/issues/3355 --- litellm/router.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index 46625b3c65..8ea1a124a4 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1450,7 +1450,9 @@ class Router: raise original_exception ### RETRY #### check if it should retry + back-off if required - if "No models available" in str(e): + if "No models available" in str( + e + ) or RouterErrors.no_deployments_available.value in str(e): timeout = litellm._calculate_retry_after( remaining_retries=num_retries, max_retries=num_retries, @@ -2948,6 +2950,11 @@ class Router: model=model, healthy_deployments=healthy_deployments, messages=messages ) + if len(healthy_deployments) == 0: + raise ValueError( + f"{RouterErrors.no_deployments_available.value}, passed model={model}" + ) + if ( self.routing_strategy == "usage-based-routing-v2" and self.lowesttpm_logger_v2 is not None From b46db8b89135a6b17f5b0797fdf20ec34735f8b0 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 19:21:19 -0700 Subject: [PATCH 33/52] feat(utils.py): json logs for raw request sent by litellm make it easier to view verbose logs in datadog --- docs/my-website/docs/debugging/local_debugging.md | 8 ++++++++ litellm/__init__.py | 2 +- litellm/_logging.py | 2 +- litellm/integrations/langsmith.py | 4 ---- litellm/utils.py | 9 ++++++++- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/docs/my-website/docs/debugging/local_debugging.md b/docs/my-website/docs/debugging/local_debugging.md index 87faef73e4..a9409bfab0 100644 --- a/docs/my-website/docs/debugging/local_debugging.md +++ b/docs/my-website/docs/debugging/local_debugging.md @@ -23,6 +23,14 @@ response = completion(model="gpt-3.5-turbo", messages=messages) response = completion("command-nightly", messages) ``` +## JSON Logs + +If you need to store the logs as JSON, just set the `litellm.json_logs = True`. + +We currently just log the raw POST request from litellm as a JSON - [**See Code**]. + +[Share feedback here](https://github.com/BerriAI/litellm/issues) + ## Logger Function But sometimes all you care about is seeing exactly what's getting sent to your api call and what's being returned - e.g. if the api call is failing, why is that happening? what are the exact params being set? diff --git a/litellm/__init__.py b/litellm/__init__.py index 49287d12fb..a3d61bce16 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -2,7 +2,7 @@ import threading, requests, os from typing import Callable, List, Optional, Dict, Union, Any, Literal from litellm.caching import Cache -from litellm._logging import set_verbose, _turn_on_debug, verbose_logger +from litellm._logging import set_verbose, _turn_on_debug, verbose_logger, json_logs from litellm.proxy._types import ( KeyManagementSystem, KeyManagementSettings, diff --git a/litellm/_logging.py b/litellm/_logging.py index 4f7e464468..f31ee41f8b 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -1,7 +1,7 @@ import logging set_verbose = False - +json_logs = False # Create a handler for the logger (you may need to adapt this based on your needs) handler = logging.StreamHandler() handler.setLevel(logging.DEBUG) diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 8e37a1ec1a..415f3d2d20 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -73,10 +73,6 @@ class LangsmithLogger: elif type(value) != dict and is_serializable(value=value): new_kwargs[key] = value - print(f"type of response: {type(response_obj)}") - for k, v in new_kwargs.items(): - print(f"key={k}, type of arg: {type(v)}, value={v}") - if isinstance(response_obj, BaseModel): try: response_obj = response_obj.model_dump() diff --git a/litellm/utils.py b/litellm/utils.py index 302d18e99c..e5f7f9d11a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1202,7 +1202,14 @@ class Logging: if verbose_logger.level == 0: # this means verbose logger was not switched on - user is in litellm.set_verbose=True print_verbose(f"\033[92m{curl_command}\033[0m\n") - verbose_logger.info(f"\033[92m{curl_command}\033[0m\n") + + if litellm.json_logs: + verbose_logger.info( + "POST Request Sent from LiteLLM", + extra={"api_base": {api_base}, **masked_headers}, + ) + else: + verbose_logger.info(f"\033[92m{curl_command}\033[0m\n") if self.logger_fn and callable(self.logger_fn): try: self.logger_fn( From 0c99ae945134167d498e85fc003dea575be86f16 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Apr 2024 21:20:26 -0700 Subject: [PATCH 34/52] docs - fix kub.yaml config on docs --- docs/my-website/docs/proxy/deploy.md | 77 ++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 20 deletions(-) diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index 992350211e..815252429d 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -272,26 +272,63 @@ Your OpenAI proxy server is now running on `http://0.0.0.0:4000`. #### Step 1. Create deployment.yaml ```yaml - apiVersion: apps/v1 - kind: Deployment - metadata: - name: litellm-deployment - spec: - replicas: 1 - selector: - matchLabels: - app: litellm - template: - metadata: - labels: - app: litellm - spec: - containers: - - name: litellm-container - image: ghcr.io/berriai/litellm-database:main-latest - env: - - name: DATABASE_URL - value: postgresql://:@:/ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: litellm-deployment +spec: + replicas: 3 + selector: + matchLabels: + app: litellm + template: + metadata: + labels: + app: litellm + spec: + containers: + - name: litellm-container + image: ghcr.io/berriai/litellm:main-latest + imagePullPolicy: Always + env: + - name: AZURE_API_KEY + value: "d6******" + - name: AZURE_API_BASE + value: "https://ope******" + - name: LITELLM_MASTER_KEY + value: "sk-1234" + - name: DATABASE_URL + value: "po**********" + args: + - "--config" + - "/app/proxy_config.yaml" # Update the path to mount the config file + volumeMounts: # Define volume mount for proxy_config.yaml + - name: config-volume + mountPath: /app + readOnly: true + livenessProbe: + httpGet: + path: /health/liveliness + port: 4000 + initialDelaySeconds: 120 + periodSeconds: 15 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 10 + readinessProbe: + httpGet: + path: /health/readiness + port: 4000 + initialDelaySeconds: 120 + periodSeconds: 15 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 10 + volumes: # Define volume to mount proxy_config.yaml + - name: config-volume + configMap: + name: litellm-config + ``` ```bash From 0cad58f5c6e93c2afd5863a944353980ff51de77 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Apr 2024 21:26:15 -0700 Subject: [PATCH 35/52] docs logging to langfuse on proxy --- docs/my-website/docs/proxy/logging.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 1a5a7f0f07..60a5d060a5 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -401,7 +401,7 @@ litellm_settings: Start the LiteLLM Proxy and make a test request to verify the logs reached your callback API ## Logging Proxy Input/Output - Langfuse -We will use the `--config` to set `litellm.success_callback = ["langfuse"]` this will log all successfull LLM calls to langfuse +We will use the `--config` to set `litellm.success_callback = ["langfuse"]` this will log all successfull LLM calls to langfuse. Make sure to set `LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY` in your environment **Step 1** Install langfuse @@ -419,7 +419,13 @@ litellm_settings: success_callback: ["langfuse"] ``` -**Step 3**: Start the proxy, make a test request +**Step 3**: Set required env variables for logging to langfuse +```shell +export LANGFUSE_PUBLIC_KEY="pk_kk" +export LANGFUSE_SECRET_KEY="sk_ss +``` + +**Step 4**: Start the proxy, make a test request Start proxy ```shell From b1e888edad2f89299d5212346304e852f7c8945e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Apr 2024 21:26:27 -0700 Subject: [PATCH 36/52] docs example logging to langfuse --- docs/my-website/docs/proxy/configs.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index 59102c24d2..7a7b430953 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -62,6 +62,7 @@ model_list: litellm_settings: # module level litellm settings - https://github.com/BerriAI/litellm/blob/main/litellm/__init__.py drop_params: True + success_callback: ["langfuse"] # OPTIONAL - if you want to start sending LLM Logs to Langfuse. Make sure to set `LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY` in your env general_settings: master_key: sk-1234 # [OPTIONAL] Only use this if you to require all calls to contain this key (Authorization: Bearer sk-1234) From 81df36b29899c3cc53b7a52a68f6018764c0e6de Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Apr 2024 21:33:03 -0700 Subject: [PATCH 37/52] docs - slack alerting --- docs/my-website/docs/proxy/configs.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index 7a7b430953..5eeb05f369 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -66,6 +66,7 @@ litellm_settings: # module level litellm settings - https://github.com/BerriAI/l general_settings: master_key: sk-1234 # [OPTIONAL] Only use this if you to require all calls to contain this key (Authorization: Bearer sk-1234) + alerting: ["slack"] # [OPTIONAL] If you want Slack Alerts for Hanging LLM requests, Slow llm responses, Budget Alerts. Make sure to set `SLACK_WEBHOOK_URL` in your env ``` :::info From 020b175ef4fb1637e08a326b94ab407c712b8f78 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 21:34:05 -0700 Subject: [PATCH 38/52] fix(lowest_tpm_rpm_v2.py): skip if item_tpm is None --- litellm/router_strategy/lowest_tpm_rpm_v2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py index 418bb43169..4bcf1eec12 100644 --- a/litellm/router_strategy/lowest_tpm_rpm_v2.py +++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py @@ -343,6 +343,8 @@ class LowestTPMLoggingHandler_v2(CustomLogger): _deployment = m if _deployment is None: continue # skip to next one + elif item_tpm is None: + continue # skip if unhealthy deployment _deployment_tpm = None if _deployment_tpm is None: From 1cd24d890669a0a3e4f08ccbee2369abb8074d4b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 30 Apr 2024 07:20:50 -0700 Subject: [PATCH 39/52] =?UTF-8?q?bump:=20version=201.35.32=20=E2=86=92=201?= =?UTF-8?q?.35.33?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 837717a97f..c14a5f4593 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.35.32" +version = "1.35.33" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -80,7 +80,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.35.32" +version = "1.35.33" version_files = [ "pyproject.toml:^version" ] From d717fa25887ac901c363d2a023149720aa26f172 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 30 Apr 2024 08:48:55 -0700 Subject: [PATCH 40/52] test(test_tpm_rpm_routing_v2.py): fix test - bump number of iteration s --- litellm/tests/test_tpm_rpm_routing_v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/tests/test_tpm_rpm_routing_v2.py b/litellm/tests/test_tpm_rpm_routing_v2.py index 78620a6e55..fe3b74bc1b 100644 --- a/litellm/tests/test_tpm_rpm_routing_v2.py +++ b/litellm/tests/test_tpm_rpm_routing_v2.py @@ -321,7 +321,7 @@ async def test_multiple_potential_deployments(sync_mode): ) # type: ignore model_ids = set() - for _ in range(5): + for _ in range(1000): if sync_mode: deployment = router.get_available_deployment( model="azure-model", From 00d1440d0dfbe03940c98b8b5562b495931e626f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 30 Apr 2024 08:55:40 -0700 Subject: [PATCH 41/52] test(test_image_generation.py): change img model for test - bedrock EOL --- litellm/tests/test_image_generation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/tests/test_image_generation.py b/litellm/tests/test_image_generation.py index 964b005829..31e7d3d903 100644 --- a/litellm/tests/test_image_generation.py +++ b/litellm/tests/test_image_generation.py @@ -136,7 +136,7 @@ def test_image_generation_bedrock(): litellm.set_verbose = True response = litellm.image_generation( prompt="A cute baby sea otter", - model="bedrock/stability.stable-diffusion-xl-v0", + model="bedrock/stability.stable-diffusion-xl-v1", aws_region_name="us-east-1", ) print(f"response: {response}") @@ -156,7 +156,7 @@ async def test_aimage_generation_bedrock_with_optional_params(): try: response = await litellm.aimage_generation( prompt="A cute baby sea otter", - model="bedrock/stability.stable-diffusion-xl-v0", + model="bedrock/stability.stable-diffusion-xl-v1", size="128x128", ) print(f"response: {response}") From 398d503590e2dc61fa1e0b31fe6cd6ed9500ee8a Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 30 Apr 2024 11:36:29 -0700 Subject: [PATCH 42/52] build(model_prices_and_context_window.json): add bedrock llama3 pricing --- ...model_prices_and_context_window_backup.json | 18 ++++++++++++++++++ model_prices_and_context_window.json | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b695d80866..4b15b8e323 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2496,6 +2496,24 @@ "litellm_provider": "bedrock", "mode": "chat" }, + "meta.llama3-8b-instruct-v1:0": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0000004, + "output_cost_per_token": 0.0000006, + "litellm_provider": "bedrock", + "mode": "chat" + }, + "meta.llama3-70b-instruct-v1:0": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00000265, + "output_cost_per_token": 0.0000035, + "litellm_provider": "bedrock", + "mode": "chat" + }, "512-x-512/50-steps/stability.stable-diffusion-xl-v0": { "max_tokens": 77, "max_input_tokens": 77, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b695d80866..4b15b8e323 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -2496,6 +2496,24 @@ "litellm_provider": "bedrock", "mode": "chat" }, + "meta.llama3-8b-instruct-v1:0": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0000004, + "output_cost_per_token": 0.0000006, + "litellm_provider": "bedrock", + "mode": "chat" + }, + "meta.llama3-70b-instruct-v1:0": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00000265, + "output_cost_per_token": 0.0000035, + "litellm_provider": "bedrock", + "mode": "chat" + }, "512-x-512/50-steps/stability.stable-diffusion-xl-v0": { "max_tokens": 77, "max_input_tokens": 77, From d6f7fa7f4e4789d524d700f24ceb6324e7febda7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 11:42:17 -0700 Subject: [PATCH 43/52] v0 prisma schema --- schema.prisma | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/schema.prisma b/schema.prisma index 5ec73c9dc1..cf979fe865 100644 --- a/schema.prisma +++ b/schema.prisma @@ -183,6 +183,16 @@ model LiteLLM_SpendLogs { end_user String? } +// View spend, model, api_key per request +model LiteLLM_ErrorLogs { + request_id String @id @default(uuid()) + model_name String @default("") // public model_name / model_group + model_id String @default("") // ID of model in ProxyModelTable + request_kwargs Json @default("{}") + exceptionType String @default("") + exceptionString String @default("") +} + // Beta - allow team members to request access to a model model LiteLLM_UserNotifications { request_id String @id From 285a3733a968bbf0c94e76d0ff5d61a80e7429af Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 30 Apr 2024 12:14:29 -0700 Subject: [PATCH 44/52] test(test_image_generation.py): fix test --- litellm/tests/test_image_generation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/tests/test_image_generation.py b/litellm/tests/test_image_generation.py index 31e7d3d903..82068a1156 100644 --- a/litellm/tests/test_image_generation.py +++ b/litellm/tests/test_image_generation.py @@ -137,7 +137,7 @@ def test_image_generation_bedrock(): response = litellm.image_generation( prompt="A cute baby sea otter", model="bedrock/stability.stable-diffusion-xl-v1", - aws_region_name="us-east-1", + aws_region_name="us-west-2", ) print(f"response: {response}") except litellm.RateLimitError as e: @@ -157,7 +157,7 @@ async def test_aimage_generation_bedrock_with_optional_params(): response = await litellm.aimage_generation( prompt="A cute baby sea otter", model="bedrock/stability.stable-diffusion-xl-v1", - size="128x128", + size="256x256", ) print(f"response: {response}") except litellm.RateLimitError as e: From ac1cabe96345ba59e63ac939e497a48ac56c0ece Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 12:16:03 -0700 Subject: [PATCH 45/52] add LiteLLM_ErrorLogs to types --- litellm/proxy/_types.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index fbe914a26f..5ec19033ac 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -912,5 +912,15 @@ class LiteLLM_SpendLogs(LiteLLMBase): request_tags: Optional[Json] = None +class LiteLLM_ErrorLogs(LiteLLMBase): + request_id: Optional[str] = str(uuid.uuid4()) + model_name: Optional[str] = "" + model_id: Optional[str] = "" + request_kwargs: Optional[dict] = {} + exception_type: Optional[str] = "" + status_code: Optional[str] = "" + exception_string: Optional[str] = "" + + class LiteLLM_SpendLogs_ResponseObject(LiteLLMBase): response: Optional[List[Union[LiteLLM_SpendLogs, Any]]] = None From c7f979e0fe3c11f5595dca0c18ad7395b728bc3c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 12:31:19 -0700 Subject: [PATCH 46/52] fix schema error logs --- schema.prisma | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/schema.prisma b/schema.prisma index cf979fe865..7f81ed16a0 100644 --- a/schema.prisma +++ b/schema.prisma @@ -189,8 +189,9 @@ model LiteLLM_ErrorLogs { model_name String @default("") // public model_name / model_group model_id String @default("") // ID of model in ProxyModelTable request_kwargs Json @default("{}") - exceptionType String @default("") - exceptionString String @default("") + exception_type String @default("") + exception_string String @default("") + status_code String @default("") } // Beta - allow team members to request access to a model From 22725bd44df06d97c35d0263fa773cb9eab069ce Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 12:31:33 -0700 Subject: [PATCH 47/52] fix types for errorLog --- litellm/proxy/_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 5ec19033ac..9c13b18540 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -916,7 +916,7 @@ class LiteLLM_ErrorLogs(LiteLLMBase): request_id: Optional[str] = str(uuid.uuid4()) model_name: Optional[str] = "" model_id: Optional[str] = "" - request_kwargs: Optional[dict] = {} + request_kwargs: Optional[Json] = {} exception_type: Optional[str] = "" status_code: Optional[str] = "" exception_string: Optional[str] = "" From 06804bc70adf1ac723b776f82ee7830434d45d17 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 12:48:17 -0700 Subject: [PATCH 48/52] fix - working exception writing --- litellm/proxy/proxy_server.py | 65 +++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 29f3c41dba..81ea8961a7 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1217,6 +1217,59 @@ def cost_tracking(): litellm.success_callback.append(_PROXY_track_cost_callback) # type: ignore +async def _PROXY_failure_handler( + kwargs, # kwargs to completion + completion_response: litellm.ModelResponse, # response from completion + start_time=None, + end_time=None, # start/end time for completion +): + global prisma_client + if prisma_client is not None: + verbose_proxy_logger.debug( + "inside _PROXY_failure_handler kwargs=", extra=kwargs + ) + + _exception = kwargs.get("exception") + traceback = kwargs.get("traceback") + _exception_type = _exception.__class__.__name__ + + _model = kwargs.get("model", None) + _status_code = _exception.status_code + + _litellm_params = kwargs.get("litellm_params", {}) or {} + _metadata = _litellm_params.get("metadata", {}) or {} + _model_id = _metadata.get("model_info", {}).get("id", None) + verbose_proxy_logger.debug( + "\nexception_type", + _exception_type, + "\nrequest_model", + _model, + "\nmodel_id", + _model_id, + "\nexception", + _exception, + "\ntraceback", + traceback, + ) + error_log = LiteLLM_ErrorLogs( + model_name=_model, + model_id=_model_id, + exception_type=_exception_type, + status_code=_status_code, + exception_string=str(_exception), + ) + + # helper function to convert to dict on pydantic v2 & v1 + error_log_dict = _get_pydantic_json_dict(error_log) + error_log_dict["request_kwargs"] = json.dumps(error_log_dict["request_kwargs"]) + + await prisma_client.db.litellm_errorlogs.create( + data=error_log_dict # type: ignore + ) + + pass + + async def _PROXY_track_cost_callback( kwargs, # kwargs to completion completion_response: litellm.ModelResponse, # response from completion @@ -1302,6 +1355,15 @@ async def _PROXY_track_cost_callback( verbose_proxy_logger.debug("error in tracking cost callback - %s", e) +def error_tracking(): + global prisma_client, custom_db_client + if prisma_client is not None or custom_db_client is not None: + if isinstance(litellm.failure_callback, list): + verbose_proxy_logger.debug("setting litellm failure callback to track cost") + if (_PROXY_failure_handler) not in litellm.failure_callback: # type: ignore + litellm.failure_callback.append(_PROXY_failure_handler) # type: ignore + + def _set_spend_logs_payload( payload: dict, prisma_client: PrismaClient, spend_logs_url: Optional[str] = None ): @@ -3194,6 +3256,9 @@ async def startup_event(): ## COST TRACKING ## cost_tracking() + ## Error Tracking ## + error_tracking() + db_writer_client = HTTPHandler() proxy_logging_obj._init_litellm_callbacks() # INITIALIZE LITELLM CALLBACKS ON SERVER STARTUP <- do this to catch any logging errors on startup, not when calls are being made From ee2a2ce559bdb22d4480e42be3988a36157c9e5d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 13:02:42 -0700 Subject: [PATCH 49/52] fix - log api_base in errors --- litellm/proxy/_types.py | 1 + litellm/proxy/proxy_server.py | 5 +++++ schema.prisma | 1 + 3 files changed, 7 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 9c13b18540..9548025d51 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -914,6 +914,7 @@ class LiteLLM_SpendLogs(LiteLLMBase): class LiteLLM_ErrorLogs(LiteLLMBase): request_id: Optional[str] = str(uuid.uuid4()) + api_base: Optional[str] = "" model_name: Optional[str] = "" model_id: Optional[str] = "" request_kwargs: Optional[Json] = {} diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 81ea8961a7..c9d94288e1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1239,6 +1239,9 @@ async def _PROXY_failure_handler( _litellm_params = kwargs.get("litellm_params", {}) or {} _metadata = _litellm_params.get("metadata", {}) or {} _model_id = _metadata.get("model_info", {}).get("id", None) + + api_base = litellm.get_api_base(model=_model, optional_params=_litellm_params) + verbose_proxy_logger.debug( "\nexception_type", _exception_type, @@ -1252,8 +1255,10 @@ async def _PROXY_failure_handler( traceback, ) error_log = LiteLLM_ErrorLogs( + request_id=str(uuid.uuid4()), model_name=_model, model_id=_model_id, + api_base=api_base, exception_type=_exception_type, status_code=_status_code, exception_string=str(_exception), diff --git a/schema.prisma b/schema.prisma index 7f81ed16a0..e9acfe1882 100644 --- a/schema.prisma +++ b/schema.prisma @@ -186,6 +186,7 @@ model LiteLLM_SpendLogs { // View spend, model, api_key per request model LiteLLM_ErrorLogs { request_id String @id @default(uuid()) + api_base String @default("") model_name String @default("") // public model_name / model_group model_id String @default("") // ID of model in ProxyModelTable request_kwargs Json @default("{}") From ad5fddef156804bafcced6a9e19c97d3d84faf57 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 13:11:09 -0700 Subject: [PATCH 50/52] fix log model_group --- litellm/proxy/_types.py | 2 +- litellm/proxy/proxy_server.py | 7 ++++--- schema.prisma | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 9548025d51..75dfdb4c4d 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -915,7 +915,7 @@ class LiteLLM_SpendLogs(LiteLLMBase): class LiteLLM_ErrorLogs(LiteLLMBase): request_id: Optional[str] = str(uuid.uuid4()) api_base: Optional[str] = "" - model_name: Optional[str] = "" + model_group: Optional[str] = "" model_id: Optional[str] = "" request_kwargs: Optional[Json] = {} exception_type: Optional[str] = "" diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c9d94288e1..f9e7756bde 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1232,13 +1232,14 @@ async def _PROXY_failure_handler( _exception = kwargs.get("exception") traceback = kwargs.get("traceback") _exception_type = _exception.__class__.__name__ - _model = kwargs.get("model", None) + _status_code = _exception.status_code _litellm_params = kwargs.get("litellm_params", {}) or {} _metadata = _litellm_params.get("metadata", {}) or {} - _model_id = _metadata.get("model_info", {}).get("id", None) + _model_id = _metadata.get("model_info", {}).get("id", "") + _model_group = _metadata.get("model_group", "") api_base = litellm.get_api_base(model=_model, optional_params=_litellm_params) @@ -1256,7 +1257,7 @@ async def _PROXY_failure_handler( ) error_log = LiteLLM_ErrorLogs( request_id=str(uuid.uuid4()), - model_name=_model, + model_group=_model_group, model_id=_model_id, api_base=api_base, exception_type=_exception_type, diff --git a/schema.prisma b/schema.prisma index e9acfe1882..cd02199835 100644 --- a/schema.prisma +++ b/schema.prisma @@ -187,7 +187,7 @@ model LiteLLM_SpendLogs { model LiteLLM_ErrorLogs { request_id String @id @default(uuid()) api_base String @default("") - model_name String @default("") // public model_name / model_group + model_group String @default("") // public model_name / model_group model_id String @default("") // ID of model in ProxyModelTable request_kwargs Json @default("{}") exception_type String @default("") From 3aad034a8b0c52275944ab3ab0276059170716b2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 13:28:26 -0700 Subject: [PATCH 51/52] feat log request kwargs in error logs --- litellm/proxy/_types.py | 2 +- litellm/proxy/proxy_server.py | 32 ++++++++++++++++---------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 75dfdb4c4d..2b3a72250f 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -917,7 +917,7 @@ class LiteLLM_ErrorLogs(LiteLLMBase): api_base: Optional[str] = "" model_group: Optional[str] = "" model_id: Optional[str] = "" - request_kwargs: Optional[Json] = {} + request_kwargs: Optional[dict] = {} exception_type: Optional[str] = "" status_code: Optional[str] = "" exception_string: Optional[str] = "" diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f9e7756bde..530438b94a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1230,39 +1230,39 @@ async def _PROXY_failure_handler( ) _exception = kwargs.get("exception") - traceback = kwargs.get("traceback") _exception_type = _exception.__class__.__name__ _model = kwargs.get("model", None) - _status_code = _exception.status_code + _optional_params = kwargs.get("optional_params", {}) + _optional_params = copy.deepcopy(_optional_params) + + for k, v in _optional_params.items(): + v = str(v) + v = v[:100] + + _status_code = "500" + try: + _status_code = str(_exception.status_code) + except: + # Don't let this fail logging the exception to the dB + pass _litellm_params = kwargs.get("litellm_params", {}) or {} _metadata = _litellm_params.get("metadata", {}) or {} _model_id = _metadata.get("model_info", {}).get("id", "") _model_group = _metadata.get("model_group", "") - api_base = litellm.get_api_base(model=_model, optional_params=_litellm_params) + _exception_string = str(_exception)[:500] - verbose_proxy_logger.debug( - "\nexception_type", - _exception_type, - "\nrequest_model", - _model, - "\nmodel_id", - _model_id, - "\nexception", - _exception, - "\ntraceback", - traceback, - ) error_log = LiteLLM_ErrorLogs( request_id=str(uuid.uuid4()), model_group=_model_group, model_id=_model_id, + request_kwargs=_optional_params, api_base=api_base, exception_type=_exception_type, status_code=_status_code, - exception_string=str(_exception), + exception_string=_exception_string, ) # helper function to convert to dict on pydantic v2 & v1 From 4b8fda4ac408825cc0c80e1f81212c774aa6af87 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 13:34:14 -0700 Subject: [PATCH 52/52] log startTime and EndTime for exceptions --- litellm/proxy/_types.py | 2 ++ litellm/proxy/proxy_server.py | 2 ++ schema.prisma | 2 ++ 3 files changed, 6 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 2b3a72250f..c910664f15 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -921,6 +921,8 @@ class LiteLLM_ErrorLogs(LiteLLMBase): exception_type: Optional[str] = "" status_code: Optional[str] = "" exception_string: Optional[str] = "" + startTime: Union[str, datetime, None] + endTime: Union[str, datetime, None] class LiteLLM_SpendLogs_ResponseObject(LiteLLMBase): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 530438b94a..3a7821d272 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1263,6 +1263,8 @@ async def _PROXY_failure_handler( exception_type=_exception_type, status_code=_status_code, exception_string=_exception_string, + startTime=kwargs.get("start_time"), + endTime=kwargs.get("end_time"), ) # helper function to convert to dict on pydantic v2 & v1 diff --git a/schema.prisma b/schema.prisma index cd02199835..b362a0ec02 100644 --- a/schema.prisma +++ b/schema.prisma @@ -186,6 +186,8 @@ model LiteLLM_SpendLogs { // View spend, model, api_key per request model LiteLLM_ErrorLogs { request_id String @id @default(uuid()) + startTime DateTime // Assuming start_time is a DateTime field + endTime DateTime // Assuming end_time is a DateTime field api_base String @default("") model_group String @default("") // public model_name / model_group model_id String @default("") // ID of model in ProxyModelTable