From 1dafb1b3b7a4c5d070efb4d3f55c86787d139183 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 8 Jun 2024 14:32:43 -0700 Subject: [PATCH 01/29] fix(utils.py): improved predibase exception mapping adds unit testing + better coverage for predibase errors --- litellm/exceptions.py | 62 ++++++++++++++--- litellm/integrations/test_httpx.py | 0 litellm/llms/predibase.py | 48 +++++++------ litellm/main.py | 14 ++-- litellm/proxy/_experimental/out/404.html | 1 - .../proxy/_experimental/out/model_hub.html | 1 - .../proxy/_experimental/out/onboarding.html | 1 - litellm/proxy/_super_secret_config.yaml | 22 ++++-- litellm/tests/test_exceptions.py | 46 +++++++++++++ litellm/utils.py | 69 +++++++++++++++++++ ruff.toml | 2 +- 11 files changed, 220 insertions(+), 46 deletions(-) create mode 100644 litellm/integrations/test_httpx.py delete mode 100644 litellm/proxy/_experimental/out/404.html delete mode 100644 litellm/proxy/_experimental/out/model_hub.html delete mode 100644 litellm/proxy/_experimental/out/onboarding.html diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 484e843b6d..886b5889da 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -20,7 +20,7 @@ class AuthenticationError(openai.AuthenticationError): # type: ignore message, llm_provider, model, - response: httpx.Response, + response: Optional[httpx.Response] = None, litellm_debug_info: Optional[str] = None, max_retries: Optional[int] = None, num_retries: Optional[int] = None, @@ -32,8 +32,14 @@ class AuthenticationError(openai.AuthenticationError): # type: ignore self.litellm_debug_info = litellm_debug_info self.max_retries = max_retries self.num_retries = num_retries + self.response = response or httpx.Response( + status_code=self.status_code, + request=httpx.Request( + method="GET", url="https://litellm.ai" + ), # mock request object + ) super().__init__( - self.message, response=response, body=None + self.message, response=self.response, body=None ) # Call the base class constructor with the parameters it needs def __str__(self): @@ -60,7 +66,7 @@ class NotFoundError(openai.NotFoundError): # type: ignore message, model, llm_provider, - response: httpx.Response, + response: Optional[httpx.Response] = None, litellm_debug_info: Optional[str] = None, max_retries: Optional[int] = None, num_retries: Optional[int] = None, @@ -72,8 +78,14 @@ class NotFoundError(openai.NotFoundError): # type: ignore self.litellm_debug_info = litellm_debug_info self.max_retries = max_retries self.num_retries = num_retries + self.response = response or httpx.Response( + status_code=self.status_code, + request=httpx.Request( + method="GET", url="https://litellm.ai" + ), # mock request object + ) super().__init__( - self.message, response=response, body=None + self.message, response=self.response, body=None ) # Call the base class constructor with the parameters it needs def __str__(self): @@ -262,7 +274,7 @@ class RateLimitError(openai.RateLimitError): # type: ignore message, llm_provider, model, - response: httpx.Response, + response: Optional[httpx.Response] = None, litellm_debug_info: Optional[str] = None, max_retries: Optional[int] = None, num_retries: Optional[int] = None, @@ -274,8 +286,18 @@ class RateLimitError(openai.RateLimitError): # type: ignore self.litellm_debug_info = litellm_debug_info self.max_retries = max_retries self.num_retries = num_retries + if response is None: + self.response = httpx.Response( + status_code=429, + request=httpx.Request( + method="POST", + url=" https://cloud.google.com/vertex-ai/", + ), + ) + else: + self.response = response super().__init__( - self.message, response=response, body=None + self.message, response=self.response, body=None ) # Call the base class constructor with the parameters it needs def __str__(self): @@ -421,7 +443,7 @@ class ServiceUnavailableError(openai.APIStatusError): # type: ignore message, llm_provider, model, - response: httpx.Response, + response: Optional[httpx.Response] = None, litellm_debug_info: Optional[str] = None, max_retries: Optional[int] = None, num_retries: Optional[int] = None, @@ -433,8 +455,18 @@ class ServiceUnavailableError(openai.APIStatusError): # type: ignore self.litellm_debug_info = litellm_debug_info self.max_retries = max_retries self.num_retries = num_retries + if response is None: + self.response = httpx.Response( + status_code=self.status_code, + request=httpx.Request( + method="POST", + url=" https://cloud.google.com/vertex-ai/", + ), + ) + else: + self.response = response super().__init__( - self.message, response=response, body=None + self.message, response=self.response, body=None ) # Call the base class constructor with the parameters it needs def __str__(self): @@ -460,7 +492,7 @@ class InternalServerError(openai.InternalServerError): # type: ignore message, llm_provider, model, - response: httpx.Response, + response: Optional[httpx.Response] = None, litellm_debug_info: Optional[str] = None, max_retries: Optional[int] = None, num_retries: Optional[int] = None, @@ -472,8 +504,18 @@ class InternalServerError(openai.InternalServerError): # type: ignore self.litellm_debug_info = litellm_debug_info self.max_retries = max_retries self.num_retries = num_retries + if response is None: + self.response = httpx.Response( + status_code=self.status_code, + request=httpx.Request( + method="POST", + url=" https://cloud.google.com/vertex-ai/", + ), + ) + else: + self.response = response super().__init__( - self.message, response=response, body=None + self.message, response=self.response, body=None ) # Call the base class constructor with the parameters it needs def __str__(self): diff --git a/litellm/integrations/test_httpx.py b/litellm/integrations/test_httpx.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/litellm/llms/predibase.py b/litellm/llms/predibase.py index a3245cdac1..66c28acee4 100644 --- a/litellm/llms/predibase.py +++ b/litellm/llms/predibase.py @@ -3,6 +3,7 @@ from functools import partial import os, types +import traceback import json from enum import Enum import requests, copy # type: ignore @@ -242,12 +243,12 @@ class PredibaseChatCompletion(BaseLLM): "details" in completion_response and "tokens" in completion_response["details"] ): - model_response.choices[0].finish_reason = completion_response[ - "details" - ]["finish_reason"] + model_response.choices[0].finish_reason = map_finish_reason( + completion_response["details"]["finish_reason"] + ) sum_logprob = 0 for token in completion_response["details"]["tokens"]: - if token["logprob"] != None: + if token["logprob"] is not None: sum_logprob += token["logprob"] model_response["choices"][0][ "message" @@ -265,7 +266,7 @@ class PredibaseChatCompletion(BaseLLM): ): sum_logprob = 0 for token in item["tokens"]: - if token["logprob"] != None: + if token["logprob"] is not None: sum_logprob += token["logprob"] if len(item["generated_text"]) > 0: message_obj = Message( @@ -275,7 +276,7 @@ class PredibaseChatCompletion(BaseLLM): else: message_obj = Message(content=None) choice_obj = Choices( - finish_reason=item["finish_reason"], + finish_reason=map_finish_reason(item["finish_reason"]), index=idx + 1, message=message_obj, ) @@ -285,10 +286,8 @@ class PredibaseChatCompletion(BaseLLM): ## CALCULATING USAGE prompt_tokens = 0 try: - prompt_tokens = len( - encoding.encode(model_response["choices"][0]["message"]["content"]) - ) ##[TODO] use a model-specific tokenizer here - except: + prompt_tokens = litellm.token_counter(messages=messages) + except Exception: # this should remain non blocking we should not block a response returning if calculating usage fails pass output_text = model_response["choices"][0]["message"].get("content", "") @@ -331,6 +330,7 @@ class PredibaseChatCompletion(BaseLLM): logging_obj, optional_params: dict, tenant_id: str, + timeout: Union[float, httpx.Timeout], acompletion=None, litellm_params=None, logger_fn=None, @@ -340,6 +340,7 @@ class PredibaseChatCompletion(BaseLLM): completion_url = "" input_text = "" base_url = "https://serving.app.predibase.com" + if "https" in model: completion_url = model elif api_base: @@ -349,7 +350,7 @@ class PredibaseChatCompletion(BaseLLM): completion_url = f"{base_url}/{tenant_id}/deployments/v2/llms/{model}" - if optional_params.get("stream", False) == True: + if optional_params.get("stream", False) is True: completion_url += "/generate_stream" else: completion_url += "/generate" @@ -393,9 +394,9 @@ class PredibaseChatCompletion(BaseLLM): }, ) ## COMPLETION CALL - if acompletion == True: + if acompletion is True: ### ASYNC STREAMING - if stream == True: + if stream is True: return self.async_streaming( model=model, messages=messages, @@ -410,6 +411,7 @@ class PredibaseChatCompletion(BaseLLM): litellm_params=litellm_params, logger_fn=logger_fn, headers=headers, + timeout=timeout, ) # type: ignore else: ### ASYNC COMPLETION @@ -428,10 +430,11 @@ class PredibaseChatCompletion(BaseLLM): litellm_params=litellm_params, logger_fn=logger_fn, headers=headers, + timeout=timeout, ) # type: ignore ### SYNC STREAMING - if stream == True: + if stream is True: response = requests.post( completion_url, headers=headers, @@ -452,7 +455,6 @@ class PredibaseChatCompletion(BaseLLM): headers=headers, data=json.dumps(data), ) - return self.process_response( model=model, response=response, @@ -480,23 +482,26 @@ class PredibaseChatCompletion(BaseLLM): stream, data: dict, optional_params: dict, + timeout: Union[float, httpx.Timeout], litellm_params=None, logger_fn=None, headers={}, ) -> ModelResponse: - self.async_handler = AsyncHTTPHandler( - timeout=httpx.Timeout(timeout=600.0, connect=5.0) - ) + + async_handler = AsyncHTTPHandler(timeout=httpx.Timeout(timeout=timeout)) try: - response = await self.async_handler.post( + response = await async_handler.post( api_base, headers=headers, data=json.dumps(data) ) except httpx.HTTPStatusError as e: raise PredibaseError( - status_code=e.response.status_code, message=e.response.text + status_code=e.response.status_code, + message="HTTPStatusError - {}".format(e.response.text), ) except Exception as e: - raise PredibaseError(status_code=500, message=str(e)) + raise PredibaseError( + status_code=500, message="{}\n{}".format(str(e), traceback.format_exc()) + ) return self.process_response( model=model, response=response, @@ -522,6 +527,7 @@ class PredibaseChatCompletion(BaseLLM): api_key, logging_obj, data: dict, + timeout: Union[float, httpx.Timeout], optional_params=None, litellm_params=None, logger_fn=None, diff --git a/litellm/main.py b/litellm/main.py index dd1fdb9f9f..2c906e9902 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -432,9 +432,9 @@ def mock_completion( if isinstance(mock_response, openai.APIError): raise mock_response raise litellm.APIError( - status_code=500, # type: ignore - message=str(mock_response), - llm_provider="openai", # type: ignore + status_code=getattr(mock_response, "status_code", 500), # type: ignore + message=getattr(mock_response, "text", str(mock_response)), + llm_provider=getattr(mock_response, "llm_provider", "openai"), # type: ignore model=model, # type: ignore request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), ) @@ -1949,7 +1949,8 @@ def completion( ) api_base = ( - optional_params.pop("api_base", None) + api_base + or optional_params.pop("api_base", None) or optional_params.pop("base_url", None) or litellm.api_base or get_secret("PREDIBASE_API_BASE") @@ -1977,12 +1978,13 @@ def completion( custom_prompt_dict=custom_prompt_dict, api_key=api_key, tenant_id=tenant_id, + timeout=timeout, ) if ( "stream" in optional_params - and optional_params["stream"] == True - and acompletion == False + and optional_params["stream"] is True + and acompletion is False ): return _model_response response = _model_response diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html deleted file mode 100644 index 4e1a8fa4c5..0000000000 --- a/litellm/proxy/_experimental/out/404.html +++ /dev/null @@ -1 +0,0 @@ -404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file diff --git a/litellm/proxy/_experimental/out/model_hub.html b/litellm/proxy/_experimental/out/model_hub.html deleted file mode 100644 index 4da591c1c8..0000000000 --- a/litellm/proxy/_experimental/out/model_hub.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding.html deleted file mode 100644 index 197ab5c319..0000000000 --- a/litellm/proxy/_experimental/out/onboarding.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_super_secret_config.yaml b/litellm/proxy/_super_secret_config.yaml index 7fa1bbc19c..2b20547565 100644 --- a/litellm/proxy/_super_secret_config.yaml +++ b/litellm/proxy/_super_secret_config.yaml @@ -8,6 +8,17 @@ model_list: - model_name: llama3-70b-8192 litellm_params: model: groq/llama3-70b-8192 +- model_name: fake-openai-endpoint + litellm_params: + model: predibase/llama-3-8b-instruct + api_base: "http://0.0.0.0:8081" + api_key: os.environ/PREDIBASE_API_KEY + tenant_id: os.environ/PREDIBASE_TENANT_ID + max_retries: 0 + temperature: 0.1 + max_new_tokens: 256 + return_full_text: false + # - litellm_params: # api_base: https://my-endpoint-europe-berri-992.openai.azure.com/ # api_key: os.environ/AZURE_EUROPE_API_KEY @@ -56,10 +67,11 @@ router_settings: litellm_settings: success_callback: ["langfuse"] + failure_callback: ["langfuse"] -general_settings: - alerting: ["email"] - key_management_system: "aws_kms" - key_management_settings: - hosted_keys: ["LITELLM_MASTER_KEY"] +# general_settings: +# alerting: ["email"] +# key_management_system: "aws_kms" +# key_management_settings: +# hosted_keys: ["LITELLM_MASTER_KEY"] diff --git a/litellm/tests/test_exceptions.py b/litellm/tests/test_exceptions.py index ee695dcd7c..1082dd2f89 100644 --- a/litellm/tests/test_exceptions.py +++ b/litellm/tests/test_exceptions.py @@ -3,6 +3,7 @@ import os import sys import traceback import subprocess, asyncio +from typing import Any sys.path.insert( 0, os.path.abspath("../..") @@ -19,6 +20,7 @@ from litellm import ( ) from concurrent.futures import ThreadPoolExecutor import pytest +from unittest.mock import patch, MagicMock litellm.vertex_project = "pathrise-convert-1606954137718" litellm.vertex_location = "us-central1" @@ -655,3 +657,47 @@ def test_litellm_predibase_exception(): # accuracy_score = counts[True]/(counts[True] + counts[False]) # print(f"accuracy_score: {accuracy_score}") + + +@pytest.mark.parametrize("provider", ["predibase"]) +def test_exception_mapping(provider): + """ + For predibase, run through a set of mock exceptions + + assert that they are being mapped correctly + """ + litellm.set_verbose = True + error_map = { + 400: litellm.BadRequestError, + 401: litellm.AuthenticationError, + 404: litellm.NotFoundError, + 408: litellm.Timeout, + 429: litellm.RateLimitError, + 500: litellm.InternalServerError, + 503: litellm.ServiceUnavailableError, + } + + for code, expected_exception in error_map.items(): + mock_response = Exception() + setattr(mock_response, "text", "This is an error message") + setattr(mock_response, "llm_provider", provider) + setattr(mock_response, "status_code", code) + + response: Any = None + try: + response = completion( + model="{}/test-model".format(provider), + messages=[{"role": "user", "content": "Hey, how's it going?"}], + mock_response=mock_response, + ) + except expected_exception: + continue + except Exception as e: + response = "{}\n{}".format(str(e), traceback.format_exc()) + pytest.fail( + "Did not raise expected exception. Expected={}, Return={},".format( + expected_exception, response + ) + ) + + pass diff --git a/litellm/utils.py b/litellm/utils.py index 410f9ad882..bc55e6ef7b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8978,6 +8978,75 @@ def exception_type( response=original_exception.response, litellm_debug_info=extra_information, ) + elif hasattr(original_exception, "status_code"): + if original_exception.status_code == 500: + exception_mapping_worked = True + raise litellm.InternalServerError( + message=f"PredibaseException - {original_exception.message}", + llm_provider="predibase", + model=model, + ) + elif original_exception.status_code == 401: + exception_mapping_worked = True + raise AuthenticationError( + message=f"PredibaseException - {original_exception.message}", + llm_provider="predibase", + model=model, + ) + elif original_exception.status_code == 400: + exception_mapping_worked = True + raise BadRequestError( + message=f"PredibaseException - {original_exception.message}", + llm_provider="predibase", + model=model, + ) + elif original_exception.status_code == 404: + exception_mapping_worked = True + raise NotFoundError( + message=f"PredibaseException - {original_exception.message}", + llm_provider="predibase", + model=model, + ) + elif original_exception.status_code == 408: + exception_mapping_worked = True + raise Timeout( + message=f"PredibaseException - {original_exception.message}", + model=model, + llm_provider=custom_llm_provider, + litellm_debug_info=extra_information, + ) + elif original_exception.status_code == 422: + exception_mapping_worked = True + raise BadRequestError( + message=f"PredibaseException - {original_exception.message}", + model=model, + llm_provider=custom_llm_provider, + litellm_debug_info=extra_information, + ) + elif original_exception.status_code == 429: + exception_mapping_worked = True + raise RateLimitError( + message=f"PredibaseException - {original_exception.message}", + model=model, + llm_provider=custom_llm_provider, + litellm_debug_info=extra_information, + ) + elif original_exception.status_code == 503: + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"PredibaseException - {original_exception.message}", + model=model, + llm_provider=custom_llm_provider, + litellm_debug_info=extra_information, + ) + elif original_exception.status_code == 504: # gateway timeout error + exception_mapping_worked = True + raise Timeout( + message=f"PredibaseException - {original_exception.message}", + model=model, + llm_provider=custom_llm_provider, + litellm_debug_info=extra_information, + ) elif custom_llm_provider == "bedrock": if ( "too many tokens" in error_str diff --git a/ruff.toml b/ruff.toml index dfb323c1b3..4894ab3fce 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,3 +1,3 @@ -ignore = ["F405"] +ignore = ["F405", "E402"] extend-select = ["E501"] line-length = 120 From b26c3c7d22f4465e47a2d4ff9fa14429213f3e7e Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 8 Jun 2024 19:43:57 -0700 Subject: [PATCH 02/29] fix(cost_calculator.py): fixes tgai unmapped model pricing Fixes error where tgai helper function returned None. Enforces stronger type hints, refactors code, adds more unit testing. --- litellm/__init__.py | 2 +- litellm/cost_calculator.py | 280 +++++++++++++++++++++++++- litellm/tests/test_completion_cost.py | 49 +++++ litellm/utils.py | 259 +----------------------- 4 files changed, 329 insertions(+), 261 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index b6e6d97dc8..e92ae355e2 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -709,6 +709,7 @@ all_embedding_models = ( openai_image_generation_models = ["dall-e-2", "dall-e-3"] from .timeout import timeout +from .cost_calculator import completion_cost from .utils import ( client, exception_type, @@ -718,7 +719,6 @@ from .utils import ( create_pretrained_tokenizer, create_tokenizer, cost_per_token, - completion_cost, supports_function_calling, supports_parallel_function_calling, supports_vision, diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 75717378b0..e90868da60 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1,6 +1,7 @@ # What is this? ## File for 'response_cost' calculation in Logging -from typing import Optional, Union, Literal +from typing import Optional, Union, Literal, List +import litellm._logging from litellm.utils import ( ModelResponse, EmbeddingResponse, @@ -8,10 +9,279 @@ from litellm.utils import ( TranscriptionResponse, TextCompletionResponse, CallTypes, - completion_cost, + cost_per_token, print_verbose, + CostPerToken, + token_counter, ) import litellm +from litellm import verbose_logger + + +# Extract the number of billion parameters from the model name +# only used for together_computer LLMs +def get_model_params_and_category(model_name) -> str: + """ + Helper function for calculating together ai pricing. + + Returns + - str - model pricing category if mapped else received model name + """ + import re + + model_name = model_name.lower() + re_params_match = re.search( + r"(\d+b)", model_name + ) # catch all decimals like 3b, 70b, etc + category = None + if re_params_match is not None: + params_match = str(re_params_match.group(1)) + params_match = params_match.replace("b", "") + if params_match is not None: + params_billion = float(params_match) + else: + return model_name + # Determine the category based on the number of parameters + if params_billion <= 3.0: + category = "together-ai-up-to-3b" + elif params_billion <= 7.0: + category = "together-ai-3.1b-7b" + elif params_billion <= 20.0: + category = "together-ai-7.1b-20b" + elif params_billion <= 40.0: + category = "together-ai-20.1b-40b" + elif params_billion <= 70.0: + category = "together-ai-40.1b-70b" + if category is not None: + return category + + return model_name + + +def get_replicate_completion_pricing(completion_response=None, total_time=0.0): + # see https://replicate.com/pricing + # for all litellm currently supported LLMs, almost all requests go to a100_80gb + a100_80gb_price_per_second_public = ( + 0.001400 # assume all calls sent to A100 80GB for now + ) + if total_time == 0.0: # total time is in ms + start_time = completion_response["created"] + end_time = getattr(completion_response, "ended", time.time()) + total_time = end_time - start_time + + return a100_80gb_price_per_second_public * total_time / 1000 + + +def completion_cost( + completion_response=None, + model: Optional[str] = None, + prompt="", + messages: List = [], + completion="", + total_time=0.0, # used for replicate, sagemaker + call_type: Literal[ + "embedding", + "aembedding", + "completion", + "acompletion", + "atext_completion", + "text_completion", + "image_generation", + "aimage_generation", + "moderation", + "amoderation", + "atranscription", + "transcription", + "aspeech", + "speech", + ] = "completion", + ### REGION ### + custom_llm_provider=None, + region_name=None, # used for bedrock pricing + ### IMAGE GEN ### + size=None, + quality=None, + n=None, # number of images + ### CUSTOM PRICING ### + custom_cost_per_token: Optional[CostPerToken] = None, + custom_cost_per_second: Optional[float] = None, +) -> float: + """ + Calculate the cost of a given completion call fot GPT-3.5-turbo, llama2, any litellm supported llm. + + Parameters: + completion_response (litellm.ModelResponses): [Required] The response received from a LiteLLM completion request. + + [OPTIONAL PARAMS] + model (str): Optional. The name of the language model used in the completion calls + prompt (str): Optional. The input prompt passed to the llm + completion (str): Optional. The output completion text from the llm + total_time (float): Optional. (Only used for Replicate LLMs) The total time used for the request in seconds + custom_cost_per_token: Optional[CostPerToken]: the cost per input + output token for the llm api call. + custom_cost_per_second: Optional[float]: the cost per second for the llm api call. + + Returns: + float: The cost in USD dollars for the completion based on the provided parameters. + + Exceptions: + Raises exception if model not in the litellm model cost map. Register model, via custom pricing or PR - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json + + + Note: + - If completion_response is provided, the function extracts token information and the model name from it. + - If completion_response is not provided, the function calculates token counts based on the model and input text. + - The cost is calculated based on the model, prompt tokens, and completion tokens. + - For certain models containing "togethercomputer" in the name, prices are based on the model size. + - For un-mapped Replicate models, the cost is calculated based on the total time used for the request. + """ + try: + if ( + (call_type == "aimage_generation" or call_type == "image_generation") + and model is not None + and isinstance(model, str) + and len(model) == 0 + and custom_llm_provider == "azure" + ): + model = "dall-e-2" # for dall-e-2, azure expects an empty model name + # Handle Inputs to completion_cost + prompt_tokens = 0 + completion_tokens = 0 + custom_llm_provider = None + if completion_response is not None: + # get input/output tokens from completion_response + prompt_tokens = completion_response.get("usage", {}).get("prompt_tokens", 0) + completion_tokens = completion_response.get("usage", {}).get( + "completion_tokens", 0 + ) + total_time = completion_response.get("_response_ms", 0) + verbose_logger.debug( + f"completion_response response ms: {completion_response.get('_response_ms')} " + ) + model = model or completion_response.get( + "model", None + ) # check if user passed an override for model, if it's none check completion_response['model'] + if hasattr(completion_response, "_hidden_params"): + if ( + completion_response._hidden_params.get("model", None) is not None + and len(completion_response._hidden_params["model"]) > 0 + ): + model = completion_response._hidden_params.get("model", model) + custom_llm_provider = completion_response._hidden_params.get( + "custom_llm_provider", "" + ) + region_name = completion_response._hidden_params.get( + "region_name", region_name + ) + size = completion_response._hidden_params.get( + "optional_params", {} + ).get( + "size", "1024-x-1024" + ) # openai default + quality = completion_response._hidden_params.get( + "optional_params", {} + ).get( + "quality", "standard" + ) # openai default + n = completion_response._hidden_params.get("optional_params", {}).get( + "n", 1 + ) # openai default + else: + if len(messages) > 0: + prompt_tokens = token_counter(model=model, messages=messages) + elif len(prompt) > 0: + prompt_tokens = token_counter(model=model, text=prompt) + completion_tokens = token_counter(model=model, text=completion) + if model is None: + raise ValueError( + f"Model is None and does not exist in passed completion_response. Passed completion_response={completion_response}, model={model}" + ) + + if ( + call_type == CallTypes.image_generation.value + or call_type == CallTypes.aimage_generation.value + ): + ### IMAGE GENERATION COST CALCULATION ### + if custom_llm_provider == "vertex_ai": + # https://cloud.google.com/vertex-ai/generative-ai/pricing + # Vertex Charges Flat $0.20 per image + return 0.020 + + # fix size to match naming convention + if "x" in size and "-x-" not in size: + size = size.replace("x", "-x-") + image_gen_model_name = f"{size}/{model}" + image_gen_model_name_with_quality = image_gen_model_name + if quality is not None: + image_gen_model_name_with_quality = f"{quality}/{image_gen_model_name}" + size = size.split("-x-") + height = int(size[0]) # if it's 1024-x-1024 vs. 1024x1024 + width = int(size[1]) + verbose_logger.debug(f"image_gen_model_name: {image_gen_model_name}") + verbose_logger.debug( + f"image_gen_model_name_with_quality: {image_gen_model_name_with_quality}" + ) + if image_gen_model_name in litellm.model_cost: + return ( + litellm.model_cost[image_gen_model_name]["input_cost_per_pixel"] + * height + * width + * n + ) + elif image_gen_model_name_with_quality in litellm.model_cost: + return ( + litellm.model_cost[image_gen_model_name_with_quality][ + "input_cost_per_pixel" + ] + * height + * width + * n + ) + else: + raise Exception( + f"Model={image_gen_model_name} not found in completion cost model map" + ) + # Calculate cost based on prompt_tokens, completion_tokens + if ( + "togethercomputer" in model + or "together_ai" in model + or custom_llm_provider == "together_ai" + ): + # together ai prices based on size of llm + # get_model_params_and_category takes a model name and returns the category of LLM size it is in model_prices_and_context_window.json + model = get_model_params_and_category(model) + # replicate llms are calculate based on time for request running + # see https://replicate.com/pricing + elif ( + model in litellm.replicate_models or "replicate" in model + ) and model not in litellm.model_cost: + # for unmapped replicate model, default to replicate's time tracking logic + return get_replicate_completion_pricing(completion_response, total_time) + + if model is None: + raise ValueError( + f"Model is None and does not exist in passed completion_response. Passed completion_response={completion_response}, model={model}" + ) + + ( + prompt_tokens_cost_usd_dollar, + completion_tokens_cost_usd_dollar, + ) = cost_per_token( + model=model, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + custom_llm_provider=custom_llm_provider, + response_time_ms=total_time, + region_name=region_name, + custom_cost_per_second=custom_cost_per_second, + custom_cost_per_token=custom_cost_per_token, + ) + _final_cost = prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar + print_verbose( + f"final cost: {_final_cost}; prompt_tokens_cost_usd_dollar: {prompt_tokens_cost_usd_dollar}; completion_tokens_cost_usd_dollar: {completion_tokens_cost_usd_dollar}" + ) + return _final_cost + except Exception as e: + raise e def response_cost_calculator( @@ -47,7 +317,7 @@ def response_cost_calculator( ) -> Optional[float]: try: response_cost: float = 0.0 - if cache_hit is not None and cache_hit == True: + if cache_hit is not None and cache_hit is True: response_cost = 0.0 else: response_object._hidden_params["optional_params"] = optional_params @@ -62,9 +332,11 @@ def response_cost_calculator( if ( model in litellm.model_cost and custom_pricing is not None - and custom_llm_provider == True + and custom_llm_provider is True ): # override defaults if custom pricing is set base_model = model + elif base_model is None: + base_model = model # base_model defaults to None if not set on model_info response_cost = completion_cost( completion_response=response_object, diff --git a/litellm/tests/test_completion_cost.py b/litellm/tests/test_completion_cost.py index 7820e2af3d..1f0dc1fc1b 100644 --- a/litellm/tests/test_completion_cost.py +++ b/litellm/tests/test_completion_cost.py @@ -517,3 +517,52 @@ def test_groq_response_cost_tracking(is_streaming): assert response_cost > 0.0 print(f"response_cost: {response_cost}") + + +def test_together_ai_qwen_completion_cost(): + input_kwargs = { + "completion_response": litellm.ModelResponse( + **{ + "id": "890db0c33c4ef94b-SJC", + "choices": [ + { + "finish_reason": "eos", + "index": 0, + "message": { + "content": "I am Qwen, a large language model created by Alibaba Cloud.", + "role": "assistant", + }, + } + ], + "created": 1717900130, + "model": "together_ai/qwen/Qwen2-72B-Instruct", + "object": "chat.completion", + "system_fingerprint": None, + "usage": { + "completion_tokens": 15, + "prompt_tokens": 23, + "total_tokens": 38, + }, + } + ), + "model": "qwen/Qwen2-72B-Instruct", + "prompt": "", + "messages": [], + "completion": "", + "total_time": 0.0, + "call_type": "completion", + "custom_llm_provider": "together_ai", + "region_name": None, + "size": None, + "quality": None, + "n": None, + "custom_cost_per_token": None, + "custom_cost_per_second": None, + } + + try: + litellm.completion_cost(**input_kwargs) + except litellm.NotFoundError: + pass + except Exception: + pytest.fail("This should have returned a 'not found error'") diff --git a/litellm/utils.py b/litellm/utils.py index 410f9ad882..cc94db113d 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3929,54 +3929,6 @@ def client(original_function): return wrapper -####### USAGE CALCULATOR ################ - - -# Extract the number of billion parameters from the model name -# only used for together_computer LLMs -def get_model_params_and_category(model_name): - import re - - model_name = model_name.lower() - params_match = re.search( - r"(\d+b)", model_name - ) # catch all decimals like 3b, 70b, etc - category = None - if params_match != None: - params_match = params_match.group(1) - params_match = params_match.replace("b", "") - params_billion = float(params_match) - # Determine the category based on the number of parameters - if params_billion <= 3.0: - category = "together-ai-up-to-3b" - elif params_billion <= 7.0: - category = "together-ai-3.1b-7b" - elif params_billion <= 20.0: - category = "together-ai-7.1b-20b" - elif params_billion <= 40.0: - category = "together-ai-20.1b-40b" - elif params_billion <= 70.0: - category = "together-ai-40.1b-70b" - return category - - return None - - -def get_replicate_completion_pricing(completion_response=None, total_time=0.0): - # see https://replicate.com/pricing - a100_40gb_price_per_second_public = 0.001150 - # for all litellm currently supported LLMs, almost all requests go to a100_80gb - a100_80gb_price_per_second_public = ( - 0.001400 # assume all calls sent to A100 80GB for now - ) - if total_time == 0.0: # total time is in ms - start_time = completion_response["created"] - end_time = getattr(completion_response, "ended", time.time()) - total_time = end_time - start_time - - return a100_80gb_price_per_second_public * total_time / 1000 - - @lru_cache(maxsize=128) def _select_tokenizer(model: str): if model in litellm.cohere_models and "command-r" in model: @@ -4363,7 +4315,7 @@ def _cost_per_token_custom_pricing_helper( def cost_per_token( - model="", + model: str = "", prompt_tokens=0, completion_tokens=0, response_time_ms=None, @@ -4388,6 +4340,8 @@ def cost_per_token( Returns: tuple: A tuple containing the cost in USD dollars for prompt tokens and completion tokens, respectively. """ + if model is None: + raise Exception("Invalid arg. Model cannot be none.") ## CUSTOM PRICING ## response_cost = _cost_per_token_custom_pricing_helper( prompt_tokens=prompt_tokens, @@ -4560,213 +4514,6 @@ def cost_per_token( ) -def completion_cost( - completion_response=None, - model=None, - prompt="", - messages: List = [], - completion="", - total_time=0.0, # used for replicate, sagemaker - call_type: Literal[ - "embedding", - "aembedding", - "completion", - "acompletion", - "atext_completion", - "text_completion", - "image_generation", - "aimage_generation", - "moderation", - "amoderation", - "atranscription", - "transcription", - "aspeech", - "speech", - ] = "completion", - ### REGION ### - custom_llm_provider=None, - region_name=None, # used for bedrock pricing - ### IMAGE GEN ### - size=None, - quality=None, - n=None, # number of images - ### CUSTOM PRICING ### - custom_cost_per_token: Optional[CostPerToken] = None, - custom_cost_per_second: Optional[float] = None, -) -> float: - """ - Calculate the cost of a given completion call fot GPT-3.5-turbo, llama2, any litellm supported llm. - - Parameters: - completion_response (litellm.ModelResponses): [Required] The response received from a LiteLLM completion request. - - [OPTIONAL PARAMS] - model (str): Optional. The name of the language model used in the completion calls - prompt (str): Optional. The input prompt passed to the llm - completion (str): Optional. The output completion text from the llm - total_time (float): Optional. (Only used for Replicate LLMs) The total time used for the request in seconds - custom_cost_per_token: Optional[CostPerToken]: the cost per input + output token for the llm api call. - custom_cost_per_second: Optional[float]: the cost per second for the llm api call. - - Returns: - float: The cost in USD dollars for the completion based on the provided parameters. - - Exceptions: - Raises exception if model not in the litellm model cost map. Register model, via custom pricing or PR - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json - - - Note: - - If completion_response is provided, the function extracts token information and the model name from it. - - If completion_response is not provided, the function calculates token counts based on the model and input text. - - The cost is calculated based on the model, prompt tokens, and completion tokens. - - For certain models containing "togethercomputer" in the name, prices are based on the model size. - - For un-mapped Replicate models, the cost is calculated based on the total time used for the request. - """ - try: - if ( - (call_type == "aimage_generation" or call_type == "image_generation") - and model is not None - and isinstance(model, str) - and len(model) == 0 - and custom_llm_provider == "azure" - ): - model = "dall-e-2" # for dall-e-2, azure expects an empty model name - # Handle Inputs to completion_cost - prompt_tokens = 0 - completion_tokens = 0 - custom_llm_provider = None - if completion_response is not None: - # get input/output tokens from completion_response - prompt_tokens = completion_response.get("usage", {}).get("prompt_tokens", 0) - completion_tokens = completion_response.get("usage", {}).get( - "completion_tokens", 0 - ) - total_time = completion_response.get("_response_ms", 0) - verbose_logger.debug( - f"completion_response response ms: {completion_response.get('_response_ms')} " - ) - model = model or completion_response.get( - "model", None - ) # check if user passed an override for model, if it's none check completion_response['model'] - if hasattr(completion_response, "_hidden_params"): - if ( - completion_response._hidden_params.get("model", None) is not None - and len(completion_response._hidden_params["model"]) > 0 - ): - model = completion_response._hidden_params.get("model", model) - custom_llm_provider = completion_response._hidden_params.get( - "custom_llm_provider", "" - ) - region_name = completion_response._hidden_params.get( - "region_name", region_name - ) - size = completion_response._hidden_params.get( - "optional_params", {} - ).get( - "size", "1024-x-1024" - ) # openai default - quality = completion_response._hidden_params.get( - "optional_params", {} - ).get( - "quality", "standard" - ) # openai default - n = completion_response._hidden_params.get("optional_params", {}).get( - "n", 1 - ) # openai default - else: - if len(messages) > 0: - prompt_tokens = token_counter(model=model, messages=messages) - elif len(prompt) > 0: - prompt_tokens = token_counter(model=model, text=prompt) - completion_tokens = token_counter(model=model, text=completion) - if model == None: - raise ValueError( - f"Model is None and does not exist in passed completion_response. Passed completion_response={completion_response}, model={model}" - ) - - if ( - call_type == CallTypes.image_generation.value - or call_type == CallTypes.aimage_generation.value - ): - ### IMAGE GENERATION COST CALCULATION ### - if custom_llm_provider == "vertex_ai": - # https://cloud.google.com/vertex-ai/generative-ai/pricing - # Vertex Charges Flat $0.20 per image - return 0.020 - - # fix size to match naming convention - if "x" in size and "-x-" not in size: - size = size.replace("x", "-x-") - image_gen_model_name = f"{size}/{model}" - image_gen_model_name_with_quality = image_gen_model_name - if quality is not None: - image_gen_model_name_with_quality = f"{quality}/{image_gen_model_name}" - size = size.split("-x-") - height = int(size[0]) # if it's 1024-x-1024 vs. 1024x1024 - width = int(size[1]) - verbose_logger.debug(f"image_gen_model_name: {image_gen_model_name}") - verbose_logger.debug( - f"image_gen_model_name_with_quality: {image_gen_model_name_with_quality}" - ) - if image_gen_model_name in litellm.model_cost: - return ( - litellm.model_cost[image_gen_model_name]["input_cost_per_pixel"] - * height - * width - * n - ) - elif image_gen_model_name_with_quality in litellm.model_cost: - return ( - litellm.model_cost[image_gen_model_name_with_quality][ - "input_cost_per_pixel" - ] - * height - * width - * n - ) - else: - raise Exception( - f"Model={image_gen_model_name} not found in completion cost model map" - ) - # Calculate cost based on prompt_tokens, completion_tokens - if ( - "togethercomputer" in model - or "together_ai" in model - or custom_llm_provider == "together_ai" - ): - # together ai prices based on size of llm - # get_model_params_and_category takes a model name and returns the category of LLM size it is in model_prices_and_context_window.json - model = get_model_params_and_category(model) - # replicate llms are calculate based on time for request running - # see https://replicate.com/pricing - elif ( - model in litellm.replicate_models or "replicate" in model - ) and model not in litellm.model_cost: - # for unmapped replicate model, default to replicate's time tracking logic - return get_replicate_completion_pricing(completion_response, total_time) - - ( - prompt_tokens_cost_usd_dollar, - completion_tokens_cost_usd_dollar, - ) = cost_per_token( - model=model, - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - custom_llm_provider=custom_llm_provider, - response_time_ms=total_time, - region_name=region_name, - custom_cost_per_second=custom_cost_per_second, - custom_cost_per_token=custom_cost_per_token, - ) - _final_cost = prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar - print_verbose( - f"final cost: {_final_cost}; prompt_tokens_cost_usd_dollar: {prompt_tokens_cost_usd_dollar}; completion_tokens_cost_usd_dollar: {completion_tokens_cost_usd_dollar}" - ) - return _final_cost - except Exception as e: - raise e - - def supports_httpx_timeout(custom_llm_provider: str) -> bool: """ Helper function to know if a provider implementation supports httpx timeout From 0253c2b2136826ae4fd8eec73716e00c2ca0fb08 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 8 Jun 2024 19:47:31 -0700 Subject: [PATCH 03/29] feat - allow ChatCompletionMessageToolCall, and Function to be used as dict --- litellm/integrations/opentelemetry.py | 2 -- litellm/utils.py | 32 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 01d9661fc6..49b304c670 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -364,8 +364,6 @@ class OpenTelemetry(CustomLogger): ) message = choice.get("message") - if not isinstance(message, dict): - message = message.dict() tool_calls = message.get("tool_calls") if tool_calls: span.set_attribute( diff --git a/litellm/utils.py b/litellm/utils.py index 410f9ad882..12440d4143 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -326,6 +326,22 @@ class Function(OpenAIObject): super(Function, self).__init__(**data) + def __contains__(self, key): + # Define custom behavior for the 'in' operator + return hasattr(self, key) + + def get(self, key, default=None): + # Custom .get() method to access attributes with a default value if the attribute doesn't exist + return getattr(self, key, default) + + def __getitem__(self, key): + # Allow dictionary-style access to attributes + return getattr(self, key) + + def __setitem__(self, key, value): + # Allow dictionary-style assignment of attributes + setattr(self, key, value) + class ChatCompletionDeltaToolCall(OpenAIObject): id: Optional[str] = None @@ -385,6 +401,22 @@ class ChatCompletionMessageToolCall(OpenAIObject): else: self.type = "function" + def __contains__(self, key): + # Define custom behavior for the 'in' operator + return hasattr(self, key) + + def get(self, key, default=None): + # Custom .get() method to access attributes with a default value if the attribute doesn't exist + return getattr(self, key, default) + + def __getitem__(self, key): + # Allow dictionary-style access to attributes + return getattr(self, key) + + def __setitem__(self, key, value): + # Allow dictionary-style assignment of attributes + setattr(self, key, value) + class Message(OpenAIObject): def __init__( From f3a845eff920f7d2ffb7d9a32956ce2c9bbd26bb Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 8 Jun 2024 19:56:35 -0700 Subject: [PATCH 04/29] build(model_prices_and_context_window.json): update together ai model pricing - account for new categories --- litellm/cost_calculator.py | 22 ++++++++++--------- ...odel_prices_and_context_window_backup.json | 19 ++++++++++------ litellm/tests/test_completion_cost.py | 11 +++++----- model_prices_and_context_window.json | 19 ++++++++++------ 4 files changed, 41 insertions(+), 30 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index e90868da60..9a763d63ed 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -42,16 +42,18 @@ def get_model_params_and_category(model_name) -> str: else: return model_name # Determine the category based on the number of parameters - if params_billion <= 3.0: - category = "together-ai-up-to-3b" - elif params_billion <= 7.0: - category = "together-ai-3.1b-7b" - elif params_billion <= 20.0: - category = "together-ai-7.1b-20b" - elif params_billion <= 40.0: - category = "together-ai-20.1b-40b" - elif params_billion <= 70.0: - category = "together-ai-40.1b-70b" + if params_billion <= 4.0: + category = "together-ai-up-to-4b" + elif params_billion <= 8.0: + category = "together-ai-4.1b-8b" + elif params_billion <= 21.0: + category = "together-ai-8.1b-21b" + elif params_billion <= 41.0: + category = "together-ai-21.1b-41b" + elif params_billion <= 80.0: + category = "together-ai-41.1b-80b" + elif params_billion <= 110.0: + category = "together-ai-81.1b-110b" if category is not None: return category diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3fe089a6b2..f2b292c928 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3009,32 +3009,37 @@ "litellm_provider": "sagemaker", "mode": "chat" }, - "together-ai-up-to-3b": { + "together-ai-up-to-4b": { "input_cost_per_token": 0.0000001, "output_cost_per_token": 0.0000001, "litellm_provider": "together_ai" }, - "together-ai-3.1b-7b": { + "together-ai-4.1b-8b": { "input_cost_per_token": 0.0000002, "output_cost_per_token": 0.0000002, "litellm_provider": "together_ai" }, - "together-ai-7.1b-20b": { + "together-ai-8.1b-21b": { "max_tokens": 1000, - "input_cost_per_token": 0.0000004, - "output_cost_per_token": 0.0000004, + "input_cost_per_token": 0.0000003, + "output_cost_per_token": 0.0000003, "litellm_provider": "together_ai" }, - "together-ai-20.1b-40b": { + "together-ai-21.1b-41b": { "input_cost_per_token": 0.0000008, "output_cost_per_token": 0.0000008, "litellm_provider": "together_ai" }, - "together-ai-40.1b-70b": { + "together-ai-41.1b-80b": { "input_cost_per_token": 0.0000009, "output_cost_per_token": 0.0000009, "litellm_provider": "together_ai" }, + "together-ai-81.1b-110b": { + "input_cost_per_token": 0.0000018, + "output_cost_per_token": 0.0000018, + "litellm_provider": "together_ai" + }, "together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1": { "input_cost_per_token": 0.0000006, "output_cost_per_token": 0.0000006, diff --git a/litellm/tests/test_completion_cost.py b/litellm/tests/test_completion_cost.py index 1f0dc1fc1b..c0be350f91 100644 --- a/litellm/tests/test_completion_cost.py +++ b/litellm/tests/test_completion_cost.py @@ -560,9 +560,8 @@ def test_together_ai_qwen_completion_cost(): "custom_cost_per_second": None, } - try: - litellm.completion_cost(**input_kwargs) - except litellm.NotFoundError: - pass - except Exception: - pytest.fail("This should have returned a 'not found error'") + response = litellm.cost_calculator.get_model_params_and_category( + model_name="qwen/Qwen2-72B-Instruct" + ) + + assert response == "together-ai-41.1b-80b" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3fe089a6b2..f2b292c928 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3009,32 +3009,37 @@ "litellm_provider": "sagemaker", "mode": "chat" }, - "together-ai-up-to-3b": { + "together-ai-up-to-4b": { "input_cost_per_token": 0.0000001, "output_cost_per_token": 0.0000001, "litellm_provider": "together_ai" }, - "together-ai-3.1b-7b": { + "together-ai-4.1b-8b": { "input_cost_per_token": 0.0000002, "output_cost_per_token": 0.0000002, "litellm_provider": "together_ai" }, - "together-ai-7.1b-20b": { + "together-ai-8.1b-21b": { "max_tokens": 1000, - "input_cost_per_token": 0.0000004, - "output_cost_per_token": 0.0000004, + "input_cost_per_token": 0.0000003, + "output_cost_per_token": 0.0000003, "litellm_provider": "together_ai" }, - "together-ai-20.1b-40b": { + "together-ai-21.1b-41b": { "input_cost_per_token": 0.0000008, "output_cost_per_token": 0.0000008, "litellm_provider": "together_ai" }, - "together-ai-40.1b-70b": { + "together-ai-41.1b-80b": { "input_cost_per_token": 0.0000009, "output_cost_per_token": 0.0000009, "litellm_provider": "together_ai" }, + "together-ai-81.1b-110b": { + "input_cost_per_token": 0.0000018, + "output_cost_per_token": 0.0000018, + "litellm_provider": "together_ai" + }, "together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1": { "input_cost_per_token": 0.0000006, "output_cost_per_token": 0.0000006, From 205ade22e5b45624f39c448b0a3fddc8517ac309 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 8 Jun 2024 20:49:18 -0700 Subject: [PATCH 05/29] =?UTF-8?q?bump:=20version=201.40.7=20=E2=86=92=201.?= =?UTF-8?q?40.8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b6deb0dacb..8255ddf796 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.40.7" +version = "1.40.8" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -84,7 +84,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.40.7" +version = "1.40.8" version_files = [ "pyproject.toml:^version" ] From 3b4a8f776d763e5b97df9cbf85af42ef8ae732ff Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 8 Jun 2024 20:57:14 -0700 Subject: [PATCH 06/29] docs - Attribute Management changes --- docs/my-website/docs/proxy/multiple_admins.md | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/my-website/docs/proxy/multiple_admins.md b/docs/my-website/docs/proxy/multiple_admins.md index 388df0d603..376ff0174d 100644 --- a/docs/my-website/docs/proxy/multiple_admins.md +++ b/docs/my-website/docs/proxy/multiple_admins.md @@ -2,11 +2,21 @@ Call management endpoints on behalf of a user. (Useful when connecting proxy to your development platform). -:::info -Requires Enterprise License for usage. -::: -## Set `LiteLLM-Changed-By` in request headers +:::tip + +Requires Enterprise License, Get in touch with us [here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) + +::: + +## 1. Switch on audit Logs +Add `store_audit_logs` to your litellm config.yaml and then start the proxy +```shell +litellm_settings: + store_audit_logs: true +``` + +## 2. Set `LiteLLM-Changed-By` in request headers Set the 'user_id' in request headers, when calling a management endpoint. [View Full List](https://litellm-api.up.railway.app/#/team%20management). @@ -26,7 +36,7 @@ curl -X POST 'http://0.0.0.0:4000/team/update' \ }' ``` -## Emitted Audit Log +## 3. Emitted Audit Log ```bash { From 8de7074614f62ec622e085c527b8a923f967e29d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 8 Jun 2024 21:02:47 -0700 Subject: [PATCH 07/29] [Feature]: Proxy: Support API-Key --- litellm/proxy/proxy_server.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5a068dd683..40fca0c046 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -368,6 +368,11 @@ from typing import Dict api_key_header = APIKeyHeader( name="Authorization", auto_error=False, description="Bearer token" ) +azure_api_key_header = APIKeyHeader( + name="API-Key", + auto_error=False, + description="Some older versions of the openai Python package will send an API-Key header with just the API key ", +) user_api_base = None user_model = None user_debug = False @@ -508,13 +513,19 @@ async def check_request_disconnection(request: Request, llm_api_call_task): async def user_api_key_auth( - request: Request, api_key: str = fastapi.Security(api_key_header) + request: Request, + api_key: str = fastapi.Security(api_key_header), + azure_api_key_header: str = fastapi.Security(azure_api_key_header), ) -> UserAPIKeyAuth: global master_key, prisma_client, llm_model_list, user_custom_auth, custom_db_client, general_settings, proxy_logging_obj try: if isinstance(api_key, str): passed_in_key = api_key api_key = _get_bearer_token(api_key=api_key) + + elif isinstance(azure_api_key_header, str): + api_key = azure_api_key_header + parent_otel_span: Optional[Span] = None if open_telemetry_logger is not None: parent_otel_span = open_telemetry_logger.tracer.start_span( From 78dbb15ccde05ab6397031014f28255824363631 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 8 Jun 2024 21:05:35 -0700 Subject: [PATCH 08/29] ci/cd run again --- litellm/tests/test_completion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 4ac727cd2b..a669dccfec 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -345,7 +345,7 @@ def test_completion_claude_3_function_call(model): drop_params=True, ) - # Add any assertions, here to check response args + # Add any assertions here to check response args print(response) assert isinstance(response.choices[0].message.tool_calls[0].function.name, str) assert isinstance( From afa0b4eee82904edf15ac59a6f9733a46bb77e08 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Jun 2024 09:18:51 -0700 Subject: [PATCH 09/29] docs - cache controls --- docs/my-website/docs/caching/all_caches.md | 88 ++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/docs/my-website/docs/caching/all_caches.md b/docs/my-website/docs/caching/all_caches.md index eb309f9b8b..1b8bbd8e09 100644 --- a/docs/my-website/docs/caching/all_caches.md +++ b/docs/my-website/docs/caching/all_caches.md @@ -212,6 +212,94 @@ If you run the code two times, response1 will use the cache from the first run t + + +## Switch Cache On / Off Per LiteLLM Call + +LiteLLM supports 4 cache-controls: + +- `no-cache`: *Optional(bool)* When `True`, Will not return a cached response, but instead call the actual endpoint. +- `no-store`: *Optional(bool)* When `True`, Will not cache the response. +- `ttl`: *Optional(int)* - Will cache the response for the user-defined amount of time (in seconds). +- `s-maxage`: *Optional(int)* Will only accept cached responses that are within user-defined range (in seconds). + +[Let us know if you need more](https://github.com/BerriAI/litellm/issues/1218) + + + +Example usage `no-cache` - When `True`, Will not return a cached response + +```python +response = litellm.completion( + model="gpt-3.5-turbo", + messages=[ + { + "role": "user", + "content": "hello who are you" + } + ], + cache={"no-cache": True}, + ) +``` + + + + + +Example usage `no-store` - When `True`, Will not cache the response. + +```python +response = litellm.completion( + model="gpt-3.5-turbo", + messages=[ + { + "role": "user", + "content": "hello who are you" + } + ], + cache={"no-store": True}, + ) +``` + + + + +Example usage `ttl` - cache the response for 10 seconds + +```python +response = litellm.completion( + model="gpt-3.5-turbo", + messages=[ + { + "role": "user", + "content": "hello who are you" + } + ], + cache={"ttl": 10}, + ) +``` + + + + +Example usage `s-maxage` - Will only accept cached responses for 60 seconds + +```python +response = litellm.completion( + model="gpt-3.5-turbo", + messages=[ + { + "role": "user", + "content": "hello who are you" + } + ], + cache={"s-maxage": 60}, + ) +``` + + + + ## Cache Context Manager - Enable, Disable, Update Cache From d5a1cc282e21ecffc2e05fca1e7e57009aa4eaa2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Jun 2024 09:29:38 -0700 Subject: [PATCH 10/29] test fix - cohere API Made a breaking change --- litellm/tests/test_completion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index a669dccfec..2428cbf48d 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -530,6 +530,7 @@ def test_completion_cohere_command_r_plus_function_call(): messages=messages, tools=tools, tool_choice="auto", + force_single_step=True, ) print(second_response) except Exception as e: From 03ef4d683a14336973cb3cf113b1b63f7a9ef338 Mon Sep 17 00:00:00 2001 From: JakobDylanC Date: Mon, 10 Jun 2024 13:48:24 -0400 Subject: [PATCH 11/29] docs: add llmcord.py to side bar nav --- docs/my-website/sidebars.js | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index b6b597d305..86bb62f092 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -255,6 +255,7 @@ const sidebars = { "projects/GPT Migrate", "projects/YiVal", "projects/LiteLLM Proxy", + "projects/llmcord.py (Discord LLM Chatbot)", ], }, ], From a27f7e2817abc2900084e4e945c3396c48b02b5a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Jun 2024 12:34:18 -0700 Subject: [PATCH 12/29] feat - log exceptions to OTEL --- litellm/proxy/proxy_server.py | 11 ++++++++++- litellm/tests/test_key_generate_prisma.py | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 40fca0c046..6bbebb5097 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -160,6 +160,7 @@ from litellm.proxy.auth.auth_checks import ( get_user_object, allowed_routes_check, get_actual_routes, + log_to_opentelemetry, ) from litellm.llms.custom_httpx.httpx_handler import HTTPHandler from litellm.exceptions import RejectedRequestError @@ -1506,7 +1507,7 @@ async def user_api_key_auth( ) if valid_token is None: # No token was found when looking up in the DB - raise Exception("Invalid token passed") + raise Exception("Invalid proxy server token passed") if valid_token_dict is not None: if user_id_information is not None and _is_user_proxy_admin( user_id_information @@ -1539,6 +1540,14 @@ async def user_api_key_auth( str(e) ) ) + + # Log this exception to OTEL + if open_telemetry_logger is not None: + await open_telemetry_logger.async_post_call_failure_hook( + original_exception=e, + user_api_key_dict=UserAPIKeyAuth(parent_otel_span=parent_otel_span), + ) + verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, litellm.BudgetExceededError): raise ProxyException( diff --git a/litellm/tests/test_key_generate_prisma.py b/litellm/tests/test_key_generate_prisma.py index 083d84c2b5..2f439862ef 100644 --- a/litellm/tests/test_key_generate_prisma.py +++ b/litellm/tests/test_key_generate_prisma.py @@ -272,7 +272,7 @@ def test_call_with_invalid_key(prisma_client): except Exception as e: print("Got Exception", e) print(e.message) - assert "Authentication Error, Invalid token passed" in e.message + assert "Authentication Error, Invalid proxy server token passed" in e.message pass From edc800afb9dc3681edea07c1cf9395f0ac8b401c Mon Sep 17 00:00:00 2001 From: JakobDylanC Date: Mon, 10 Jun 2024 15:53:57 -0400 Subject: [PATCH 13/29] docs: fix llmcord.py side bar link --- docs/my-website/sidebars.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 86bb62f092..50cfaed2ba 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -255,7 +255,7 @@ const sidebars = { "projects/GPT Migrate", "projects/YiVal", "projects/LiteLLM Proxy", - "projects/llmcord.py (Discord LLM Chatbot)", + "projects/llmcord.py Discord LLM Chatbot", ], }, ], From f00fa12e69de239652bb5756c4bfdcf26fac9c22 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Jun 2024 12:58:24 -0700 Subject: [PATCH 14/29] feat - viewing spend report per customer / team --- litellm/proxy/proxy_server.py | 167 +++++++++++++++++++++++----------- 1 file changed, 116 insertions(+), 51 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 40fca0c046..6f782ba308 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7814,6 +7814,10 @@ async def get_global_spend_report( default=None, description="Time till which to view spend", ), + group_by: Optional[Literal["team", "customer"]] = fastapi.Query( + default="team", + description="Group spend by internal team or customer", + ), ): """ Get Daily Spend per Team, based on specific startTime and endTime. Per team, view usage by each key, model @@ -7860,69 +7864,130 @@ async def get_global_spend_report( f"Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys" ) - # first get data from spend logs -> SpendByModelApiKey - # then read data from "SpendByModelApiKey" to format the response obj - sql_query = """ + if group_by == "team": + # first get data from spend logs -> SpendByModelApiKey + # then read data from "SpendByModelApiKey" to format the response obj + sql_query = """ - WITH SpendByModelApiKey AS ( - SELECT - date_trunc('day', sl."startTime") AS group_by_day, - COALESCE(tt.team_alias, 'Unassigned Team') AS team_name, - sl.model, - sl.api_key, - SUM(sl.spend) AS model_api_spend, - SUM(sl.total_tokens) AS model_api_tokens - FROM - "LiteLLM_SpendLogs" sl - LEFT JOIN - "LiteLLM_TeamTable" tt - ON - sl.team_id = tt.team_id - WHERE - sl."startTime" BETWEEN $1::date AND $2::date - GROUP BY - date_trunc('day', sl."startTime"), - tt.team_alias, - sl.model, - sl.api_key - ) + WITH SpendByModelApiKey AS ( + SELECT + date_trunc('day', sl."startTime") AS group_by_day, + COALESCE(tt.team_alias, 'Unassigned Team') AS team_name, + sl.model, + sl.api_key, + SUM(sl.spend) AS model_api_spend, + SUM(sl.total_tokens) AS model_api_tokens + FROM + "LiteLLM_SpendLogs" sl + LEFT JOIN + "LiteLLM_TeamTable" tt + ON + sl.team_id = tt.team_id + WHERE + sl."startTime" BETWEEN $1::date AND $2::date + GROUP BY + date_trunc('day', sl."startTime"), + tt.team_alias, + sl.model, + sl.api_key + ) + SELECT + group_by_day, + jsonb_agg(jsonb_build_object( + 'team_name', team_name, + 'total_spend', total_spend, + 'metadata', metadata + )) AS teams + FROM ( + SELECT + group_by_day, + team_name, + SUM(model_api_spend) AS total_spend, + jsonb_agg(jsonb_build_object( + 'model', model, + 'api_key', api_key, + 'spend', model_api_spend, + 'total_tokens', model_api_tokens + )) AS metadata + FROM + SpendByModelApiKey + GROUP BY + group_by_day, + team_name + ) AS aggregated + GROUP BY + group_by_day + ORDER BY + group_by_day; + """ + + db_response = await prisma_client.db.query_raw( + sql_query, start_date_obj, end_date_obj + ) + if db_response is None: + return [] + + return db_response + + elif group_by == "customer": + sql_query = """ + + WITH SpendByModelApiKey AS ( + SELECT + date_trunc('day', sl."startTime") AS group_by_day, + sl.end_user AS customer, + sl.model, + sl.api_key, + SUM(sl.spend) AS model_api_spend, + SUM(sl.total_tokens) AS model_api_tokens + FROM + "LiteLLM_SpendLogs" sl + WHERE + sl."startTime" BETWEEN $1::date AND $2::date + GROUP BY + date_trunc('day', sl."startTime"), + customer, + sl.model, + sl.api_key + ) SELECT group_by_day, jsonb_agg(jsonb_build_object( - 'team_name', team_name, + 'customer', customer, 'total_spend', total_spend, 'metadata', metadata - )) AS teams - FROM ( - SELECT - group_by_day, - team_name, - SUM(model_api_spend) AS total_spend, - jsonb_agg(jsonb_build_object( - 'model', model, - 'api_key', api_key, - 'spend', model_api_spend, - 'total_tokens', model_api_tokens - )) AS metadata - FROM - SpendByModelApiKey - GROUP BY - group_by_day, - team_name - ) AS aggregated + )) AS customers + FROM + ( + SELECT + group_by_day, + customer, + SUM(model_api_spend) AS total_spend, + jsonb_agg(jsonb_build_object( + 'model', model, + 'api_key', api_key, + 'spend', model_api_spend, + 'total_tokens', model_api_tokens + )) AS metadata + FROM + SpendByModelApiKey + GROUP BY + group_by_day, + customer + ) AS aggregated GROUP BY group_by_day ORDER BY group_by_day; - """ + """ - db_response = await prisma_client.db.query_raw( - sql_query, start_date_obj, end_date_obj - ) - if db_response is None: - return [] + db_response = await prisma_client.db.query_raw( + sql_query, start_date_obj, end_date_obj + ) + if db_response is None: + return [] - return db_response + return db_response except Exception as e: raise HTTPException( From 911ab531708a21b09c62c40634dc07894fde565b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Jun 2024 13:44:24 -0700 Subject: [PATCH 15/29] docs - get spend report per customer --- docs/my-website/docs/proxy/cost_tracking.md | 77 ++++++++++++++++++++- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/proxy/cost_tracking.md b/docs/my-website/docs/proxy/cost_tracking.md index de1a63a4c3..b63fab1067 100644 --- a/docs/my-website/docs/proxy/cost_tracking.md +++ b/docs/my-website/docs/proxy/cost_tracking.md @@ -138,14 +138,22 @@ Navigate to the Usage Tab on the LiteLLM UI (found on https://your-proxy-endpoin ## API Endpoints to get Spend -#### Getting Spend Reports - To Charge Other Teams, API Keys +#### Getting Spend Reports - To Charge Other Teams, Customers -Use the `/global/spend/report` endpoint to get daily spend per team, with a breakdown of spend per API Key, Model +Use the `/global/spend/report` endpoint to get daily spend report per +- team +- customer [this is `user` passed to `/chat/completions` request](#how-to-track-spend-with-litellm) + + + + ##### Example Request +👉 Key Change: Specify `group_by=team` + ```shell -curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end_date=2024-06-30' \ +curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end_date=2024-06-30&group_by=team' \ -H 'Authorization: Bearer sk-1234' ``` @@ -254,6 +262,69 @@ Output from script ``` + + + + + + + + + +##### Example Request + +👉 Key Change: Specify `group_by=customer` + + +```shell +curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end_date=2024-06-30&group_by=customer' \ + -H 'Authorization: Bearer sk-1234' +``` + +##### Example Response + + +```shell +[ + { + "group_by_day": "2024-04-30T00:00:00+00:00", + "customers": [ + { + "customer": "palantir", + "total_spend": 0.0015265, + "metadata": [ # see the spend by unique(key + model) + { + "model": "gpt-4", + "spend": 0.00123, + "total_tokens": 28, + "api_key": "88dc28.." # the hashed api key + }, + { + "model": "gpt-4", + "spend": 0.00123, + "total_tokens": 28, + "api_key": "a73dc2.." # the hashed api key + }, + { + "model": "chatgpt-v-2", + "spend": 0.000214, + "total_tokens": 122, + "api_key": "898c28.." # the hashed api key + }, + { + "model": "gpt-3.5-turbo", + "spend": 0.0000825, + "total_tokens": 85, + "api_key": "84dc28.." # the hashed api key + } + ] + } + ] + } +] +``` + + From a9006b965f6290065a3210d8a16e5f0269adc4a5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Jun 2024 14:32:28 -0700 Subject: [PATCH 16/29] fix - support fallbacks as list --- litellm/router.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index bfd1dafe98..c4ea521c5c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2056,12 +2056,15 @@ class Router: verbose_router_logger.debug(f"inside model fallbacks: {fallbacks}") generic_fallback_idx: Optional[int] = None ## check for specific model group-specific fallbacks - for idx, item in enumerate(fallbacks): - if list(item.keys())[0] == model_group: - fallback_model_group = item[model_group] - break - elif list(item.keys())[0] == "*": - generic_fallback_idx = idx + if isinstance(fallbacks, list): + fallback_model_group = fallbacks + else: + for idx, item in enumerate(fallbacks): + if list(item.keys())[0] == model_group: + fallback_model_group = item[model_group] + break + elif list(item.keys())[0] == "*": + generic_fallback_idx = idx ## if none, check for generic fallback if ( fallback_model_group is None From 1656bbf03ee20b5a8b39e2f2337fd3d3eb310011 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Jun 2024 14:33:32 -0700 Subject: [PATCH 17/29] doc - client side fallbacks --- docs/my-website/docs/proxy/reliability.md | 72 +++++++++-------------- 1 file changed, 27 insertions(+), 45 deletions(-) diff --git a/docs/my-website/docs/proxy/reliability.md b/docs/my-website/docs/proxy/reliability.md index e39a6765fc..6d02346ace 100644 --- a/docs/my-website/docs/proxy/reliability.md +++ b/docs/my-website/docs/proxy/reliability.md @@ -2,7 +2,7 @@ import Image from '@theme/IdealImage'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# 🔥 Fallbacks, Retries, Timeouts, Load Balancing +# 🔥 Load Balancing, Fallbacks, Retries, Timeouts Retry call with multiple instances of the same model. @@ -13,7 +13,7 @@ If the error is a context window exceeded error, fall back to a larger model gro [**See Code**](https://github.com/BerriAI/litellm/blob/main/litellm/router.py) ## Quick Start - Load Balancing -### Step 1 - Set deployments on config +#### Step 1 - Set deployments on config **Example config below**. Here requests with `model=gpt-3.5-turbo` will be routed across multiple instances of `azure/gpt-3.5-turbo` ```yaml @@ -38,13 +38,13 @@ model_list: rpm: 1440 ``` -### Step 2: Start Proxy with config +#### Step 2: Start Proxy with config ```shell $ litellm --config /path/to/config.yaml ``` -### Step 3: Use proxy - Call a model group [Load Balancing] +### Test - Load Balancing Curl Command ```shell curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -61,25 +61,44 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ' ``` -### Usage - Call a specific model deployment -If you want to call a specific model defined in the `config.yaml`, you can call the `litellm_params: model` -In this example it will call `azure/gpt-turbo-small-ca`. Defined in the config on Step 1 + +### Test - Client Side Fallbacks ```bash curl --location 'http://0.0.0.0:4000/chat/completions' \ --header 'Content-Type: application/json' \ --data ' { - "model": "azure/gpt-turbo-small-ca", + "model": "zephyr-beta", "messages": [ { "role": "user", "content": "what llm are you" } ], + "fallbacks": [{"zephyr-beta": ["gpt-3.5-turbo"]}], + "context_window_fallbacks": [{"zephyr-beta": ["gpt-3.5-turbo"]}], + "num_retries": 2, + "timeout": 10 } ' ``` + + ## Fallbacks + Retries + Timeouts + Cooldowns @@ -114,43 +133,6 @@ litellm_settings: context_window_fallbacks: [{"zephyr-beta": ["gpt-3.5-turbo-16k"]}, {"gpt-3.5-turbo": ["gpt-3.5-turbo-16k"]}] # fallback to gpt-3.5-turbo-16k if context window error allowed_fails: 3 # cooldown model if it fails > 1 call in a minute. ``` - -**Set dynamically** - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ ---header 'Content-Type: application/json' \ ---data ' { - "model": "zephyr-beta", - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ], - "fallbacks": [{"zephyr-beta": ["gpt-3.5-turbo"]}], - "context_window_fallbacks": [{"zephyr-beta": ["gpt-3.5-turbo"]}], - "num_retries": 2, - "timeout": 10 - } -' -``` - -### Test it! - - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Content-Type: application/json' \ - --data-raw '{ - "model": "zephyr-beta", # 👈 MODEL NAME to fallback from - "messages": [ - {"role": "user", "content": "what color is red"} - ], - "mock_testing_fallbacks": true - }' -``` - ## Advanced - Context Window Fallbacks (Pre-Call Checks + Fallbacks) **Before call is made** check if a call is within model context window with **`enable_pre_call_checks: true`**. From b52802bbfab00343c79c78de05cd13bfee3dcec1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Jun 2024 14:33:42 -0700 Subject: [PATCH 18/29] doc - client side fallbacks --- docs/my-website/docs/proxy/reliability.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/proxy/reliability.md b/docs/my-website/docs/proxy/reliability.md index 6d02346ace..dd383bf0e6 100644 --- a/docs/my-website/docs/proxy/reliability.md +++ b/docs/my-website/docs/proxy/reliability.md @@ -76,7 +76,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ "content": "what llm are you" } ], - "fallbacks": [{"zephyr-beta": ["gpt-3.5-turbo"]}], + "fallbacks": ["gpt-3.5-turbo"]}], "context_window_fallbacks": [{"zephyr-beta": ["gpt-3.5-turbo"]}], "num_retries": 2, "timeout": 10 From 66fd69b7049c0f878b461f2265051b9d63c791a1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Jun 2024 14:49:50 -0700 Subject: [PATCH 19/29] docs - using client side fallbacks --- docs/my-website/docs/proxy/reliability.md | 202 +++++++++++++++++++--- 1 file changed, 178 insertions(+), 24 deletions(-) diff --git a/docs/my-website/docs/proxy/reliability.md b/docs/my-website/docs/proxy/reliability.md index dd383bf0e6..d53e4bb292 100644 --- a/docs/my-website/docs/proxy/reliability.md +++ b/docs/my-website/docs/proxy/reliability.md @@ -45,44 +45,198 @@ $ litellm --config /path/to/config.yaml ``` ### Test - Load Balancing -Curl Command + +Here requests with model=gpt-3.5-turbo will be routed across multiple instances of azure/gpt-3.5-turbo + +👉 Key Change: `model="gpt-3.5-turbo"` + +**Check the `model_id` in Response Headers to make sure the requests are being load balanced** + + + + + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ] +) + +print(response) +``` + + + + +Pass `metadata` as part of the request body + ```shell curl --location 'http://0.0.0.0:4000/chat/completions' \ ---header 'Content-Type: application/json' \ ---data ' { - "model": "gpt-3.5-turbo", - "messages": [ + --header 'Content-Type: application/json' \ + --data '{ + "model": "gpt-3.5-turbo", + "messages": [ { - "role": "user", - "content": "what llm are you" + "role": "user", + "content": "what llm are you" } - ], - } -' + ] +}' ``` + + + +```python +from langchain.chat_models import ChatOpenAI +from langchain.prompts.chat import ( + ChatPromptTemplate, + HumanMessagePromptTemplate, + SystemMessagePromptTemplate, +) +from langchain.schema import HumanMessage, SystemMessage +import os + +os.environ["OPENAI_API_KEY"] = "anything" + +chat = ChatOpenAI( + openai_api_base="http://0.0.0.0:4000", + model="gpt-3.5-turbo", +) + +messages = [ + SystemMessage( + content="You are a helpful assistant that im using to make a test request to." + ), + HumanMessage( + content="test from litellm. tell me why it's amazing in 1 sentence" + ), +] +response = chat(messages) + +print(response) +``` + + + + + + ### Test - Client Side Fallbacks +In this request the following will occur: +1. The request to `model="zephyr-beta"` will fail +2. litellm proxy will loop through all the model_groups specified in `fallbacks=["gpt-3.5-turbo"]` +3. The request to `model="gpt-3.5-turbo"` will succeed and the client making the request will get a response from gpt-3.5-turbo -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ ---header 'Content-Type: application/json' \ ---data ' { - "model": "zephyr-beta", - "messages": [ +👉 Key Change: `"fallbacks": ["gpt-3.5-turbo"]` + + + + + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="zephyr-beta", + messages = [ { - "role": "user", - "content": "what llm are you" + "role": "user", + "content": "this is a test request, write a short poem" + } + ], + extra_body={ + "metadata": { + "fallbacks": ["gpt-3.5-turbo"] } - ], - "fallbacks": ["gpt-3.5-turbo"]}], - "context_window_fallbacks": [{"zephyr-beta": ["gpt-3.5-turbo"]}], - "num_retries": 2, - "timeout": 10 } -' +) + +print(response) ``` + + + + +Pass `metadata` as part of the request body + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "zephyr-beta"", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + "metadata": { + "fallbacks": ["gpt-3.5-turbo"] + } +}' +``` + + + +```python +from langchain.chat_models import ChatOpenAI +from langchain.prompts.chat import ( + ChatPromptTemplate, + HumanMessagePromptTemplate, + SystemMessagePromptTemplate, +) +from langchain.schema import HumanMessage, SystemMessage +import os + +os.environ["OPENAI_API_KEY"] = "anything" + +chat = ChatOpenAI( + openai_api_base="http://0.0.0.0:4000", + model="zephyr-beta", + extra_body={ + "metadata": { + "fallbacks": ["gpt-3.5-turbo"] + } + } +) + +messages = [ + SystemMessage( + content="You are a helpful assistant that im using to make a test request to." + ), + HumanMessage( + content="test from litellm. tell me why it's amazing in 1 sentence" + ), +] +response = chat(messages) + +print(response) +``` + + + + + + + - -## Fallbacks + Retries + Timeouts + Cooldowns +## Advanced +### Fallbacks + Retries + Timeouts + Cooldowns **Set via config** ```yaml @@ -282,7 +279,7 @@ litellm_settings: context_window_fallbacks: [{"zephyr-beta": ["gpt-3.5-turbo-16k"]}, {"gpt-3.5-turbo": ["gpt-3.5-turbo-16k"]}] # fallback to gpt-3.5-turbo-16k if context window error allowed_fails: 3 # cooldown model if it fails > 1 call in a minute. ``` -## Advanced - Context Window Fallbacks (Pre-Call Checks + Fallbacks) +### Context Window Fallbacks (Pre-Call Checks + Fallbacks) **Before call is made** check if a call is within model context window with **`enable_pre_call_checks: true`**. @@ -418,7 +415,7 @@ print(response) -## Advanced - EU-Region Filtering (Pre-Call Checks) +### EU-Region Filtering (Pre-Call Checks) **Before call is made** check if a call is within model context window with **`enable_pre_call_checks: true`**. @@ -481,7 +478,7 @@ print(response) print(f"response.headers.get('x-litellm-model-api-base')") ``` -## Advanced - Custom Timeouts, Stream Timeouts - Per Model +### Custom Timeouts, Stream Timeouts - Per Model For each model you can set `timeout` & `stream_timeout` under `litellm_params` ```yaml model_list: @@ -510,7 +507,7 @@ $ litellm --config /path/to/config.yaml ``` -## Advanced - Setting Dynamic Timeouts - Per Request +### Setting Dynamic Timeouts - Per Request LiteLLM Proxy supports setting a `timeout` per request From fcd1d077f548ec75f4274f29187cb5f036166196 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Jun 2024 15:26:09 -0700 Subject: [PATCH 23/29] doc fix llm cord --- .../{llmcord.py (Discord LLM Chatbot).md => llm_cord.md} | 2 ++ docs/my-website/sidebars.js | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) rename docs/my-website/docs/projects/{llmcord.py (Discord LLM Chatbot).md => llm_cord.md} (93%) diff --git a/docs/my-website/docs/projects/llmcord.py (Discord LLM Chatbot).md b/docs/my-website/docs/projects/llm_cord.md similarity index 93% rename from docs/my-website/docs/projects/llmcord.py (Discord LLM Chatbot).md rename to docs/my-website/docs/projects/llm_cord.md index f8acb9383c..6a28d5c884 100644 --- a/docs/my-website/docs/projects/llmcord.py (Discord LLM Chatbot).md +++ b/docs/my-website/docs/projects/llm_cord.md @@ -1,3 +1,5 @@ +# llmcord.py + llmcord.py lets you and your friends chat with LLMs directly in your Discord server. It works with practically any LLM, remote or locally hosted. Github: https://github.com/jakobdylanc/discord-llm-chatbot diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 50cfaed2ba..ff110bb626 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -255,7 +255,7 @@ const sidebars = { "projects/GPT Migrate", "projects/YiVal", "projects/LiteLLM Proxy", - "projects/llmcord.py Discord LLM Chatbot", + "projects/llm_cord", ], }, ], From 3347a58e11e1865454dca79fd9dd9502d3cfc340 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Jun 2024 15:48:36 -0700 Subject: [PATCH 24/29] ui - chat ui fix --- .../src/components/chat_ui.tsx | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/chat_ui.tsx b/ui/litellm-dashboard/src/components/chat_ui.tsx index 407e33dca3..d96db60c48 100644 --- a/ui/litellm-dashboard/src/components/chat_ui.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui.tsx @@ -119,9 +119,24 @@ const ChatUI: React.FC = ({ // Now, 'options' contains the list you wanted console.log(options); // You can log it to verify the list - - // setModelInfo(options) should be inside the if block to avoid setting it when no data is available - setModelInfo(options); + + // if options.length > 0, only store unique values + if (options.length > 0) { + const uniqueModels = Array.from(new Set(options)); + + console.log("Unique models:", uniqueModels); + + // sort uniqueModels alphabetically + uniqueModels.sort((a: any, b: any) => a.label.localeCompare(b.label)); + + + console.log("Model info:", modelInfo); + + // setModelInfo(options) should be inside the if block to avoid setting it when no data is available + setModelInfo(uniqueModels); + } + + setSelectedModel(fetchedAvailableModels.data[0].id); } } catch (error) { From 8b30d3f4e976557090ca8962df36dc71a259ea52 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 10 Jun 2024 17:26:31 -0700 Subject: [PATCH 25/29] Fix typos: Enterpise -> Enterprise --- ui/litellm-dashboard/src/components/model_dashboard.tsx | 4 ++-- ui/litellm-dashboard/src/components/navbar.tsx | 2 +- ui/litellm-dashboard/src/components/usage.tsx | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 73e5a7a8f1..d16d8db135 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -1130,7 +1130,7 @@ const ModelDashboard: React.FC = ({ setSelectedAPIKey(key); }} > - ✨ {key["key_alias"]} (Enterpise only Feature) + ✨ {key["key_alias"]} (Enterprise only Feature) ); } @@ -1165,7 +1165,7 @@ const ModelDashboard: React.FC = ({ setSelectedCustomer(user); }} > - ✨ {user} (Enterpise only Feature) + ✨ {user} (Enterprise only Feature) ); }) diff --git a/ui/litellm-dashboard/src/components/navbar.tsx b/ui/litellm-dashboard/src/components/navbar.tsx index 4f587afe9f..6f33d16912 100644 --- a/ui/litellm-dashboard/src/components/navbar.tsx +++ b/ui/litellm-dashboard/src/components/navbar.tsx @@ -114,7 +114,7 @@ const Navbar: React.FC = ({ textDecoration: "underline", }} > - Get enterpise license + Get enterprise license ) : null} diff --git a/ui/litellm-dashboard/src/components/usage.tsx b/ui/litellm-dashboard/src/components/usage.tsx index 732df45245..ad1aa0e574 100644 --- a/ui/litellm-dashboard/src/components/usage.tsx +++ b/ui/litellm-dashboard/src/components/usage.tsx @@ -832,7 +832,7 @@ const UsagePage: React.FC = ({ // @ts-ignore disabled={true} > - ✨ {tag} (Enterpise only Feature) + ✨ {tag} (Enterprise only Feature) ); })} From b2c9b5bbf4bca6640b82b97dddffbb90422ee47f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 10 Jun 2024 17:33:43 -0700 Subject: [PATCH 26/29] feat(proxy/_logging.py): support controlling log level when json logs=True allow user to control log level when json_logs=True --- litellm/proxy/_logging.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/_logging.py b/litellm/proxy/_logging.py index f453cef395..655da7b294 100644 --- a/litellm/proxy/_logging.py +++ b/litellm/proxy/_logging.py @@ -1,7 +1,12 @@ import json import logging from logging import Formatter -import sys +import os +from litellm import json_logs + +# Set default log level to INFO +log_level = os.getenv("LITELLM_LOG", "INFO") +numeric_level: str = getattr(logging, log_level.upper()) class JsonFormatter(Formatter): @@ -16,6 +21,14 @@ class JsonFormatter(Formatter): logger = logging.root handler = logging.StreamHandler() -handler.setFormatter(JsonFormatter()) +if json_logs: + handler.setFormatter(JsonFormatter()) +else: + formatter = logging.Formatter( + "\033[92m%(asctime)s - %(name)s:%(levelname)s\033[0m: %(filename)s:%(lineno)s - %(message)s", + datefmt="%H:%M:%S", + ) + + handler.setFormatter(formatter) logger.handlers = [handler] -logger.setLevel(logging.INFO) +logger.setLevel(numeric_level) From 28fa8d294a83e380031e09b65a68d02859588bca Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 10 Jun 2024 17:35:02 -0700 Subject: [PATCH 27/29] assistants.md: Remove extra trailing backslash that prevents `curl` command from working. --- docs/my-website/docs/assistants.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/assistants.md b/docs/my-website/docs/assistants.md index 2380fe5c64..65d4ce1181 100644 --- a/docs/my-website/docs/assistants.md +++ b/docs/my-website/docs/assistants.md @@ -150,7 +150,7 @@ $ litellm --config /path/to/config.yaml ```bash curl "http://0.0.0.0:4000/v1/assistants?order=desc&limit=20" \ -H "Content-Type: application/json" \ - -H "Authorization: Bearer sk-1234" \ + -H "Authorization: Bearer sk-1234" ``` **Create a Thread** From 3a01a485f701db3ca7355a59b069b9f672f82481 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 10 Jun 2024 17:39:33 -0700 Subject: [PATCH 28/29] docs(debugging.md): add controlling log level to docs --- docs/my-website/docs/proxy/debugging.md | 41 ++++++++++++++++++++++++- docs/my-website/docs/proxy/prod.md | 6 ++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/proxy/debugging.md b/docs/my-website/docs/proxy/debugging.md index b9f2ba8da9..571a97c0ec 100644 --- a/docs/my-website/docs/proxy/debugging.md +++ b/docs/my-website/docs/proxy/debugging.md @@ -42,6 +42,14 @@ Set `JSON_LOGS="True"` in your env: ```bash export JSON_LOGS="True" ``` +**OR** + +Set `json_logs: true` in your yaml: + +```yaml +litellm_settings: + json_logs: true +``` Start proxy @@ -49,4 +57,35 @@ Start proxy $ litellm ``` -The proxy will now all logs in json format. \ No newline at end of file +The proxy will now all logs in json format. + +## Control Log Output + +Turn off fastapi's default 'INFO' logs + +1. Turn on 'json logs' +```yaml +litellm_settings: + json_logs: true +``` + +2. Set `LITELLM_LOG` to 'ERROR' + +Only get logs if an error occurs. + +```bash +LITELLM_LOG="ERROR" +``` + +3. Start proxy + + +```bash +$ litellm +``` + +Expected Output: + +```bash +# no info statements +``` \ No newline at end of file diff --git a/docs/my-website/docs/proxy/prod.md b/docs/my-website/docs/proxy/prod.md index 35c8c575ba..587164fe6c 100644 --- a/docs/my-website/docs/proxy/prod.md +++ b/docs/my-website/docs/proxy/prod.md @@ -21,6 +21,7 @@ general_settings: litellm_settings: set_verbose: False # Switch off Debug Logging, ensure your logs do not have any debugging on + json_logs: true # Get debug logs in json format ``` Set slack webhook url in your env @@ -28,6 +29,11 @@ Set slack webhook url in your env export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/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" ``` +Turn off FASTAPI's default info logs +```bash +export LITELLM_LOG="ERROR" +``` + :::info Need Help or want dedicated support ? Talk to a founder [here]: (https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) From 717163780de6f81f749f0addf7a51b282e305633 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 10 Jun 2024 17:56:26 -0700 Subject: [PATCH 29/29] assistants.md: Add "Get a Thread" example --- docs/my-website/docs/assistants.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/my-website/docs/assistants.md b/docs/my-website/docs/assistants.md index 2380fe5c64..4953ee38db 100644 --- a/docs/my-website/docs/assistants.md +++ b/docs/my-website/docs/assistants.md @@ -162,6 +162,14 @@ curl http://0.0.0.0:4000/v1/threads \ -d '' ``` +**Get a Thread** + +```bash +curl http://0.0.0.0:4000/v1/threads/{thread_id} \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" +``` + **Add Messages to the Thread** ```bash