From fcf4ea3608765b2591da346cce769f179c547b1b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 26 Feb 2025 09:39:27 -0800 Subject: [PATCH] build: merge squashed commit Squashed commit of the following: commit 6678e15381389e976a7554fcdf5913035cf66ece Author: Ishaan Jaff Date: Wed Feb 26 09:29:15 2025 -0800 test_prompt_caching commit bd86e0ac47d1f7980d11e8522b9dbf1535c9f42f Author: Ishaan Jaff Date: Wed Feb 26 08:57:16 2025 -0800 test_prompt_caching commit 2fc21ad51ede374cc8d46296e4b808981c54f712 Author: Ishaan Jaff Date: Wed Feb 26 08:13:45 2025 -0800 test_aprompt_caching commit d94cff55ff686f1ae82f452f33c6a041d785dcb3 Author: Ishaan Jaff Date: Wed Feb 26 08:13:12 2025 -0800 test_prompt_caching commit 49c5e7811e55d06ca875ca163786fe0123e7ddf6 Author: Ishaan Jaff Date: Wed Feb 26 07:43:53 2025 -0800 ui new build commit cb8d5e59174e2000b7cfd5574c1843d273e5e238 Author: Ishaan Jaff Date: Wed Feb 26 07:38:56 2025 -0800 (UI) - Create Key flow for existing users (#8844) * working create user button * working create user for a key flow * allow searching users * working create user + key * use clear sections on create key * better search for users * fix create key * ui fix create key button - make it neater / cleaner * ui fix all keys table commit 335ba304674d5e15090adbb68d70efe5680ceaff Author: Krrish Dholakia Date: Wed Feb 26 08:53:17 2025 -0800 fix: fix file name commit b8c5b31a4e51a1a33665e9ad248e469a5e49780c Author: Krrish Dholakia Date: Tue Feb 25 22:54:46 2025 -0800 fix: fix utils commit ac6e503461b038d85ef1cfa29ae15e29d6b189c1 Author: Krrish Dholakia Date: Mon Feb 24 10:43:31 2025 -0800 fix(main.py): fix openai message for assistant msg if role is missing - openai allows this Fixes https://github.com/BerriAI/litellm/issues/8661 commit de3989dbc5ad3141b76a57f3cc2dc4b7630cf76f Author: Krrish Dholakia Date: Mon Feb 24 21:19:25 2025 -0800 fix(get_litellm_params.py): handle no-log being passed in via kwargs Fixes https://github.com/BerriAI/litellm/issues/8380 --- .../litellm_core_utils/get_litellm_params.py | 2 +- litellm/litellm_core_utils/litellm_logging.py | 12 +- litellm/main.py | 4 +- .../proxy/_experimental/out/onboarding.html | 1 - litellm/proxy/_new_secret_config.yaml | 21 +++- litellm/utils.py | 25 +++- .../{test_main.py => test_rerank_main.py} | 0 tests/litellm/test_main.py | 119 ++++++++++++++++++ 8 files changed, 172 insertions(+), 12 deletions(-) delete mode 100644 litellm/proxy/_experimental/out/onboarding.html rename tests/litellm/rerank_api/{test_main.py => test_rerank_main.py} (100%) create mode 100644 tests/litellm/test_main.py diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 3d8394f7af..c0fbb1cb97 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -75,7 +75,7 @@ def get_litellm_params( "model_info": model_info, "proxy_server_request": proxy_server_request, "preset_cache_key": preset_cache_key, - "no-log": no_log, + "no-log": no_log or kwargs.get("no-log"), "stream_response": {}, # litellm_call_id: ModelResponse Dict "input_cost_per_token": input_cost_per_token, "input_cost_per_second": input_cost_per_second, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index d44fb07637..4d082e29ad 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3,7 +3,6 @@ # Logging function -> log the exact model details + what's being sent | Non-Blocking import copy import datetime -from functools import lru_cache import json import os import re @@ -13,6 +12,7 @@ import time import traceback import uuid from datetime import datetime as dt_object +from functools import lru_cache from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union, cast from pydantic import BaseModel @@ -2513,15 +2513,19 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 # auth can be disabled on local deployments of arize phoenix if arize_phoenix_config.otlp_auth_headers is not None: - os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = arize_phoenix_config.otlp_auth_headers - + os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = ( + arize_phoenix_config.otlp_auth_headers + ) + for callback in _in_memory_loggers: if ( isinstance(callback, OpenTelemetry) and callback.callback_name == "arize_phoenix" ): return callback # type: ignore - _otel_logger = OpenTelemetry(config=otel_config, callback_name="arize_phoenix") + _otel_logger = OpenTelemetry( + config=otel_config, callback_name="arize_phoenix" + ) _in_memory_loggers.append(_otel_logger) return _otel_logger # type: ignore elif logging_integration == "otel": diff --git a/litellm/main.py b/litellm/main.py index ece484f1f2..c52bfd7c92 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -94,7 +94,7 @@ from litellm.utils import ( read_config_args, supports_httpx_timeout, token_counter, - validate_chat_completion_messages, + validate_and_fix_openai_messages, validate_chat_completion_tool_choice, ) @@ -851,7 +851,7 @@ def completion( # type: ignore # noqa: PLR0915 if model is None: raise ValueError("model param not passed in.") # validate messages - messages = validate_chat_completion_messages(messages=messages) + messages = validate_and_fix_openai_messages(messages=messages) # validate tool_choice tool_choice = validate_chat_completion_tool_choice(tool_choice=tool_choice) ######### unpacking kwargs ##################### diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding.html deleted file mode 100644 index c3d130695e..0000000000 --- a/litellm/proxy/_experimental/out/onboarding.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 0145043a33..abf2e07c88 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,5 +1,24 @@ model_list: - model_name: claude-3.5 litellm_params: +<<<<<<< HEAD model: claude-3-5-sonnet-latest - api_key: os.environ/ANTHROPIC_API_KEY \ No newline at end of file + api_key: os.environ/ANTHROPIC_API_KEY +======= + model: openai/gpt-3.5-turbo + api_key: os.environ/OPENAI_API_KEY + api_base: http://0.0.0.0:8090 + - model_name: deepseek-r1 + litellm_params: + model: bedrock/deepseek_r1/arn:aws:bedrock:us-west-2:888602223428:imported-model/bnnr6463ejgf + - model_name: deepseek-r1-api + litellm_params: + model: deepseek/deepseek-reasoner + - model_name: cohere.embed-english-v3 + litellm_params: + model: bedrock/cohere.embed-english-v3 + api_key: os.environ/COHERE_API_KEY + +litellm_settings: + callbacks: ["langfuse"] +>>>>>>> f86a609ea (fix(get_litellm_params.py): handle no-log being passed in via kwargs) diff --git a/litellm/utils.py b/litellm/utils.py index facc2ac59b..8f19e74509 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5932,6 +5932,18 @@ def convert_to_dict(message: Union[BaseModel, dict]) -> dict: ) +def validate_and_fix_openai_messages(messages: List): + """ + Ensures all messages are valid OpenAI chat completion messages. + + Handles missing role for assistant messages. + """ + for message in messages: + if not message.get("role"): + message["role"] = "assistant" + return validate_chat_completion_messages(messages=messages) + + def validate_chat_completion_messages(messages: List[AllMessageValues]): """ Ensures all messages are valid OpenAI chat completion messages. @@ -6282,11 +6294,18 @@ def get_end_user_id_for_cost_tracking( return None return end_user_id -def should_use_cohere_v1_client(api_base: Optional[str], present_version_params: List[str]): + +def should_use_cohere_v1_client( + api_base: Optional[str], present_version_params: List[str] +): if not api_base: return False - uses_v1_params = ("max_chunks_per_doc" in present_version_params) and ('max_tokens_per_doc' not in present_version_params) - return api_base.endswith("/v1/rerank") or (uses_v1_params and not api_base.endswith("/v2/rerank")) + uses_v1_params = ("max_chunks_per_doc" in present_version_params) and ( + "max_tokens_per_doc" not in present_version_params + ) + return api_base.endswith("/v1/rerank") or ( + uses_v1_params and not api_base.endswith("/v2/rerank") + ) def is_prompt_caching_valid_prompt( diff --git a/tests/litellm/rerank_api/test_main.py b/tests/litellm/rerank_api/test_rerank_main.py similarity index 100% rename from tests/litellm/rerank_api/test_main.py rename to tests/litellm/rerank_api/test_rerank_main.py diff --git a/tests/litellm/test_main.py b/tests/litellm/test_main.py new file mode 100644 index 0000000000..b838434915 --- /dev/null +++ b/tests/litellm/test_main.py @@ -0,0 +1,119 @@ +import json +import os +import sys + +import pytest +from fastapi.testclient import TestClient + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path + +from unittest.mock import MagicMock, patch + +import litellm + + +@pytest.fixture +def openai_api_response(): + mock_response_data = { + "id": "chatcmpl-B0W3vmiM78Xkgx7kI7dr7PC949DMS", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": None, + "message": { + "content": "", + "refusal": None, + "role": "assistant", + "audio": None, + "function_call": None, + "tool_calls": None, + }, + } + ], + "created": 1739462947, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "service_tier": "default", + "system_fingerprint": "fp_bd83329f63", + "usage": { + "completion_tokens": 1, + "prompt_tokens": 121, + "total_tokens": 122, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, + }, + } + + return mock_response_data + + +def test_completion_missing_role(openai_api_response): + from openai import OpenAI + + from litellm.types.utils import ModelResponse + + client = OpenAI(api_key="test_api_key") + + mock_raw_response = MagicMock() + mock_raw_response.headers = { + "x-request-id": "123", + "openai-organization": "org-123", + "x-ratelimit-limit-requests": "100", + "x-ratelimit-remaining-requests": "99", + } + mock_raw_response.parse.return_value = ModelResponse(**openai_api_response) + + print(f"openai_api_response: {openai_api_response}") + + with patch.object( + client.chat.completions.with_raw_response, "create", mock_raw_response + ) as mock_create: + litellm.completion( + model="gpt-4o-mini", + messages=[ + {"role": "user", "content": "Hey"}, + { + "content": "", + "tool_calls": [ + { + "id": "call_m0vFJjQmTH1McvaHBPR2YFwY", + "function": { + "arguments": '{"input": "dksjsdkjdhskdjshdskhjkhlk"}', + "name": "tool_name", + }, + "type": "function", + "index": 0, + }, + { + "id": "call_Vw6RaqV2n5aaANXEdp5pYxo2", + "function": { + "arguments": '{"input": "jkljlkjlkjlkjlk"}', + "name": "tool_name", + }, + "type": "function", + "index": 1, + }, + { + "id": "call_hBIKwldUEGlNh6NlSXil62K4", + "function": { + "arguments": '{"input": "jkjlkjlkjlkj;lj"}', + "name": "tool_name", + }, + "type": "function", + "index": 2, + }, + ], + }, + ], + client=client, + ) + + mock_create.assert_called_once()