ai-workspace-services/litellm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/BerriAI/litellm into feat/github-copilot-thinking-reasoning-support

Browse Source
This commit is contained in:
Tim Elfrink 2025-08-20 07:58:11 +02:00
commit 972d7f7133
26 changed files with 1510 additions and 1269 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Dockerfile
View File

@ -65,8 +65,8 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
# Install semantic_router without dependencies
RUN pip install semantic_router --no-deps
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Generate prisma client
RUN prisma generate

4
docker/Dockerfile.database
View File

@ -57,8 +57,8 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
# Install semantic_router without dependencies
RUN pip install semantic_router --no-deps
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# ensure pyjwt is used, not jwt
RUN pip uninstall jwt -y

4
docker/Dockerfile.non_root
View File

@ -47,8 +47,8 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ \
&& rm -f *.whl \
&& rm -rf /wheels
# Install semantic_router without dependencies
RUN pip install semantic_router --no-deps
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Ensure correct JWT library is used (pyjwt not jwt)
RUN pip uninstall jwt -y && \

3
docker/install_auto_router.sh Executable file
View File

@ -0,0 +1,3 @@
#!/bin/bash
pip install semantic_router==0.1.11 --no-deps
pip install aurelio-sdk==0.0.19

1
litellm/integrations/datadog/datadog_llm_obs.py
View File

@ -422,6 +422,7 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
"cache_hit": standard_logging_payload.get("cache_hit", "unknown"),
"cache_key": standard_logging_payload.get("cache_key", "unknown"),
"saved_cache_cost": standard_logging_payload.get("saved_cache_cost", 0),
"guardrail_information": standard_logging_payload.get("guardrail_information", None),
}
#########################################################

44
litellm/litellm_core_utils/llm_cost_calc/utils.py
View File

@ -113,15 +113,20 @@ def _generic_cost_per_character(
return prompt_cost, completion_cost
def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, float]:
def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, float, float, float]:
"""
Return prompt cost for a given model and usage.
Return prompt cost, completion cost, and cache costs for a given model and usage.
If input_tokens > threshold and `input_cost_per_token_above_[x]k_tokens` or `input_cost_per_token_above_[x]_tokens` is set,
then we use the corresponding threshold cost.
then we use the corresponding threshold cost for all token types.
Returns:
Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
"""
prompt_base_cost = cast(float, _get_cost_per_unit(model_info, "input_cost_per_token"))
completion_base_cost = cast(float, _get_cost_per_unit(model_info, "output_cost_per_token"))
cache_creation_cost = cast(float, _get_cost_per_unit(model_info, "cache_creation_input_token_cost"))
cache_read_cost = cast(float, _get_cost_per_unit(model_info, "cache_read_input_token_cost"))
## CHECK IF ABOVE THRESHOLD
threshold: Optional[float] = None
@ -141,13 +146,28 @@ def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, fl
f"output_cost_per_token_above_{threshold_str}_tokens",
completion_base_cost,
))
# Apply tiered pricing to cache costs
cache_creation_tiered_key = f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
cache_read_tiered_key = f"cache_read_input_token_cost_above_{threshold_str}_tokens"
if cache_creation_tiered_key in model_info:
cache_creation_cost = cast(float, _get_cost_per_unit(
model_info, cache_creation_tiered_key, cache_creation_cost
))
if cache_read_tiered_key in model_info:
cache_read_cost = cast(float, _get_cost_per_unit(
model_info, cache_read_tiered_key, cache_read_cost
))
break
except (IndexError, ValueError):
continue
except Exception:
continue
return prompt_base_cost, completion_base_cost
return prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost
def calculate_cost_component(
@ -262,28 +282,22 @@ def generic_cost_per_token(
if text_tokens == 0:
text_tokens = usage.prompt_tokens - cache_hit_tokens - audio_tokens
prompt_base_cost, completion_base_cost = _get_token_base_cost(
prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost = _get_token_base_cost(
model_info=model_info, usage=usage
)
prompt_cost = float(text_tokens) * prompt_base_cost
### CACHE READ COST
prompt_cost += calculate_cost_component(
model_info, "cache_read_input_token_cost", cache_hit_tokens
)
### CACHE READ COST - Now uses tiered pricing
prompt_cost += float(cache_hit_tokens) * cache_read_cost
### AUDIO COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_audio_token", audio_tokens
)
### CACHE WRITING COST
prompt_cost += calculate_cost_component(
model_info,
"cache_creation_input_token_cost",
usage._cache_creation_input_tokens,
)
### CACHE WRITING COST - Now uses tiered pricing
prompt_cost += float(usage._cache_creation_input_tokens or 0) * cache_creation_cost
### CHARACTER COST

3
litellm/llms/azure/chat/gpt_5_transformation.py
View File

@ -10,6 +10,7 @@ from .gpt_transformation import AzureOpenAIConfig
class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
"""Azure specific handling for gpt-5 models."""
GPT5_SERIES_ROUTE = "gpt5_series/"
@classmethod
@ -23,7 +24,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
def get_supported_openai_params(self, model: str) -> List[str]:
return OpenAIGPT5Config.get_supported_openai_params(self, model=model)
def map_openai_params(
self,
non_default_params: dict,

8
litellm/llms/openai/chat/gpt_5_transformation.py
View File

@ -15,14 +15,19 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
- Mapping ``max_tokens`` -> ``max_completion_tokens``.
- Dropping unsupported ``temperature`` values when requested.
"""
@classmethod
def is_model_gpt_5_model(cls, model: str) -> bool:
return "gpt-5" in model
def get_supported_openai_params(self, model: str) -> list:
from litellm.utils import supports_tool_choice
base_gpt_series_params = super().get_supported_openai_params(model=model)
gpt_5_only_params = ["reasoning_effort"]
base_gpt_series_params.extend(gpt_5_only_params)
if not supports_tool_choice(model=model):
base_gpt_series_params.remove("tool_choice")
return base_gpt_series_params
def map_openai_params(
@ -61,4 +66,3 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
model=model,
drop_params=drop_params,
)

106
litellm/model_prices_and_context_window_backup.json
View File

@ -2457,7 +2457,7 @@
},
"azure/gpt-5-chat": {
"max_tokens": 128000,
"max_input_tokens": 400000,
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"input_cost_per_token": 1.25e-06,
"output_cost_per_token": 1e-05,
@ -2483,14 +2483,14 @@
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_tool_choice": false,
"supports_native_streaming": true,
"supports_reasoning": true,
"source": "https://azure.microsoft.com/en-us/blog/gpt-5-in-azure-ai-foundry-the-future-of-ai-apps-and-agents-starts-here/"
},
"azure/gpt-5-chat-latest": {
"max_tokens": 128000,
"max_input_tokens": 400000,
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"input_cost_per_token": 1.25e-06,
"output_cost_per_token": 1e-05,
@ -2516,7 +2516,7 @@
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_tool_choice": false,
"supports_native_streaming": true,
"supports_reasoning": true
},
@ -6437,11 +6437,13 @@
"supports_computer_use": true
},
"claude-4-sonnet-20250514": {
"max_tokens": 64000,
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 1000000,
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
"input_cost_per_token_above_200k_tokens": 6e-06,
"output_cost_per_token_above_200k_tokens": 2.25e-05,
"search_context_cost_per_query": {
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01,
@ -6449,6 +6451,8 @@
},
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "anthropic",
"mode": "chat",
"supports_function_calling": true,
@ -14649,6 +14653,50 @@
"mode": "chat",
"supports_tool_choice": true
},
"together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": {
"input_cost_per_token": 2e-07,
"output_cost_per_token": 6e-06,
"max_input_tokens": 262000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"mode": "chat",
"supports_tool_choice": false,
"source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8"
},
"together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": {
"input_cost_per_token": 2e-06,
"output_cost_per_token": 2e-06,
"max_input_tokens": 256000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"mode": "chat",
"supports_tool_choice": false,
"source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct"
},
"together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": {
"input_cost_per_token": 6.5e-07,
"output_cost_per_token": 3e-06,
"max_input_tokens": 256000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"mode": "chat",
"supports_tool_choice": false,
"source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507"
},
"together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": {
"input_cost_per_token": 2e-07,
"output_cost_per_token": 6e-07,
"max_input_tokens": 40000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"mode": "chat",
"supports_tool_choice": false,
"source": "https://www.together.ai/models/qwen3-235b-a22b-fp8-tput"
},
"together_ai/deepseek-ai/DeepSeek-V3": {
"input_cost_per_token": 1.25e-06,
"output_cost_per_token": 1.25e-06,
@ -14673,6 +14721,17 @@
"mode": "chat",
"supports_tool_choice": true
},
"together_ai/deepseek-ai/DeepSeek-R1-0528-tput": {
"input_cost_per_token": 5.5e-07,
"output_cost_per_token": 2.19e-06,
"max_input_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"mode": "chat",
"supports_tool_choice": false,
"source": "https://www.together.ai/models/deepseek-r1-0528-throughput"
},
"together_ai/mistralai/Mistral-Small-24B-Instruct-2501": {
"litellm_provider": "together_ai",
"supports_function_calling": true,
@ -14690,6 +14749,39 @@
"mode": "chat",
"source": "https://www.together.ai/models/kimi-k2-instruct"
},
"together_ai/openai/gpt-oss-120b": {
"input_cost_per_token": 1.5e-07,
"output_cost_per_token": 6e-07,
"max_input_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_tool_choice": false,
"supports_parallel_function_calling": false,
"mode": "chat",
"source": "https://www.together.ai/models/gpt-oss-120b"
},
"together_ai/OpenAI/gpt-oss-20B": {
"input_cost_per_token": 5e-08,
"output_cost_per_token": 2e-07,
"max_input_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_tool_choice": false,
"supports_parallel_function_calling": false,
"mode": "chat",
"source": "https://www.together.ai/models/gpt-oss-20b"
},
"together_ai/zai-org/GLM-4.5-Air-FP8": {
"input_cost_per_token": 2e-07,
"output_cost_per_token": 1.1e-06,
"max_input_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_tool_choice": false,
"supports_parallel_function_calling": false,
"mode": "chat",
"source": "https://www.together.ai/models/glm-4-5-air"
},
"ollama/codegemma": {
"max_tokens": 8192,
"max_input_tokens": 8192,

11
litellm/proxy/guardrails/guardrail_hooks/aim/aim.py
View File

@ -118,7 +118,7 @@ class AimGuardrail(CustomGuardrail):
litellm_call_id=call_id,
)
response = await self.async_handler.post(
f"{self.api_base}/detect/openai/v2",
f"{self.api_base}/fw/v1/analyze",
headers=headers,
json={"messages": data.get("messages", [])},
)
@ -183,14 +183,17 @@ class AimGuardrail(CustomGuardrail):
)
call_id = request_data.get("litellm_call_id")
response = await self.async_handler.post(
f"{self.api_base}/detect/output/v2",
f"{self.api_base}/fw/v1/analyze",
headers=self._build_aim_headers(
hook=hook,
key_alias=key_alias,
user_email=user_email,
litellm_call_id=call_id,
),
json={"output": output, "messages": request_data.get("messages", [])},
json={
"messages": request_data.get("messages", [])
+ [{"role": "assistant", "content": output}]
},
)
response.raise_for_status()
res = response.json()
@ -297,7 +300,7 @@ class AimGuardrail(CustomGuardrail):
)
call_id = request_data.get("litellm_call_id")
async with connect(
f"{self.ws_api_base}/detect/output/ws",
f"{self.ws_api_base}/fw/v1/analyze/stream",
additional_headers=self._build_aim_headers(
hook="output",
key_alias=user_api_key_dict.key_alias,

16
litellm/vector_stores/main.py
View File

@ -358,6 +358,22 @@ def search(
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
_is_async = kwargs.pop("asearch", False) is True
# pull credentials from registry if available
vector_store_id_for_credentials = kwargs.get("vector_store_id", vector_store_id)
if (
litellm.vector_store_registry is not None
and vector_store_id_for_credentials is not None
):
try:
registry_credentials = (
litellm.vector_store_registry.get_credentials_for_vector_store(
vector_store_id_for_credentials
)
)
kwargs.update(registry_credentials)
except Exception:
pass
# get llm provider logic
litellm_params = GenericLiteLLMParams(**kwargs)

106
model_prices_and_context_window.json
View File

@ -2457,7 +2457,7 @@
},
"azure/gpt-5-chat": {
"max_tokens": 128000,
"max_input_tokens": 400000,
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"input_cost_per_token": 1.25e-06,
"output_cost_per_token": 1e-05,
@ -2483,14 +2483,14 @@
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_tool_choice": false,
"supports_native_streaming": true,
"supports_reasoning": true,
"source": "https://azure.microsoft.com/en-us/blog/gpt-5-in-azure-ai-foundry-the-future-of-ai-apps-and-agents-starts-here/"
},
"azure/gpt-5-chat-latest": {
"max_tokens": 128000,
"max_input_tokens": 400000,
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"input_cost_per_token": 1.25e-06,
"output_cost_per_token": 1e-05,
@ -2516,7 +2516,7 @@
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_tool_choice": false,
"supports_native_streaming": true,
"supports_reasoning": true
},
@ -6437,11 +6437,13 @@
"supports_computer_use": true
},
"claude-4-sonnet-20250514": {
"max_tokens": 64000,
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 1000000,
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
"input_cost_per_token_above_200k_tokens": 6e-06,
"output_cost_per_token_above_200k_tokens": 2.25e-05,
"search_context_cost_per_query": {
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01,
@ -6449,6 +6451,8 @@
},
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "anthropic",
"mode": "chat",
"supports_function_calling": true,
@ -14649,6 +14653,50 @@
"mode": "chat",
"supports_tool_choice": true
},
"together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": {
"input_cost_per_token": 2e-07,
"output_cost_per_token": 6e-06,
"max_input_tokens": 262000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"mode": "chat",
"supports_tool_choice": false,
"source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8"
},
"together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": {
"input_cost_per_token": 2e-06,
"output_cost_per_token": 2e-06,
"max_input_tokens": 256000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"mode": "chat",
"supports_tool_choice": false,
"source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct"
},
"together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": {
"input_cost_per_token": 6.5e-07,
"output_cost_per_token": 3e-06,
"max_input_tokens": 256000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"mode": "chat",
"supports_tool_choice": false,
"source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507"
},
"together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": {
"input_cost_per_token": 2e-07,
"output_cost_per_token": 6e-07,
"max_input_tokens": 40000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"mode": "chat",
"supports_tool_choice": false,
"source": "https://www.together.ai/models/qwen3-235b-a22b-fp8-tput"
},
"together_ai/deepseek-ai/DeepSeek-V3": {
"input_cost_per_token": 1.25e-06,
"output_cost_per_token": 1.25e-06,
@ -14673,6 +14721,17 @@
"mode": "chat",
"supports_tool_choice": true
},
"together_ai/deepseek-ai/DeepSeek-R1-0528-tput": {
"input_cost_per_token": 5.5e-07,
"output_cost_per_token": 2.19e-06,
"max_input_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"mode": "chat",
"supports_tool_choice": false,
"source": "https://www.together.ai/models/deepseek-r1-0528-throughput"
},
"together_ai/mistralai/Mistral-Small-24B-Instruct-2501": {
"litellm_provider": "together_ai",
"supports_function_calling": true,
@ -14690,6 +14749,39 @@
"mode": "chat",
"source": "https://www.together.ai/models/kimi-k2-instruct"
},
"together_ai/openai/gpt-oss-120b": {
"input_cost_per_token": 1.5e-07,
"output_cost_per_token": 6e-07,
"max_input_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_tool_choice": false,
"supports_parallel_function_calling": false,
"mode": "chat",
"source": "https://www.together.ai/models/gpt-oss-120b"
},
"together_ai/OpenAI/gpt-oss-20B": {
"input_cost_per_token": 5e-08,
"output_cost_per_token": 2e-07,
"max_input_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_tool_choice": false,
"supports_parallel_function_calling": false,
"mode": "chat",
"source": "https://www.together.ai/models/gpt-oss-20b"
},
"together_ai/zai-org/GLM-4.5-Air-FP8": {
"input_cost_per_token": 2e-07,
"output_cost_per_token": 1.1e-06,
"max_input_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_tool_choice": false,
"supports_parallel_function_calling": false,
"mode": "chat",
"source": "https://www.together.ai/models/glm-4-5-air"
},
"ollama/codegemma": {
"max_tokens": 8192,
"max_input_tokens": 8192,

7
tests/local_testing/test_aim_guardrails.py
View File

@ -216,12 +216,13 @@ async def test_post_call__with_anonymized_entities__it_deanonymizes_output():
) as mock_post:
def mock_post_detect_side_effect(url, *args, **kwargs):
if url.endswith("/detect/openai/v2"):
request_body = kwargs.get("json", {})
if request_body["messages"][-1]["role"] == "user":
return response_with_detections
elif url.endswith("/detect/output/v2"):
elif request_body["messages"][-1]["role"] == "assistant":
return response_without_detections
else:
raise ValueError("Unexpected URL: {}".format(url))
raise ValueError("Unexpected request: {}".format(request_body))
mock_post.side_effect = mock_post_detect_side_effect

45
tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py
View File

@ -435,6 +435,8 @@ def create_standard_logging_payload_with_latency_metrics() -> StandardLoggingPay
start_time=1234567890.0,
end_time=1234567890.5,
duration=0.5, # 500ms
guardrail_request={"input": "test input message", "user_id": "test_user"},
guardrail_response={"output": "filtered output", "flagged": False, "score": 0.1},
)
hidden_params = StandardLoggingHiddenParams(
@ -567,3 +569,46 @@ def test_latency_metrics_edge_cases(mock_env_vars):
)
metadata = logger._get_dd_llm_obs_payload_metadata(standard_payload)
assert "guardrail_overhead_time_ms" not in metadata
def test_guardrail_information_in_metadata(mock_env_vars):
"""Test that guardrail_information is included in metadata with input/output fields"""
with patch('litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client'), \
patch('asyncio.create_task'):
logger = DataDogLLMObsLogger()
# Create a standard payload with guardrail information
standard_payload = create_standard_logging_payload_with_latency_metrics()
kwargs = {
"standard_logging_object": standard_payload,
"litellm_params": {"metadata": {}}
}
start_time = datetime.now()
end_time = datetime.now()
# Create the payload and verify guardrail_information is in metadata
payload = logger.create_llm_obs_payload(kwargs, start_time, end_time)
metadata = payload["meta"]["metadata"]
# Verify guardrail_information is present in metadata
assert "guardrail_information" in metadata
assert metadata["guardrail_information"] is not None
# Verify the guardrail information structure
guardrail_info = metadata["guardrail_information"]
assert guardrail_info["guardrail_name"] == "test_guardrail"
assert guardrail_info["guardrail_status"] == "success"
assert guardrail_info["duration"] == 0.5
# Verify input/output fields are present
assert "guardrail_request" in guardrail_info
assert "guardrail_response" in guardrail_info
# Validate the input/output content
assert guardrail_info["guardrail_request"]["input"] == "test input message"
assert guardrail_info["guardrail_request"]["user_id"] == "test_user"
assert guardrail_info["guardrail_response"]["output"] == "filtered output"
assert guardrail_info["guardrail_response"]["flagged"] == False
assert guardrail_info["guardrail_response"]["score"] == 0.1

119
tests/test_litellm/test_cost_calculator.py
View File

@ -482,6 +482,125 @@ def test_gemini_25_implicit_caching_cost():
print(f"✓ Gemini 2.5 implicit caching cost calculation is correct: ${result:.8f}")
def test_log_context_cost_calculation():
"""
Test that log context cost calculation works correctly with tiered pricing.
This test verifies that when using extended context (above 200k tokens),
the log context costs are calculated using the appropriate tiered rates.
"""
from litellm import completion_cost
from litellm.types.utils import (
Choices,
Message,
ModelResponse,
PromptTokensDetailsWrapper,
Usage,
)
# Create a mock response with extended context usage
extended_context_response = ModelResponse(
id="test-extended-context-response",
created=1750733889,
model="claude-4-sonnet-20250514",
object="chat.completion",
system_fingerprint=None,
choices=[
Choices(
finish_reason="stop",
index=0,
message=Message(
content="This is a test response for extended context cost calculation.",
role="assistant",
tool_calls=None,
function_call=None,
),
)
],
usage=Usage(
total_tokens=350000, # Above 200k threshold
prompt_tokens=300000, # Above 200k threshold
completion_tokens=50000,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=300000,
cached_tokens=0, # No cache hits
audio_tokens=None,
image_tokens=None,
character_count=None,
video_length_seconds=None,
),
completion_tokens_details=None,
_cache_creation_input_tokens=1000, # Some tokens added to cache
),
)
# Calculate the cost using the extended context model
result = completion_cost(
completion_response=extended_context_response,
model="claude-4-sonnet-20250514",
custom_llm_provider="anthropic",
)
# Debug: Print the actual result
print(f"DEBUG: Actual cost result: ${result:.6f}")
# Get model info to understand the pricing
from litellm import get_model_info
model_info = get_model_info(model="claude-4-sonnet-20250514", custom_llm_provider="anthropic")
# Calculate expected cost based on actual model pricing
input_cost_per_token = model_info.get("input_cost_per_token", 0)
output_cost_per_token = model_info.get("output_cost_per_token", 0)
cache_creation_cost_per_token = model_info.get("cache_creation_input_token_cost", 0)
# Check if tiered pricing is applied
input_cost_above_200k = model_info.get("input_cost_per_token_above_200k_tokens", input_cost_per_token)
output_cost_above_200k = model_info.get("output_cost_per_token_above_200k_tokens", output_cost_per_token)
cache_creation_above_200k = model_info.get("cache_creation_input_token_cost_above_200k_tokens", cache_creation_cost_per_token)
print(f"DEBUG: Base input cost per token: ${input_cost_per_token:.2e}")
print(f"DEBUG: Base output cost per token: ${output_cost_per_token:.2e}")
print(f"DEBUG: Base cache creation cost per token: ${cache_creation_cost_per_token:.2e}")
# Handle tiered pricing - if not available, use base pricing
if input_cost_above_200k is not None:
print(f"DEBUG: Tiered input cost per token (>200k): ${input_cost_above_200k:.2e}")
else:
print(f"DEBUG: No tiered input pricing available, using base pricing")
input_cost_above_200k = input_cost_per_token
if output_cost_above_200k is not None:
print(f"DEBUG: Tiered output cost per token (>200k): ${output_cost_above_200k:.2e}")
else:
print(f"DEBUG: No tiered output pricing available, using base pricing")
output_cost_above_200k = output_cost_per_token
if cache_creation_above_200k is not None:
print(f"DEBUG: Tiered cache creation cost per token (>200k): ${cache_creation_above_200k:.2e}")
else:
print(f"DEBUG: No tiered cache creation pricing available, using base pricing")
cache_creation_above_200k = cache_creation_cost_per_token
# Since we're above 200k tokens, we should use tiered pricing if available
expected_input_cost = 300000 * input_cost_above_200k
expected_output_cost = 50000 * output_cost_above_200k
expected_cache_cost = 1000 * cache_creation_above_200k
expected_total = expected_input_cost + expected_output_cost + expected_cache_cost
print(f"DEBUG: Expected total: ${expected_total:.6f}")
# Allow for small floating point differences
assert (
abs(result - expected_total) < 1e-6
), f"Expected cost ${expected_total:.6f}, but got ${result:.6f}"
print(f"✓ Log context cost calculation with tiered pricing is correct: ${result:.6f}")
print(f" - Input tokens (300k): ${expected_input_cost:.6f}")
print(f" - Output tokens (50k): ${expected_output_cost:.6f}")
print(f" - Cache creation (1k): ${expected_cache_cost:.6f}")
print(f" - Total: ${result:.6f}")
def test_gemini_25_explicit_caching_cost_direct_usage():
"""
Test that Gemini 2.5 models correctly calculate costs with explicit caching.

90
tests/test_litellm/test_utils.py
View File

@ -376,23 +376,23 @@ def test_cohere_embedding_optional_params():
def validate_model_cost_values(model_data, exceptions=None):
"""
Validates that cost values in model data do not exceed 1.
Args:
model_data (dict): The model data dictionary
exceptions (list, optional): List of model IDs that are allowed to have costs > 1
Returns:
tuple: (is_valid, violations) where is_valid is a boolean and violations is a list of error messages
"""
if exceptions is None:
exceptions = []
violations = []
# Define all cost-related fields to check
cost_fields = [
"input_cost_per_token",
"output_cost_per_token",
"output_cost_per_token",
"input_cost_per_character",
"output_cost_per_character",
"input_cost_per_image",
@ -431,22 +431,22 @@ def validate_model_cost_values(model_data, exceptions=None):
"output_cost_per_reasoning_token",
"citation_cost_per_token",
]
# Also check nested cost fields
nested_cost_fields = [
"search_context_cost_per_query",
]
for model_id, model_info in model_data.items():
# Skip if this model is in exceptions
if model_id in exceptions:
continue
# Check direct cost fields
for field in cost_fields:
if field in model_info and model_info[field] is not None:
cost_value = model_info[field]
# Convert string values to float if needed
if isinstance(cost_value, str):
try:
@ -454,12 +454,12 @@ def validate_model_cost_values(model_data, exceptions=None):
except (ValueError, TypeError):
# Skip if we can't convert to float
continue
if isinstance(cost_value, (int, float)) and cost_value > 1:
violations.append(
f"Model '{model_id}' has {field} = {cost_value} which exceeds 1"
)
# Check nested cost fields
for field in nested_cost_fields:
if field in model_info and model_info[field] is not None:
@ -473,12 +473,12 @@ def validate_model_cost_values(model_data, exceptions=None):
except (ValueError, TypeError):
# Skip if we can't convert to float
continue
if isinstance(nested_value, (int, float)) and nested_value > 1:
violations.append(
f"Model '{model_id}' has {field}.{nested_field} = {nested_value} which exceeds 1"
)
return len(violations) == 0, violations
@ -497,7 +497,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"supports_computer_use": {"type": "boolean"},
"cache_creation_input_audio_token_cost": {"type": "number"},
"cache_creation_input_token_cost": {"type": "number"},
"cache_creation_input_token_cost_above_200k_tokens": {"type": "number"},
"cache_read_input_token_cost": {"type": "number"},
"cache_read_input_token_cost_above_200k_tokens": {"type": "number"},
"cache_read_input_audio_token_cost": {"type": "number"},
"deprecation_date": {"type": "string"},
"input_cost_per_audio_per_second": {"type": "number"},
@ -653,7 +655,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
# Validate schema
validate(actual_json, INTENDED_SCHEMA)
# Validate cost values
# Define exceptions for models that are allowed to have costs > 1
# Add model IDs here if they legitimately have costs > 1
@ -661,9 +663,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
# Add any model IDs that should be exempt from the cost validation
# Example: "expensive-model-id",
]
is_valid, violations = validate_model_cost_values(actual_json, exceptions)
if not is_valid:
error_message = "Cost validation failed:\n" + "\n".join(violations)
error_message += "\n\nTo add exceptions, add the model ID to the 'exceptions' list in the test function."
@ -2330,25 +2332,31 @@ def test_block_key_hashing_logic():
("", False, ""), # Empty string should not be hashed
("sk-", True, hash_token("sk-")), # Edge case: just "sk-"
]
for input_key, should_be_hashed, expected_output in test_cases:
# Simulate the logic from block_key() function
if input_key.startswith("sk-"):
hashed_token = hash_token(token=input_key)
else:
hashed_token = input_key
assert hashed_token == expected_output, f"Failed for input: {input_key}"
# Additional verification: if it should be hashed, verify it's actually a hash
if should_be_hashed:
# SHA-256 hashes are 64 characters long and contain only hex digits
assert len(hashed_token) == 64, f"Hash length should be 64, got {len(hashed_token)} for {input_key}"
assert all(c in '0123456789abcdef' for c in hashed_token), f"Hash should contain only hex digits for {input_key}"
assert (
len(hashed_token) == 64
), f"Hash length should be 64, got {len(hashed_token)} for {input_key}"
assert all(
c in "0123456789abcdef" for c in hashed_token
), f"Hash should contain only hex digits for {input_key}"
else:
# If not hashed, it should be the original string
assert hashed_token == input_key, f"Non-hashed key should remain unchanged: {input_key}"
assert (
hashed_token == input_key
), f"Non-hashed key should remain unchanged: {input_key}"
print("✅ All block_key hashing logic tests passed!")
@ -2357,36 +2365,38 @@ def test_generate_gcp_iam_access_token():
Test the _generate_gcp_iam_access_token function with mocked GCP IAM client.
"""
from unittest.mock import Mock, patch
service_account = "projects/-/serviceAccounts/test@project.iam.gserviceaccount.com"
expected_token = "test-access-token-12345"
# Mock the GCP IAM client and its response
mock_response = Mock()
mock_response.access_token = expected_token
mock_client = Mock()
mock_client.generate_access_token.return_value = mock_response
# Mock the iam_credentials_v1 module
mock_iam_credentials_v1 = Mock()
mock_iam_credentials_v1.IAMCredentialsClient = Mock(return_value=mock_client)
mock_iam_credentials_v1.GenerateAccessTokenRequest = Mock()
# Test successful token generation by mocking sys.modules
with patch.dict('sys.modules', {'google.cloud.iam_credentials_v1': mock_iam_credentials_v1}):
with patch.dict(
"sys.modules", {"google.cloud.iam_credentials_v1": mock_iam_credentials_v1}
):
from litellm._redis import _generate_gcp_iam_access_token
result = _generate_gcp_iam_access_token(service_account)
assert result == expected_token
mock_iam_credentials_v1.IAMCredentialsClient.assert_called_once()
mock_client.generate_access_token.assert_called_once()
# Verify the request was created with correct parameters
mock_iam_credentials_v1.GenerateAccessTokenRequest.assert_called_once_with(
name=service_account,
scope=['https://www.googleapis.com/auth/cloud-platform']
scope=["https://www.googleapis.com/auth/cloud-platform"],
)
@ -2398,17 +2408,17 @@ def test_generate_gcp_iam_access_token_import_error():
from litellm._redis import _generate_gcp_iam_access_token
# Mock the import to fail when the function tries to import google.cloud.iam_credentials_v1
original_import = __builtins__['__import__']
original_import = __builtins__["__import__"]
def mock_import(name, *args, **kwargs):
if name == 'google.cloud.iam_credentials_v1':
if name == "google.cloud.iam_credentials_v1":
raise ImportError("No module named 'google.cloud.iam_credentials_v1'")
return original_import(name, *args, **kwargs)
with patch('builtins.__import__', side_effect=mock_import):
with patch("builtins.__import__", side_effect=mock_import):
with pytest.raises(ImportError) as exc_info:
_generate_gcp_iam_access_token("test-service-account")
assert "google-cloud-iam is required" in str(exc_info.value)
assert "pip install google-cloud-iam" in str(exc_info.value)
@ -2428,4 +2438,4 @@ def test_model_info_for_vertex_ai_deepseek_model():
assert model_info["input_cost_per_token"] is not None
assert model_info["output_cost_per_token"] is not None
print("vertex deepseek model info", model_info)
print("vertex deepseek model info", model_info)

39
tests/test_litellm/vector_stores/test_vector_store_registry.py
View File

@ -13,8 +13,11 @@ sys.path.insert(
) # Adds the parent directory to the system path
from datetime import datetime, timezone
from unittest.mock import MagicMock, patch
import litellm
from litellm.types.vector_stores import LiteLLM_ManagedVectorStore
from litellm.vector_stores.main import search
from litellm.vector_stores.vector_store_registry import VectorStoreRegistry
@ -113,3 +116,39 @@ def test_add_vector_store_to_registry():
assert len(registry.vector_stores) == 3
# Original store should still be there unchanged
assert registry.vector_stores[0]["vector_store_name"] == "existing_store_1"
def test_search_uses_registry_credentials():
"""search() should pull credentials from vector_store_registry when available"""
vector_store = LiteLLM_ManagedVectorStore(
vector_store_id="vs1",
custom_llm_provider="bedrock",
created_at=datetime.now(timezone.utc),
updated_at=datetime.now(timezone.utc),
)
registry = VectorStoreRegistry([vector_store])
original_registry = getattr(litellm, "vector_store_registry", None)
litellm.vector_store_registry = registry
try:
logger = MagicMock()
logger._response_cost_calculator.return_value = 0
with patch.object(
registry,
"get_credentials_for_vector_store",
return_value={"aws_access_key_id": "ABC", "aws_secret_access_key": "DEF", "aws_region_name": "us-east-1"},
) as mock_get_creds, patch(
"litellm.vector_stores.main.ProviderConfigManager.get_provider_vector_stores_config",
return_value=MagicMock(),
), patch(
"litellm.vector_stores.main.base_llm_http_handler.vector_store_search_handler",
return_value={},
) as mock_handler:
search(vector_store_id="vs1", query="test", litellm_logging_obj=logger)
mock_get_creds.assert_called_once_with("vs1")
called_params = mock_handler.call_args.kwargs["litellm_params"]
assert getattr(called_params, "aws_access_key_id") == "ABC"
assert getattr(called_params, "aws_secret_access_key") == "DEF"
assert getattr(called_params, "aws_region_name") == "us-east-1"
finally:
litellm.vector_store_registry = original_registry

113
tests/vector_store_tests/test_bedrock_vector_store.py
View File

@ -111,4 +111,115 @@ async def test_bedrock_search_with_router():
vector_store_id="T37J8R4WTM",
custom_llm_provider="bedrock",
)
print(search_response)
print(search_response)
@pytest.mark.asyncio
async def test_bedrock_search_with_credentials_managed_registry():
"""
Test that the vector store search uses the credential accessor from the registry
when AWS environment variables are not set, ensuring credentials are managed properly.
"""
from unittest.mock import patch, MagicMock
from litellm.router import Router
from litellm.types.vector_stores import LiteLLM_ManagedVectorStore
from litellm.types.utils import CredentialItem
from litellm.vector_stores.vector_store_registry import VectorStoreRegistry
from datetime import datetime, timezone
import litellm
# Store original registry and credential list
original_registry = getattr(litellm, "vector_store_registry", None)
original_credential_list = getattr(litellm, "credential_list", [])
try:
# Set up test AWS credentials in the credential system
test_credentials = CredentialItem(
credential_name="bedrock-litellm-website-knowledgebase",
credential_info={
"provider": "aws",
"description": "Test AWS credentials for bedrock"
},
credential_values={
"aws_access_key_id": "test_access_key",
"aws_secret_access_key": "test_secret_key",
"aws_region_name": "us-east-1",
}
)
# Set up the credential list
litellm.credential_list = [test_credentials]
# Create vector store with credential reference
vector_store = LiteLLM_ManagedVectorStore(
vector_store_id="T37J8R4WTM",
custom_llm_provider="bedrock",
created_at=datetime.now(timezone.utc),
updated_at=datetime.now(timezone.utc),
litellm_credential_name="bedrock-litellm-website-knowledgebase",
)
# Set up registry
registry = VectorStoreRegistry([vector_store])
litellm.vector_store_registry = registry
# Verify credentials can be retrieved from registry
retrieved_credentials = registry.get_credentials_for_vector_store("T37J8R4WTM")
assert retrieved_credentials, "Should retrieve credentials from registry"
assert retrieved_credentials.get("aws_access_key_id") == "test_access_key"
assert retrieved_credentials.get("aws_secret_access_key") == "test_secret_key"
assert retrieved_credentials.get("aws_region_name") == "us-east-1"
# Create router and perform search
_router = Router(model_list=[])
# Mock the credential injection process to verify it's called
with patch.object(registry, 'get_credentials_for_vector_store', wraps=registry.get_credentials_for_vector_store) as mock_get_creds:
# Mock the actual search call to avoid making real API calls
with patch('litellm.vector_stores.main.base_llm_http_handler.vector_store_search_handler') as mock_handler:
mock_handler.return_value = {
"data": [
{
"id": "test_result",
"text": "Mock search result",
"score": 0.9,
"metadata": {}
}
]
}
search_response = await _router.avector_store_search(
query="what happens after we add a model",
vector_store_id="T37J8R4WTM",
custom_llm_provider="bedrock",
)
# Verify the search was called
mock_handler.assert_called_once()
call_kwargs = mock_handler.call_args[1]
# Verify that the credential accessor was called with the correct vector store ID
mock_get_creds.assert_called_with("T37J8R4WTM")
# Verify the credentials were injected into the search call
litellm_params = call_kwargs.get("litellm_params", {})
# The key test: verify that credentials from the registry were used
# Since we have a registry with credentials, they should be present in the params
assert hasattr(litellm_params, 'aws_access_key_id'), "aws_access_key_id should be in litellm_params"
assert hasattr(litellm_params, 'aws_secret_access_key'), "aws_secret_access_key should be in litellm_params"
assert hasattr(litellm_params, 'aws_region_name'), "aws_region_name should be in litellm_params"
# Verify we got the expected response
assert search_response["data"][0]["id"] == "test_result"
print(f"✅ Test passed: Credential accessor was called with vector store ID: T37J8R4WTM")
print(f"✅ Retrieved credentials: {retrieved_credentials}")
print(f"✅ Credentials were injected into search call")
print(f"✅ Search completed successfully using registry credentials")
finally:
# Restore original state
litellm.vector_store_registry = original_registry
litellm.credential_list = original_credential_list

2
ui/litellm-dashboard/src/app/page.tsx
View File

@ -8,7 +8,7 @@ import { Team } from "@/components/key_team_helpers/key_list"
import Navbar from "@/components/navbar"
import { ThemeProvider } from "@/contexts/ThemeContext"
import UserDashboard from "@/components/user_dashboard"
import ModelDashboard from "@/components/model_dashboard"
import ModelDashboard from "@/components/templates/model_dashboard"
import ViewUserDashboard from "@/components/view_users"
import Teams from "@/components/teams"
import Organizations from "@/components/organizations"

45
ui/litellm-dashboard/src/components/callback_info_helpers.tsx
View File

@ -3,6 +3,7 @@ export enum Callbacks {
CustomCallbackAPI = "Custom Callback API",
Datadog = "Datadog",
Langfuse = "Langfuse",
LangfuseOtel = "LangfuseOtel",
LangSmith = "LangSmith",
Lago = "Lago",
OpenMeter = "OpenMeter",
@ -16,6 +17,7 @@ export const callback_map: Record<string, string> = {
CustomCallbackAPI: "custom_callback_api",
Datadog: "datadog",
Langfuse: "langfuse",
LangfuseOtel: "langfuse_otel",
LangSmith: "langsmith",
Lago: "lago",
OpenMeter: "openmeter",
@ -45,6 +47,7 @@ interface CallbackInfo {
logo: string;
supports_key_team_logging: boolean;
dynamic_params: Record<string, "text" | "password" | "select" | "upload">;
description: string | null;
}
export const callbackInfo: Record<string, CallbackInfo> = {
@ -55,7 +58,18 @@ export const callbackInfo: Record<string, CallbackInfo> = {
"langfuse_public_key": "text",
"langfuse_secret_key": "password",
"langfuse_host": "text"
}
},
description: "Langfuse v2 Logging Integration"
},
[Callbacks.LangfuseOtel]: {
logo: `${asset_logos_folder}langfuse.png`,
supports_key_team_logging: true,
dynamic_params: {
"langfuse_public_key": "text",
"langfuse_secret_key": "password",
"langfuse_host": "text"
},
description: "Langfuse v3 OTEL Logging Integration"
},
[Callbacks.Arize]: {
logo: `${asset_logos_folder}arize.png`,
@ -63,7 +77,8 @@ export const callbackInfo: Record<string, CallbackInfo> = {
dynamic_params: {
"arize_api_key": "password",
"arize_space_id": "text",
}
},
description: "Arize Logging Integration"
},
[Callbacks.LangSmith]: {
logo: `${asset_logos_folder}langsmith.png`,
@ -72,42 +87,50 @@ export const callbackInfo: Record<string, CallbackInfo> = {
"langsmith_api_key": "password",
"langsmith_project": "text",
"langsmith_base_url": "text"
}
},
},
description: "Langsmith Logging Integration"
},
[Callbacks.Braintrust]: {
logo: `${asset_logos_folder}braintrust.png`,
supports_key_team_logging: false,
dynamic_params: {}
dynamic_params: {},
description: "Braintrust Logging Integration"
},
[Callbacks.CustomCallbackAPI]: {
logo: `${asset_logos_folder}custom.svg`,
supports_key_team_logging: true,
dynamic_params: {}
dynamic_params: {},
description: "Custom Callback API Logging Integration"
},
[Callbacks.Datadog]: {
logo: `${asset_logos_folder}datadog.png`,
supports_key_team_logging: false,
dynamic_params: {}
dynamic_params: {},
description: "Datadog Logging Integration"
},
[Callbacks.Lago]: {
logo: `${asset_logos_folder}lago.svg`,
supports_key_team_logging: false,
dynamic_params: {}
dynamic_params: {},
description: "Lago Billing Logging Integration"
},
[Callbacks.OpenMeter]: {
logo: `${asset_logos_folder}openmeter.png`,
supports_key_team_logging: false,
dynamic_params: {}
dynamic_params: {},
description: "OpenMeter Logging Integration"
},
[Callbacks.OTel]: {
logo: `${asset_logos_folder}otel.png`,
supports_key_team_logging: false,
dynamic_params: {}
dynamic_params: {},
description: "OpenTelemetry Logging Integration"
},
[Callbacks.S3]: {
logo: `${asset_logos_folder}aws.svg`,
supports_key_team_logging: false,
dynamic_params: {}
dynamic_params: {},
description: "S3 Bucket (AWS) Logging Integration"
}
};

63
ui/litellm-dashboard/src/components/model_dashboard/model_dashboard.tsx
View File

@ -1,63 +0,0 @@
import { useState } from 'react';
import { Select, SelectItem, Text } from "@tremor/react";
interface ModelData {
team_id: string;
team_name: string;
// Add other properties as needed
}
interface ModelDashboardProps {
modelData: ModelData[];
}
export default function ModelDashboard({ modelData }: ModelDashboardProps) {
const [selectedTeam, setSelectedTeam] = useState<string | null>(null);
const getTeamName = (teamId: string): string => {
const team = modelData.find(item => item.team_id === teamId);
return team?.team_name || 'Unknown Team';
};
return (
<div className="flex flex-col space-y-4">
<div className="flex justify-between items-center mb-6">
<div>
<Text className="text-lg font-medium">Model Management</Text>
<Text className="text-gray-500">Add and manage models for the proxy</Text>
</div>
<div className="flex items-center gap-4">
<div className="flex items-center gap-2">
<Text>Filter by Public Model Name:</Text>
<Select
className="w-64"
defaultValue="all"
>
<SelectItem value="all">All Models</SelectItem>
{/* Add model options here */}
</Select>
</div>
<div className="flex items-center gap-2">
<Text>Filter by Team:</Text>
<Select
className="w-64"
value={selectedTeam ?? "all"}
onValueChange={(value) => setSelectedTeam(value === "all" ? null : value)}
>
<SelectItem value="all">All Teams</SelectItem>
{Array.from(new Set(modelData.map(model => model.team_id)))
.filter(teamId => teamId !== null)
.map(teamId => (
<SelectItem key={teamId} value={teamId}>
{getTeamName(teamId)}
</SelectItem>
))}
</Select>
</div>
</div>
</div>
</div>
);
}

241
ui/litellm-dashboard/src/components/model_dashboard/columns.tsx → ui/litellm-dashboard/src/components/molecules/models/columns.tsx
View File

@ -1,10 +1,10 @@
import { ColumnDef } from "@tanstack/react-table";
import { Button, Badge, Icon } from "@tremor/react";
import { Tooltip } from "antd";
import { getProviderLogoAndName } from "../provider_info_helpers";
import { ModelData } from "./types";
import { TrashIcon, PencilIcon, PencilAltIcon } from "@heroicons/react/outline";
import DeleteModelButton from "../delete_model_button";
import { getProviderLogoAndName } from "../../provider_info_helpers";
import { ModelData } from "../../model_dashboard/types";
import { TrashIcon, PencilIcon, PencilAltIcon, KeyIcon } from "@heroicons/react/outline";
import DeleteModelButton from "../../delete_model_button";
import { useState } from "react";
export const columns = (
@ -21,7 +21,7 @@ export const columns = (
setExpandedRows: (expandedRows: Set<string>) => void,
): ColumnDef<ModelData>[] => [
{
header: "Model ID",
header: () => <span className="text-sm font-semibold">Model ID</span>,
accessorKey: "model_info.id",
cell: ({ row }) => {
const model = row.original;
@ -38,77 +38,117 @@ export const columns = (
},
},
{
header: "Public Model Name",
header: () => <span className="text-sm font-semibold">Model Information</span>,
accessorKey: "model_name",
size: 250, // Fixed column width
cell: ({ row }) => {
const model = row.original;
const displayName = getDisplayModelName(row.original) || "-";
const tooltipContent = (
<div>
<div><strong>Provider:</strong> {model.provider || "-"}</div>
<div><strong>Public Model Name:</strong> {displayName}</div>
<div><strong>LiteLLM Model Name:</strong> {model.litellm_model_name || "-"}</div>
</div>
);
return (
<Tooltip title={displayName}>
<div className="text-xs truncate whitespace-nowrap">
{displayName}
<Tooltip title={tooltipContent}>
<div className="flex items-start space-x-2 min-w-0 w-full max-w-[250px]">
{/* Provider Icon */}
<div className="flex-shrink-0 mt-0.5">
{model.provider ? (
<img
src={getProviderLogoAndName(model.provider).logo}
alt={`${model.provider} logo`}
className="w-4 h-4"
onError={(e) => {
const target = e.target as HTMLImageElement;
const parent = target.parentElement;
if (parent) {
const fallbackDiv = document.createElement('div');
fallbackDiv.className = 'w-4 h-4 rounded-full bg-gray-200 flex items-center justify-center text-xs';
fallbackDiv.textContent = model.provider?.charAt(0) || '-';
parent.replaceChild(fallbackDiv, target);
}
}}
/>
) : (
<div className="w-4 h-4 rounded-full bg-gray-200 flex items-center justify-center text-xs">
-
</div>
)}
</div>
{/* Model Names Container */}
<div className="flex flex-col min-w-0 flex-1">
{/* Public Model Name */}
<div className="text-xs font-medium text-gray-900 truncate max-w-[210px]">
{displayName}
</div>
{/* LiteLLM Model Name */}
<div className="text-xs text-gray-500 truncate mt-0.5 max-w-[210px]">
{model.litellm_model_name || "-"}
</div>
</div>
</div>
</Tooltip>
);
},
},
{
header: "Provider",
accessorKey: "provider",
header: () => <span className="text-sm font-semibold">Credentials</span>,
accessorKey: "litellm_credential_name",
size: 180, // Fixed column width
cell: ({ row }) => {
const model = row.original;
return (
<div className="flex items-center space-x-2">
{model.provider && (
<img
src={getProviderLogoAndName(model.provider).logo}
alt={`${model.provider} logo`}
className="w-4 h-4"
onError={(e) => {
const target = e.target as HTMLImageElement;
const parent = target.parentElement;
if (parent) {
const fallbackDiv = document.createElement('div');
fallbackDiv.className = 'w-4 h-4 rounded-full bg-gray-200 flex items-center justify-center text-xs';
fallbackDiv.textContent = model.provider?.charAt(0) || '-';
parent.replaceChild(fallbackDiv, target);
}
}}
/>
)}
<p className="text-xs">{model.provider || "-"}</p>
const credentialName = model.litellm_params?.litellm_credential_name;
return credentialName ? (
<Tooltip title={`Credential: ${credentialName}`}>
<div className="flex items-center space-x-2 max-w-[180px]">
<KeyIcon className="w-4 h-4 text-blue-500 flex-shrink-0" />
<span className="text-xs truncate" title={credentialName}>
{credentialName}
</span>
</div>
</Tooltip>
) : (
<div className="flex items-center space-x-2 max-w-[180px]">
<KeyIcon className="w-4 h-4 text-gray-300 flex-shrink-0" />
<span className="text-xs text-gray-400">No credentials</span>
</div>
);
},
},
{
header: "LiteLLM Model Name",
accessorKey: "litellm_model_name",
cell: ({ row }) => {
const model = row.original;
return (
<Tooltip title={model.litellm_model_name}>
<div className="text-xs truncate whitespace-nowrap">
{model.litellm_model_name || "-"}
</div>
</Tooltip>
);
},
},
{
header: "Created At",
accessorKey: "model_info.created_at",
header: () => <span className="text-sm font-semibold">Created By</span>,
accessorKey: "model_info.created_by",
sortingFn: "datetime",
size: 160, // Fixed column width
cell: ({ row }) => {
const model = row.original;
const createdBy = model.model_info.created_by;
const createdAt = model.model_info.created_at
? new Date(model.model_info.created_at).toLocaleDateString()
: null;
return (
<span className="text-xs">
{model.model_info.created_at ? new Date(model.model_info.created_at).toLocaleDateString() : "-"}
</span>
<div className="flex flex-col min-w-0 max-w-[160px]">
{/* Created By - Primary */}
<div className="text-xs font-medium text-gray-900 truncate" title={createdBy || "Unknown"}>
{createdBy || "Unknown"}
</div>
{/* Created At - Secondary */}
<div className="text-xs text-gray-500 truncate mt-0.5" title={createdAt || "Unknown date"}>
{createdAt || "Unknown date"}
</div>
</div>
);
},
},
{
header: "Updated At",
header: () => <span className="text-sm font-semibold">Updated At</span>,
accessorKey: "model_info.updated_at",
sortingFn: "datetime",
cell: ({ row }) => {
@ -121,51 +161,45 @@ export const columns = (
},
},
{
header: "Created By",
accessorKey: "model_info.created_by",
cell: ({ row }) => {
const model = row.original;
return (
<span className="text-xs">
{model.model_info.created_by || "-"}
</span>
);
},
},
{
header: () => (
<Tooltip title="Cost per 1M tokens">
<span>Input Cost</span>
</Tooltip>
),
header: () => <span className="text-sm font-semibold">Costs</span>,
accessorKey: "input_cost",
size: 120, // Fixed column width
cell: ({ row }) => {
const model = row.original;
const inputCost = model.input_cost;
const outputCost = model.output_cost;
// If both costs are missing or undefined, show "-"
if (!inputCost && !outputCost) {
return (
<div className="max-w-[120px]">
<span className="text-xs text-gray-400">-</span>
</div>
);
}
return (
<pre className="text-xs">
{model.input_cost || "-"}
</pre>
<Tooltip title="Cost per 1M tokens">
<div className="flex flex-col min-w-0 max-w-[120px]">
{/* Input Cost - Primary */}
{inputCost && (
<div className="text-xs font-medium text-gray-900 truncate">
In: ${inputCost}
</div>
)}
{/* Output Cost - Secondary */}
{outputCost && (
<div className="text-xs text-gray-500 truncate mt-0.5">
Out: ${outputCost}
</div>
)}
</div>
</Tooltip>
);
},
},
{
header: () => (
<Tooltip title="Cost per 1M tokens">
<span>Output Cost</span>
</Tooltip>
),
accessorKey: "output_cost",
cell: ({ row }) => {
const model = row.original;
return (
<pre className="text-xs">
{model.output_cost || "-"}
</pre>
);
},
},
{
header: "Team ID",
header: () => <span className="text-sm font-semibold">Team ID</span>,
accessorKey: "model_info.team_id",
cell: ({ row }) => {
const model = row.original;
@ -188,7 +222,7 @@ export const columns = (
},
},
{
header: "Model Access Group",
header: () => <span className="text-sm font-semibold">Model Access Group</span>,
accessorKey: "model_info.model_access_group",
enableSorting: false,
cell: ({ row }) => {
@ -252,23 +286,7 @@ export const columns = (
},
},
{
header: "Credentials",
accessorKey: "litellm_credential_name",
cell: ({ row }) => {
const model = row.original;
return model.litellm_params && model.litellm_params.litellm_credential_name ? (
<div className="overflow-hidden">
<Tooltip title={model.litellm_params.litellm_credential_name}>
{model.litellm_params.litellm_credential_name.slice(0, 7)}...
</Tooltip>
</div>
) : (
<span className="text-gray-400">-</span>
);
},
},
{
header: "Status",
header: () => <span className="text-sm font-semibold">Status</span>,
accessorKey: "model_info.db_model",
cell: ({ row }) => {
const model = row.original;
@ -292,17 +310,6 @@ export const columns = (
const canEditModel = userRole === "Admin" || model.model_info?.created_by === userID;
return (
<div className="flex items-center justify-end gap-2 pr-4">
<Icon
icon={PencilAltIcon}
size="sm"
onClick={() => {
if (canEditModel) {
setSelectedModelId(model.model_info.id);
setEditModel(true);
}
}}
className={!canEditModel ? "opacity-50 cursor-not-allowed" : "cursor-pointer"}
/>
<Icon
icon={TrashIcon}
size="sm"

2
ui/litellm-dashboard/src/components/networking.tsx
View File

@ -4931,7 +4931,7 @@ export const deletePassThroughEndpointsCall = async (
try {
let url = proxyBaseUrl
? `${proxyBaseUrl}/config/pass_through_endpoint?endpoint_id=${endpointId}`
: `/config/pass_through_endpoint${endpointId}`;
: `/config/pass_through_endpoint?endpoint_id=${endpointId}`;
//message.info("Requesting model data");
const response = await fetch(url, {

2
ui/litellm-dashboard/src/components/settings.tsx
View File

@ -315,7 +315,7 @@ const Settings: React.FC<SettingsPageProps> = ({
addForm.validateFields().then((values) => {
// Call API to add the callback
let payload;
if (values.callback === "langfuse") {
if (values.callback === "langfuse" || values.callback === "langfuse_otel") {
payload = {
environment_variables: {
LANGFUSE_PUBLIC_KEY: values.langfusePublicKey,

46
ui/litellm-dashboard/src/components/team/LoggingSettings.tsx
View File

@ -164,18 +164,21 @@ const LoggingSettings: React.FC<LoggingSettingsProps> = ({
>
{allCallbacks.map((callbackName) => {
const logo = callbackInfo[callbackName]?.logo;
const description = callbackInfo[callbackName]?.description;
return (
<Option key={callbackName} value={callbackName} label={callbackName}>
<div className="flex items-center space-x-2">
{logo && (
<img
src={logo}
alt={callbackName}
className="w-4 h-4 object-contain"
/>
)}
<span>{callbackName}</span>
</div>
<Tooltip title={description} placement="right">
<div className="flex items-center space-x-2">
{logo && (
<img
src={logo}
alt={callbackName}
className="w-4 h-4 object-contain"
/>
)}
<span>{callbackName}</span>
</div>
</Tooltip>
</Option>
);
})}
@ -257,18 +260,21 @@ const LoggingSettings: React.FC<LoggingSettingsProps> = ({
>
{supportedCallbacks.map((callbackName) => {
const logo = callbackInfo[callbackName]?.logo;
const description = callbackInfo[callbackName]?.description;
return (
<Option key={callbackName} value={callbackName} label={callbackName}>
<div className="flex items-center space-x-2">
{logo && (
<img
src={logo}
alt={callbackName}
className="w-4 h-4 object-contain"
/>
)}
<span>{callbackName}</span>
</div>
<Tooltip title={description} placement="right">
<div className="flex items-center space-x-2">
{logo && (
<img
src={logo}
alt={callbackName}
className="w-4 h-4 object-contain"
/>
)}
<span>{callbackName}</span>
</div>
</Tooltip>
</Option>
);
})}

1655
ui/litellm-dashboard/src/components/model_dashboard.tsx → ui/litellm-dashboard/src/components/templates/model_dashboard.tsx
View File

File diff suppressed because it is too large Load Diff