diff --git a/Dockerfile b/Dockerfile index 9261d55d7f..addc109e10 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,8 +65,8 @@ COPY --from=builder /wheels/ /wheels/ # Install the built wheel using pip; again using a wildcard if it's the only file RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels -# Install semantic_router without dependencies -RUN pip install semantic_router --no-deps +# Install semantic_router and aurelio-sdk using script +RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh # Generate prisma client RUN prisma generate diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 956ec76dbe..351c4f6bc4 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -57,8 +57,8 @@ COPY --from=builder /wheels/ /wheels/ # Install the built wheel using pip; again using a wildcard if it's the only file RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels -# Install semantic_router without dependencies -RUN pip install semantic_router --no-deps +# Install semantic_router and aurelio-sdk using script +RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh # ensure pyjwt is used, not jwt RUN pip uninstall jwt -y diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 388dc6d076..c24c82b289 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -47,8 +47,8 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ \ && rm -f *.whl \ && rm -rf /wheels -# Install semantic_router without dependencies -RUN pip install semantic_router --no-deps +# Install semantic_router and aurelio-sdk using script +RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh # Ensure correct JWT library is used (pyjwt not jwt) RUN pip uninstall jwt -y && \ diff --git a/docker/install_auto_router.sh b/docker/install_auto_router.sh new file mode 100755 index 0000000000..794f9a2bbc --- /dev/null +++ b/docker/install_auto_router.sh @@ -0,0 +1,3 @@ +#!/bin/bash +pip install semantic_router==0.1.11 --no-deps +pip install aurelio-sdk==0.0.19 \ No newline at end of file diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index 8b1838c1ed..4f9c640977 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -422,6 +422,7 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger): "cache_hit": standard_logging_payload.get("cache_hit", "unknown"), "cache_key": standard_logging_payload.get("cache_key", "unknown"), "saved_cache_cost": standard_logging_payload.get("saved_cache_cost", 0), + "guardrail_information": standard_logging_payload.get("guardrail_information", None), } ######################################################### diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 4b6cffd06c..fe74778824 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -113,15 +113,20 @@ def _generic_cost_per_character( return prompt_cost, completion_cost -def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, float]: +def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, float, float, float]: """ - Return prompt cost for a given model and usage. + Return prompt cost, completion cost, and cache costs for a given model and usage. If input_tokens > threshold and `input_cost_per_token_above_[x]k_tokens` or `input_cost_per_token_above_[x]_tokens` is set, - then we use the corresponding threshold cost. + then we use the corresponding threshold cost for all token types. + + Returns: + Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost) """ prompt_base_cost = cast(float, _get_cost_per_unit(model_info, "input_cost_per_token")) completion_base_cost = cast(float, _get_cost_per_unit(model_info, "output_cost_per_token")) + cache_creation_cost = cast(float, _get_cost_per_unit(model_info, "cache_creation_input_token_cost")) + cache_read_cost = cast(float, _get_cost_per_unit(model_info, "cache_read_input_token_cost")) ## CHECK IF ABOVE THRESHOLD threshold: Optional[float] = None @@ -141,13 +146,28 @@ def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, fl f"output_cost_per_token_above_{threshold_str}_tokens", completion_base_cost, )) + + # Apply tiered pricing to cache costs + cache_creation_tiered_key = f"cache_creation_input_token_cost_above_{threshold_str}_tokens" + cache_read_tiered_key = f"cache_read_input_token_cost_above_{threshold_str}_tokens" + + if cache_creation_tiered_key in model_info: + cache_creation_cost = cast(float, _get_cost_per_unit( + model_info, cache_creation_tiered_key, cache_creation_cost + )) + + if cache_read_tiered_key in model_info: + cache_read_cost = cast(float, _get_cost_per_unit( + model_info, cache_read_tiered_key, cache_read_cost + )) + break except (IndexError, ValueError): continue except Exception: continue - return prompt_base_cost, completion_base_cost + return prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost def calculate_cost_component( @@ -262,28 +282,22 @@ def generic_cost_per_token( if text_tokens == 0: text_tokens = usage.prompt_tokens - cache_hit_tokens - audio_tokens - prompt_base_cost, completion_base_cost = _get_token_base_cost( + prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost = _get_token_base_cost( model_info=model_info, usage=usage ) prompt_cost = float(text_tokens) * prompt_base_cost - ### CACHE READ COST - prompt_cost += calculate_cost_component( - model_info, "cache_read_input_token_cost", cache_hit_tokens - ) + ### CACHE READ COST - Now uses tiered pricing + prompt_cost += float(cache_hit_tokens) * cache_read_cost ### AUDIO COST prompt_cost += calculate_cost_component( model_info, "input_cost_per_audio_token", audio_tokens ) - ### CACHE WRITING COST - prompt_cost += calculate_cost_component( - model_info, - "cache_creation_input_token_cost", - usage._cache_creation_input_tokens, - ) + ### CACHE WRITING COST - Now uses tiered pricing + prompt_cost += float(usage._cache_creation_input_tokens or 0) * cache_creation_cost ### CHARACTER COST diff --git a/litellm/llms/azure/chat/gpt_5_transformation.py b/litellm/llms/azure/chat/gpt_5_transformation.py index b8c032f4e9..d563a2889c 100644 --- a/litellm/llms/azure/chat/gpt_5_transformation.py +++ b/litellm/llms/azure/chat/gpt_5_transformation.py @@ -10,6 +10,7 @@ from .gpt_transformation import AzureOpenAIConfig class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): """Azure specific handling for gpt-5 models.""" + GPT5_SERIES_ROUTE = "gpt5_series/" @classmethod @@ -23,7 +24,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): def get_supported_openai_params(self, model: str) -> List[str]: return OpenAIGPT5Config.get_supported_openai_params(self, model=model) - + def map_openai_params( self, non_default_params: dict, diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index a0b76abb20..9a8bb74d44 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -15,14 +15,19 @@ class OpenAIGPT5Config(OpenAIGPTConfig): - Mapping ``max_tokens`` -> ``max_completion_tokens``. - Dropping unsupported ``temperature`` values when requested. """ + @classmethod def is_model_gpt_5_model(cls, model: str) -> bool: return "gpt-5" in model - + def get_supported_openai_params(self, model: str) -> list: + from litellm.utils import supports_tool_choice + base_gpt_series_params = super().get_supported_openai_params(model=model) gpt_5_only_params = ["reasoning_effort"] base_gpt_series_params.extend(gpt_5_only_params) + if not supports_tool_choice(model=model): + base_gpt_series_params.remove("tool_choice") return base_gpt_series_params def map_openai_params( @@ -61,4 +66,3 @@ class OpenAIGPT5Config(OpenAIGPTConfig): model=model, drop_params=drop_params, ) - diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a48c26bc87..1333bc37ad 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2457,7 +2457,7 @@ }, "azure/gpt-5-chat": { "max_tokens": 128000, - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "input_cost_per_token": 1.25e-06, "output_cost_per_token": 1e-05, @@ -2483,14 +2483,14 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, + "supports_tool_choice": false, "supports_native_streaming": true, "supports_reasoning": true, "source": "https://azure.microsoft.com/en-us/blog/gpt-5-in-azure-ai-foundry-the-future-of-ai-apps-and-agents-starts-here/" }, "azure/gpt-5-chat-latest": { "max_tokens": 128000, - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "input_cost_per_token": 1.25e-06, "output_cost_per_token": 1e-05, @@ -2516,7 +2516,7 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, + "supports_tool_choice": false, "supports_native_streaming": true, "supports_reasoning": true }, @@ -6437,11 +6437,13 @@ "supports_computer_use": true }, "claude-4-sonnet-20250514": { - "max_tokens": 64000, - "max_input_tokens": 200000, - "max_output_tokens": 64000, + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, + "input_cost_per_token_above_200k_tokens": 6e-06, + "output_cost_per_token_above_200k_tokens": 2.25e-05, "search_context_cost_per_query": { "search_context_size_low": 0.01, "search_context_size_medium": 0.01, @@ -6449,6 +6451,8 @@ }, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, + "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, "litellm_provider": "anthropic", "mode": "chat", "supports_function_calling": true, @@ -14649,6 +14653,50 @@ "mode": "chat", "supports_tool_choice": true }, + "together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-06, + "max_input_tokens": 262000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "supports_tool_choice": false, + "source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8" + }, + "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 2e-06, + "max_input_tokens": 256000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "supports_tool_choice": false, + "source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct" + }, + "together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": { + "input_cost_per_token": 6.5e-07, + "output_cost_per_token": 3e-06, + "max_input_tokens": 256000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "supports_tool_choice": false, + "source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507" + }, + "together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-07, + "max_input_tokens": 40000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "supports_tool_choice": false, + "source": "https://www.together.ai/models/qwen3-235b-a22b-fp8-tput" + }, "together_ai/deepseek-ai/DeepSeek-V3": { "input_cost_per_token": 1.25e-06, "output_cost_per_token": 1.25e-06, @@ -14673,6 +14721,17 @@ "mode": "chat", "supports_tool_choice": true }, + "together_ai/deepseek-ai/DeepSeek-R1-0528-tput": { + "input_cost_per_token": 5.5e-07, + "output_cost_per_token": 2.19e-06, + "max_input_tokens": 128000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "supports_tool_choice": false, + "source": "https://www.together.ai/models/deepseek-r1-0528-throughput" + }, "together_ai/mistralai/Mistral-Small-24B-Instruct-2501": { "litellm_provider": "together_ai", "supports_function_calling": true, @@ -14690,6 +14749,39 @@ "mode": "chat", "source": "https://www.together.ai/models/kimi-k2-instruct" }, + "together_ai/openai/gpt-oss-120b": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "max_input_tokens": 128000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "source": "https://www.together.ai/models/gpt-oss-120b" + }, + "together_ai/OpenAI/gpt-oss-20B": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2e-07, + "max_input_tokens": 128000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "source": "https://www.together.ai/models/gpt-oss-20b" + }, + "together_ai/zai-org/GLM-4.5-Air-FP8": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 1.1e-06, + "max_input_tokens": 128000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "source": "https://www.together.ai/models/glm-4-5-air" + }, "ollama/codegemma": { "max_tokens": 8192, "max_input_tokens": 8192, diff --git a/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py b/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py index 621defa2d4..3c03d70cef 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py +++ b/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py @@ -118,7 +118,7 @@ class AimGuardrail(CustomGuardrail): litellm_call_id=call_id, ) response = await self.async_handler.post( - f"{self.api_base}/detect/openai/v2", + f"{self.api_base}/fw/v1/analyze", headers=headers, json={"messages": data.get("messages", [])}, ) @@ -183,14 +183,17 @@ class AimGuardrail(CustomGuardrail): ) call_id = request_data.get("litellm_call_id") response = await self.async_handler.post( - f"{self.api_base}/detect/output/v2", + f"{self.api_base}/fw/v1/analyze", headers=self._build_aim_headers( hook=hook, key_alias=key_alias, user_email=user_email, litellm_call_id=call_id, ), - json={"output": output, "messages": request_data.get("messages", [])}, + json={ + "messages": request_data.get("messages", []) + + [{"role": "assistant", "content": output}] + }, ) response.raise_for_status() res = response.json() @@ -297,7 +300,7 @@ class AimGuardrail(CustomGuardrail): ) call_id = request_data.get("litellm_call_id") async with connect( - f"{self.ws_api_base}/detect/output/ws", + f"{self.ws_api_base}/fw/v1/analyze/stream", additional_headers=self._build_aim_headers( hook="output", key_alias=user_api_key_dict.key_alias, diff --git a/litellm/vector_stores/main.py b/litellm/vector_stores/main.py index 80d6341146..01e06e1306 100644 --- a/litellm/vector_stores/main.py +++ b/litellm/vector_stores/main.py @@ -358,6 +358,22 @@ def search( litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) _is_async = kwargs.pop("asearch", False) is True + # pull credentials from registry if available + vector_store_id_for_credentials = kwargs.get("vector_store_id", vector_store_id) + if ( + litellm.vector_store_registry is not None + and vector_store_id_for_credentials is not None + ): + try: + registry_credentials = ( + litellm.vector_store_registry.get_credentials_for_vector_store( + vector_store_id_for_credentials + ) + ) + kwargs.update(registry_credentials) + except Exception: + pass + # get llm provider logic litellm_params = GenericLiteLLMParams(**kwargs) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a48c26bc87..1333bc37ad 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -2457,7 +2457,7 @@ }, "azure/gpt-5-chat": { "max_tokens": 128000, - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "input_cost_per_token": 1.25e-06, "output_cost_per_token": 1e-05, @@ -2483,14 +2483,14 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, + "supports_tool_choice": false, "supports_native_streaming": true, "supports_reasoning": true, "source": "https://azure.microsoft.com/en-us/blog/gpt-5-in-azure-ai-foundry-the-future-of-ai-apps-and-agents-starts-here/" }, "azure/gpt-5-chat-latest": { "max_tokens": 128000, - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "input_cost_per_token": 1.25e-06, "output_cost_per_token": 1e-05, @@ -2516,7 +2516,7 @@ "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, + "supports_tool_choice": false, "supports_native_streaming": true, "supports_reasoning": true }, @@ -6437,11 +6437,13 @@ "supports_computer_use": true }, "claude-4-sonnet-20250514": { - "max_tokens": 64000, - "max_input_tokens": 200000, - "max_output_tokens": 64000, + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, + "input_cost_per_token_above_200k_tokens": 6e-06, + "output_cost_per_token_above_200k_tokens": 2.25e-05, "search_context_cost_per_query": { "search_context_size_low": 0.01, "search_context_size_medium": 0.01, @@ -6449,6 +6451,8 @@ }, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, + "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, "litellm_provider": "anthropic", "mode": "chat", "supports_function_calling": true, @@ -14649,6 +14653,50 @@ "mode": "chat", "supports_tool_choice": true }, + "together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-06, + "max_input_tokens": 262000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "supports_tool_choice": false, + "source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8" + }, + "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 2e-06, + "max_input_tokens": 256000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "supports_tool_choice": false, + "source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct" + }, + "together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": { + "input_cost_per_token": 6.5e-07, + "output_cost_per_token": 3e-06, + "max_input_tokens": 256000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "supports_tool_choice": false, + "source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507" + }, + "together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-07, + "max_input_tokens": 40000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "supports_tool_choice": false, + "source": "https://www.together.ai/models/qwen3-235b-a22b-fp8-tput" + }, "together_ai/deepseek-ai/DeepSeek-V3": { "input_cost_per_token": 1.25e-06, "output_cost_per_token": 1.25e-06, @@ -14673,6 +14721,17 @@ "mode": "chat", "supports_tool_choice": true }, + "together_ai/deepseek-ai/DeepSeek-R1-0528-tput": { + "input_cost_per_token": 5.5e-07, + "output_cost_per_token": 2.19e-06, + "max_input_tokens": 128000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "supports_tool_choice": false, + "source": "https://www.together.ai/models/deepseek-r1-0528-throughput" + }, "together_ai/mistralai/Mistral-Small-24B-Instruct-2501": { "litellm_provider": "together_ai", "supports_function_calling": true, @@ -14690,6 +14749,39 @@ "mode": "chat", "source": "https://www.together.ai/models/kimi-k2-instruct" }, + "together_ai/openai/gpt-oss-120b": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "max_input_tokens": 128000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "source": "https://www.together.ai/models/gpt-oss-120b" + }, + "together_ai/OpenAI/gpt-oss-20B": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2e-07, + "max_input_tokens": 128000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "source": "https://www.together.ai/models/gpt-oss-20b" + }, + "together_ai/zai-org/GLM-4.5-Air-FP8": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 1.1e-06, + "max_input_tokens": 128000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "source": "https://www.together.ai/models/glm-4-5-air" + }, "ollama/codegemma": { "max_tokens": 8192, "max_input_tokens": 8192, diff --git a/tests/local_testing/test_aim_guardrails.py b/tests/local_testing/test_aim_guardrails.py index b11a424ac3..3a9b6e9a3d 100644 --- a/tests/local_testing/test_aim_guardrails.py +++ b/tests/local_testing/test_aim_guardrails.py @@ -216,12 +216,13 @@ async def test_post_call__with_anonymized_entities__it_deanonymizes_output(): ) as mock_post: def mock_post_detect_side_effect(url, *args, **kwargs): - if url.endswith("/detect/openai/v2"): + request_body = kwargs.get("json", {}) + if request_body["messages"][-1]["role"] == "user": return response_with_detections - elif url.endswith("/detect/output/v2"): + elif request_body["messages"][-1]["role"] == "assistant": return response_without_detections else: - raise ValueError("Unexpected URL: {}".format(url)) + raise ValueError("Unexpected request: {}".format(request_body)) mock_post.side_effect = mock_post_detect_side_effect diff --git a/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py b/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py index 25263e31f9..b4575a7ebd 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py @@ -435,6 +435,8 @@ def create_standard_logging_payload_with_latency_metrics() -> StandardLoggingPay start_time=1234567890.0, end_time=1234567890.5, duration=0.5, # 500ms + guardrail_request={"input": "test input message", "user_id": "test_user"}, + guardrail_response={"output": "filtered output", "flagged": False, "score": 0.1}, ) hidden_params = StandardLoggingHiddenParams( @@ -567,3 +569,46 @@ def test_latency_metrics_edge_cases(mock_env_vars): ) metadata = logger._get_dd_llm_obs_payload_metadata(standard_payload) assert "guardrail_overhead_time_ms" not in metadata + + +def test_guardrail_information_in_metadata(mock_env_vars): + """Test that guardrail_information is included in metadata with input/output fields""" + with patch('litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client'), \ + patch('asyncio.create_task'): + logger = DataDogLLMObsLogger() + + # Create a standard payload with guardrail information + standard_payload = create_standard_logging_payload_with_latency_metrics() + + kwargs = { + "standard_logging_object": standard_payload, + "litellm_params": {"metadata": {}} + } + + start_time = datetime.now() + end_time = datetime.now() + + # Create the payload and verify guardrail_information is in metadata + payload = logger.create_llm_obs_payload(kwargs, start_time, end_time) + metadata = payload["meta"]["metadata"] + + # Verify guardrail_information is present in metadata + assert "guardrail_information" in metadata + assert metadata["guardrail_information"] is not None + + # Verify the guardrail information structure + guardrail_info = metadata["guardrail_information"] + assert guardrail_info["guardrail_name"] == "test_guardrail" + assert guardrail_info["guardrail_status"] == "success" + assert guardrail_info["duration"] == 0.5 + + # Verify input/output fields are present + assert "guardrail_request" in guardrail_info + assert "guardrail_response" in guardrail_info + + # Validate the input/output content + assert guardrail_info["guardrail_request"]["input"] == "test input message" + assert guardrail_info["guardrail_request"]["user_id"] == "test_user" + assert guardrail_info["guardrail_response"]["output"] == "filtered output" + assert guardrail_info["guardrail_response"]["flagged"] == False + assert guardrail_info["guardrail_response"]["score"] == 0.1 diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 9bff907700..5d9e7876cf 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -482,6 +482,125 @@ def test_gemini_25_implicit_caching_cost(): print(f"✓ Gemini 2.5 implicit caching cost calculation is correct: ${result:.8f}") + +def test_log_context_cost_calculation(): + """ + Test that log context cost calculation works correctly with tiered pricing. + + This test verifies that when using extended context (above 200k tokens), + the log context costs are calculated using the appropriate tiered rates. + """ + from litellm import completion_cost + from litellm.types.utils import ( + Choices, + Message, + ModelResponse, + PromptTokensDetailsWrapper, + Usage, + ) + + # Create a mock response with extended context usage + extended_context_response = ModelResponse( + id="test-extended-context-response", + created=1750733889, + model="claude-4-sonnet-20250514", + object="chat.completion", + system_fingerprint=None, + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message( + content="This is a test response for extended context cost calculation.", + role="assistant", + tool_calls=None, + function_call=None, + ), + ) + ], + usage=Usage( + total_tokens=350000, # Above 200k threshold + prompt_tokens=300000, # Above 200k threshold + completion_tokens=50000, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=300000, + cached_tokens=0, # No cache hits + audio_tokens=None, + image_tokens=None, + character_count=None, + video_length_seconds=None, + ), + completion_tokens_details=None, + _cache_creation_input_tokens=1000, # Some tokens added to cache + ), + ) + + # Calculate the cost using the extended context model + result = completion_cost( + completion_response=extended_context_response, + model="claude-4-sonnet-20250514", + custom_llm_provider="anthropic", + ) + + # Debug: Print the actual result + print(f"DEBUG: Actual cost result: ${result:.6f}") + + # Get model info to understand the pricing + from litellm import get_model_info + model_info = get_model_info(model="claude-4-sonnet-20250514", custom_llm_provider="anthropic") + + # Calculate expected cost based on actual model pricing + input_cost_per_token = model_info.get("input_cost_per_token", 0) + output_cost_per_token = model_info.get("output_cost_per_token", 0) + cache_creation_cost_per_token = model_info.get("cache_creation_input_token_cost", 0) + + # Check if tiered pricing is applied + input_cost_above_200k = model_info.get("input_cost_per_token_above_200k_tokens", input_cost_per_token) + output_cost_above_200k = model_info.get("output_cost_per_token_above_200k_tokens", output_cost_per_token) + cache_creation_above_200k = model_info.get("cache_creation_input_token_cost_above_200k_tokens", cache_creation_cost_per_token) + + print(f"DEBUG: Base input cost per token: ${input_cost_per_token:.2e}") + print(f"DEBUG: Base output cost per token: ${output_cost_per_token:.2e}") + print(f"DEBUG: Base cache creation cost per token: ${cache_creation_cost_per_token:.2e}") + + # Handle tiered pricing - if not available, use base pricing + if input_cost_above_200k is not None: + print(f"DEBUG: Tiered input cost per token (>200k): ${input_cost_above_200k:.2e}") + else: + print(f"DEBUG: No tiered input pricing available, using base pricing") + input_cost_above_200k = input_cost_per_token + + if output_cost_above_200k is not None: + print(f"DEBUG: Tiered output cost per token (>200k): ${output_cost_above_200k:.2e}") + else: + print(f"DEBUG: No tiered output pricing available, using base pricing") + output_cost_above_200k = output_cost_per_token + + if cache_creation_above_200k is not None: + print(f"DEBUG: Tiered cache creation cost per token (>200k): ${cache_creation_above_200k:.2e}") + else: + print(f"DEBUG: No tiered cache creation pricing available, using base pricing") + cache_creation_above_200k = cache_creation_cost_per_token + + # Since we're above 200k tokens, we should use tiered pricing if available + expected_input_cost = 300000 * input_cost_above_200k + expected_output_cost = 50000 * output_cost_above_200k + expected_cache_cost = 1000 * cache_creation_above_200k + expected_total = expected_input_cost + expected_output_cost + expected_cache_cost + + print(f"DEBUG: Expected total: ${expected_total:.6f}") + + # Allow for small floating point differences + assert ( + abs(result - expected_total) < 1e-6 + ), f"Expected cost ${expected_total:.6f}, but got ${result:.6f}" + + print(f"✓ Log context cost calculation with tiered pricing is correct: ${result:.6f}") + print(f" - Input tokens (300k): ${expected_input_cost:.6f}") + print(f" - Output tokens (50k): ${expected_output_cost:.6f}") + print(f" - Cache creation (1k): ${expected_cache_cost:.6f}") + print(f" - Total: ${result:.6f}") + def test_gemini_25_explicit_caching_cost_direct_usage(): """ Test that Gemini 2.5 models correctly calculate costs with explicit caching. diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index cb53638d83..bd39fbfc9c 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -376,23 +376,23 @@ def test_cohere_embedding_optional_params(): def validate_model_cost_values(model_data, exceptions=None): """ Validates that cost values in model data do not exceed 1. - + Args: model_data (dict): The model data dictionary exceptions (list, optional): List of model IDs that are allowed to have costs > 1 - + Returns: tuple: (is_valid, violations) where is_valid is a boolean and violations is a list of error messages """ if exceptions is None: exceptions = [] - + violations = [] - + # Define all cost-related fields to check cost_fields = [ "input_cost_per_token", - "output_cost_per_token", + "output_cost_per_token", "input_cost_per_character", "output_cost_per_character", "input_cost_per_image", @@ -431,22 +431,22 @@ def validate_model_cost_values(model_data, exceptions=None): "output_cost_per_reasoning_token", "citation_cost_per_token", ] - + # Also check nested cost fields nested_cost_fields = [ "search_context_cost_per_query", ] - + for model_id, model_info in model_data.items(): # Skip if this model is in exceptions if model_id in exceptions: continue - + # Check direct cost fields for field in cost_fields: if field in model_info and model_info[field] is not None: cost_value = model_info[field] - + # Convert string values to float if needed if isinstance(cost_value, str): try: @@ -454,12 +454,12 @@ def validate_model_cost_values(model_data, exceptions=None): except (ValueError, TypeError): # Skip if we can't convert to float continue - + if isinstance(cost_value, (int, float)) and cost_value > 1: violations.append( f"Model '{model_id}' has {field} = {cost_value} which exceeds 1" ) - + # Check nested cost fields for field in nested_cost_fields: if field in model_info and model_info[field] is not None: @@ -473,12 +473,12 @@ def validate_model_cost_values(model_data, exceptions=None): except (ValueError, TypeError): # Skip if we can't convert to float continue - + if isinstance(nested_value, (int, float)) and nested_value > 1: violations.append( f"Model '{model_id}' has {field}.{nested_field} = {nested_value} which exceeds 1" ) - + return len(violations) == 0, violations @@ -497,7 +497,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_computer_use": {"type": "boolean"}, "cache_creation_input_audio_token_cost": {"type": "number"}, "cache_creation_input_token_cost": {"type": "number"}, + "cache_creation_input_token_cost_above_200k_tokens": {"type": "number"}, "cache_read_input_token_cost": {"type": "number"}, + "cache_read_input_token_cost_above_200k_tokens": {"type": "number"}, "cache_read_input_audio_token_cost": {"type": "number"}, "deprecation_date": {"type": "string"}, "input_cost_per_audio_per_second": {"type": "number"}, @@ -653,7 +655,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): # Validate schema validate(actual_json, INTENDED_SCHEMA) - + # Validate cost values # Define exceptions for models that are allowed to have costs > 1 # Add model IDs here if they legitimately have costs > 1 @@ -661,9 +663,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): # Add any model IDs that should be exempt from the cost validation # Example: "expensive-model-id", ] - + is_valid, violations = validate_model_cost_values(actual_json, exceptions) - + if not is_valid: error_message = "Cost validation failed:\n" + "\n".join(violations) error_message += "\n\nTo add exceptions, add the model ID to the 'exceptions' list in the test function." @@ -2330,25 +2332,31 @@ def test_block_key_hashing_logic(): ("", False, ""), # Empty string should not be hashed ("sk-", True, hash_token("sk-")), # Edge case: just "sk-" ] - + for input_key, should_be_hashed, expected_output in test_cases: # Simulate the logic from block_key() function if input_key.startswith("sk-"): hashed_token = hash_token(token=input_key) else: hashed_token = input_key - + assert hashed_token == expected_output, f"Failed for input: {input_key}" - + # Additional verification: if it should be hashed, verify it's actually a hash if should_be_hashed: # SHA-256 hashes are 64 characters long and contain only hex digits - assert len(hashed_token) == 64, f"Hash length should be 64, got {len(hashed_token)} for {input_key}" - assert all(c in '0123456789abcdef' for c in hashed_token), f"Hash should contain only hex digits for {input_key}" + assert ( + len(hashed_token) == 64 + ), f"Hash length should be 64, got {len(hashed_token)} for {input_key}" + assert all( + c in "0123456789abcdef" for c in hashed_token + ), f"Hash should contain only hex digits for {input_key}" else: # If not hashed, it should be the original string - assert hashed_token == input_key, f"Non-hashed key should remain unchanged: {input_key}" - + assert ( + hashed_token == input_key + ), f"Non-hashed key should remain unchanged: {input_key}" + print("✅ All block_key hashing logic tests passed!") @@ -2357,36 +2365,38 @@ def test_generate_gcp_iam_access_token(): Test the _generate_gcp_iam_access_token function with mocked GCP IAM client. """ from unittest.mock import Mock, patch - + service_account = "projects/-/serviceAccounts/test@project.iam.gserviceaccount.com" expected_token = "test-access-token-12345" - + # Mock the GCP IAM client and its response mock_response = Mock() mock_response.access_token = expected_token - + mock_client = Mock() mock_client.generate_access_token.return_value = mock_response - + # Mock the iam_credentials_v1 module mock_iam_credentials_v1 = Mock() mock_iam_credentials_v1.IAMCredentialsClient = Mock(return_value=mock_client) mock_iam_credentials_v1.GenerateAccessTokenRequest = Mock() - + # Test successful token generation by mocking sys.modules - with patch.dict('sys.modules', {'google.cloud.iam_credentials_v1': mock_iam_credentials_v1}): + with patch.dict( + "sys.modules", {"google.cloud.iam_credentials_v1": mock_iam_credentials_v1} + ): from litellm._redis import _generate_gcp_iam_access_token - + result = _generate_gcp_iam_access_token(service_account) - + assert result == expected_token mock_iam_credentials_v1.IAMCredentialsClient.assert_called_once() mock_client.generate_access_token.assert_called_once() - + # Verify the request was created with correct parameters mock_iam_credentials_v1.GenerateAccessTokenRequest.assert_called_once_with( name=service_account, - scope=['https://www.googleapis.com/auth/cloud-platform'] + scope=["https://www.googleapis.com/auth/cloud-platform"], ) @@ -2398,17 +2408,17 @@ def test_generate_gcp_iam_access_token_import_error(): from litellm._redis import _generate_gcp_iam_access_token # Mock the import to fail when the function tries to import google.cloud.iam_credentials_v1 - original_import = __builtins__['__import__'] - + original_import = __builtins__["__import__"] + def mock_import(name, *args, **kwargs): - if name == 'google.cloud.iam_credentials_v1': + if name == "google.cloud.iam_credentials_v1": raise ImportError("No module named 'google.cloud.iam_credentials_v1'") return original_import(name, *args, **kwargs) - - with patch('builtins.__import__', side_effect=mock_import): + + with patch("builtins.__import__", side_effect=mock_import): with pytest.raises(ImportError) as exc_info: _generate_gcp_iam_access_token("test-service-account") - + assert "google-cloud-iam is required" in str(exc_info.value) assert "pip install google-cloud-iam" in str(exc_info.value) @@ -2428,4 +2438,4 @@ def test_model_info_for_vertex_ai_deepseek_model(): assert model_info["input_cost_per_token"] is not None assert model_info["output_cost_per_token"] is not None - print("vertex deepseek model info", model_info) \ No newline at end of file + print("vertex deepseek model info", model_info) diff --git a/tests/test_litellm/vector_stores/test_vector_store_registry.py b/tests/test_litellm/vector_stores/test_vector_store_registry.py index 785b55d7e0..fb585e1122 100644 --- a/tests/test_litellm/vector_stores/test_vector_store_registry.py +++ b/tests/test_litellm/vector_stores/test_vector_store_registry.py @@ -13,8 +13,11 @@ sys.path.insert( ) # Adds the parent directory to the system path from datetime import datetime, timezone +from unittest.mock import MagicMock, patch +import litellm from litellm.types.vector_stores import LiteLLM_ManagedVectorStore +from litellm.vector_stores.main import search from litellm.vector_stores.vector_store_registry import VectorStoreRegistry @@ -113,3 +116,39 @@ def test_add_vector_store_to_registry(): assert len(registry.vector_stores) == 3 # Original store should still be there unchanged assert registry.vector_stores[0]["vector_store_name"] == "existing_store_1" + + + +def test_search_uses_registry_credentials(): + """search() should pull credentials from vector_store_registry when available""" + vector_store = LiteLLM_ManagedVectorStore( + vector_store_id="vs1", + custom_llm_provider="bedrock", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ) + registry = VectorStoreRegistry([vector_store]) + original_registry = getattr(litellm, "vector_store_registry", None) + litellm.vector_store_registry = registry + try: + logger = MagicMock() + logger._response_cost_calculator.return_value = 0 + with patch.object( + registry, + "get_credentials_for_vector_store", + return_value={"aws_access_key_id": "ABC", "aws_secret_access_key": "DEF", "aws_region_name": "us-east-1"}, + ) as mock_get_creds, patch( + "litellm.vector_stores.main.ProviderConfigManager.get_provider_vector_stores_config", + return_value=MagicMock(), + ), patch( + "litellm.vector_stores.main.base_llm_http_handler.vector_store_search_handler", + return_value={}, + ) as mock_handler: + search(vector_store_id="vs1", query="test", litellm_logging_obj=logger) + mock_get_creds.assert_called_once_with("vs1") + called_params = mock_handler.call_args.kwargs["litellm_params"] + assert getattr(called_params, "aws_access_key_id") == "ABC" + assert getattr(called_params, "aws_secret_access_key") == "DEF" + assert getattr(called_params, "aws_region_name") == "us-east-1" + finally: + litellm.vector_store_registry = original_registry diff --git a/tests/vector_store_tests/test_bedrock_vector_store.py b/tests/vector_store_tests/test_bedrock_vector_store.py index 5f473fac00..be4f5bd80e 100644 --- a/tests/vector_store_tests/test_bedrock_vector_store.py +++ b/tests/vector_store_tests/test_bedrock_vector_store.py @@ -111,4 +111,115 @@ async def test_bedrock_search_with_router(): vector_store_id="T37J8R4WTM", custom_llm_provider="bedrock", ) - print(search_response) \ No newline at end of file + print(search_response) + + + +@pytest.mark.asyncio +async def test_bedrock_search_with_credentials_managed_registry(): + """ + Test that the vector store search uses the credential accessor from the registry + when AWS environment variables are not set, ensuring credentials are managed properly. + """ + from unittest.mock import patch, MagicMock + from litellm.router import Router + from litellm.types.vector_stores import LiteLLM_ManagedVectorStore + from litellm.types.utils import CredentialItem + from litellm.vector_stores.vector_store_registry import VectorStoreRegistry + from datetime import datetime, timezone + import litellm + + # Store original registry and credential list + original_registry = getattr(litellm, "vector_store_registry", None) + original_credential_list = getattr(litellm, "credential_list", []) + + try: + # Set up test AWS credentials in the credential system + test_credentials = CredentialItem( + credential_name="bedrock-litellm-website-knowledgebase", + credential_info={ + "provider": "aws", + "description": "Test AWS credentials for bedrock" + }, + credential_values={ + "aws_access_key_id": "test_access_key", + "aws_secret_access_key": "test_secret_key", + "aws_region_name": "us-east-1", + } + ) + + # Set up the credential list + litellm.credential_list = [test_credentials] + + # Create vector store with credential reference + vector_store = LiteLLM_ManagedVectorStore( + vector_store_id="T37J8R4WTM", + custom_llm_provider="bedrock", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + litellm_credential_name="bedrock-litellm-website-knowledgebase", + ) + + # Set up registry + registry = VectorStoreRegistry([vector_store]) + litellm.vector_store_registry = registry + + # Verify credentials can be retrieved from registry + retrieved_credentials = registry.get_credentials_for_vector_store("T37J8R4WTM") + assert retrieved_credentials, "Should retrieve credentials from registry" + assert retrieved_credentials.get("aws_access_key_id") == "test_access_key" + assert retrieved_credentials.get("aws_secret_access_key") == "test_secret_key" + assert retrieved_credentials.get("aws_region_name") == "us-east-1" + + # Create router and perform search + _router = Router(model_list=[]) + + # Mock the credential injection process to verify it's called + with patch.object(registry, 'get_credentials_for_vector_store', wraps=registry.get_credentials_for_vector_store) as mock_get_creds: + # Mock the actual search call to avoid making real API calls + with patch('litellm.vector_stores.main.base_llm_http_handler.vector_store_search_handler') as mock_handler: + mock_handler.return_value = { + "data": [ + { + "id": "test_result", + "text": "Mock search result", + "score": 0.9, + "metadata": {} + } + ] + } + + search_response = await _router.avector_store_search( + query="what happens after we add a model", + vector_store_id="T37J8R4WTM", + custom_llm_provider="bedrock", + ) + + # Verify the search was called + mock_handler.assert_called_once() + call_kwargs = mock_handler.call_args[1] + + # Verify that the credential accessor was called with the correct vector store ID + mock_get_creds.assert_called_with("T37J8R4WTM") + + # Verify the credentials were injected into the search call + litellm_params = call_kwargs.get("litellm_params", {}) + + # The key test: verify that credentials from the registry were used + # Since we have a registry with credentials, they should be present in the params + assert hasattr(litellm_params, 'aws_access_key_id'), "aws_access_key_id should be in litellm_params" + assert hasattr(litellm_params, 'aws_secret_access_key'), "aws_secret_access_key should be in litellm_params" + assert hasattr(litellm_params, 'aws_region_name'), "aws_region_name should be in litellm_params" + + # Verify we got the expected response + assert search_response["data"][0]["id"] == "test_result" + + print(f"✅ Test passed: Credential accessor was called with vector store ID: T37J8R4WTM") + print(f"✅ Retrieved credentials: {retrieved_credentials}") + print(f"✅ Credentials were injected into search call") + print(f"✅ Search completed successfully using registry credentials") + + finally: + # Restore original state + litellm.vector_store_registry = original_registry + litellm.credential_list = original_credential_list \ No newline at end of file diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 22f0e29d11..a6bb6245eb 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -8,7 +8,7 @@ import { Team } from "@/components/key_team_helpers/key_list" import Navbar from "@/components/navbar" import { ThemeProvider } from "@/contexts/ThemeContext" import UserDashboard from "@/components/user_dashboard" -import ModelDashboard from "@/components/model_dashboard" +import ModelDashboard from "@/components/templates/model_dashboard" import ViewUserDashboard from "@/components/view_users" import Teams from "@/components/teams" import Organizations from "@/components/organizations" diff --git a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx index c72495f29a..6d52292c9f 100644 --- a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx @@ -3,6 +3,7 @@ export enum Callbacks { CustomCallbackAPI = "Custom Callback API", Datadog = "Datadog", Langfuse = "Langfuse", + LangfuseOtel = "LangfuseOtel", LangSmith = "LangSmith", Lago = "Lago", OpenMeter = "OpenMeter", @@ -16,6 +17,7 @@ export const callback_map: Record = { CustomCallbackAPI: "custom_callback_api", Datadog: "datadog", Langfuse: "langfuse", + LangfuseOtel: "langfuse_otel", LangSmith: "langsmith", Lago: "lago", OpenMeter: "openmeter", @@ -45,6 +47,7 @@ interface CallbackInfo { logo: string; supports_key_team_logging: boolean; dynamic_params: Record; + description: string | null; } export const callbackInfo: Record = { @@ -55,7 +58,18 @@ export const callbackInfo: Record = { "langfuse_public_key": "text", "langfuse_secret_key": "password", "langfuse_host": "text" - } + }, + description: "Langfuse v2 Logging Integration" + }, + [Callbacks.LangfuseOtel]: { + logo: `${asset_logos_folder}langfuse.png`, + supports_key_team_logging: true, + dynamic_params: { + "langfuse_public_key": "text", + "langfuse_secret_key": "password", + "langfuse_host": "text" + }, + description: "Langfuse v3 OTEL Logging Integration" }, [Callbacks.Arize]: { logo: `${asset_logos_folder}arize.png`, @@ -63,7 +77,8 @@ export const callbackInfo: Record = { dynamic_params: { "arize_api_key": "password", "arize_space_id": "text", - } + }, + description: "Arize Logging Integration" }, [Callbacks.LangSmith]: { logo: `${asset_logos_folder}langsmith.png`, @@ -72,42 +87,50 @@ export const callbackInfo: Record = { "langsmith_api_key": "password", "langsmith_project": "text", "langsmith_base_url": "text" - } - }, + }, + description: "Langsmith Logging Integration" + }, [Callbacks.Braintrust]: { logo: `${asset_logos_folder}braintrust.png`, supports_key_team_logging: false, - dynamic_params: {} + dynamic_params: {}, + description: "Braintrust Logging Integration" }, [Callbacks.CustomCallbackAPI]: { logo: `${asset_logos_folder}custom.svg`, supports_key_team_logging: true, - dynamic_params: {} + dynamic_params: {}, + description: "Custom Callback API Logging Integration" }, [Callbacks.Datadog]: { logo: `${asset_logos_folder}datadog.png`, supports_key_team_logging: false, - dynamic_params: {} + dynamic_params: {}, + description: "Datadog Logging Integration" }, [Callbacks.Lago]: { logo: `${asset_logos_folder}lago.svg`, supports_key_team_logging: false, - dynamic_params: {} + dynamic_params: {}, + description: "Lago Billing Logging Integration" }, [Callbacks.OpenMeter]: { logo: `${asset_logos_folder}openmeter.png`, supports_key_team_logging: false, - dynamic_params: {} + dynamic_params: {}, + description: "OpenMeter Logging Integration" }, [Callbacks.OTel]: { logo: `${asset_logos_folder}otel.png`, supports_key_team_logging: false, - dynamic_params: {} + dynamic_params: {}, + description: "OpenTelemetry Logging Integration" }, [Callbacks.S3]: { logo: `${asset_logos_folder}aws.svg`, supports_key_team_logging: false, - dynamic_params: {} + dynamic_params: {}, + description: "S3 Bucket (AWS) Logging Integration" } }; \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/model_dashboard/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard/model_dashboard.tsx deleted file mode 100644 index 1f8146aa10..0000000000 --- a/ui/litellm-dashboard/src/components/model_dashboard/model_dashboard.tsx +++ /dev/null @@ -1,63 +0,0 @@ -import { useState } from 'react'; -import { Select, SelectItem, Text } from "@tremor/react"; - -interface ModelData { - team_id: string; - team_name: string; - // Add other properties as needed -} - -interface ModelDashboardProps { - modelData: ModelData[]; -} - -export default function ModelDashboard({ modelData }: ModelDashboardProps) { - const [selectedTeam, setSelectedTeam] = useState(null); - - const getTeamName = (teamId: string): string => { - const team = modelData.find(item => item.team_id === teamId); - return team?.team_name || 'Unknown Team'; - }; - - return ( -
-
-
- Model Management - Add and manage models for the proxy -
- -
-
- Filter by Public Model Name: - -
- -
- Filter by Team: - -
-
-
-
- ); -} \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/model_dashboard/columns.tsx b/ui/litellm-dashboard/src/components/molecules/models/columns.tsx similarity index 52% rename from ui/litellm-dashboard/src/components/model_dashboard/columns.tsx rename to ui/litellm-dashboard/src/components/molecules/models/columns.tsx index a2327dfe71..48034d6579 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/columns.tsx +++ b/ui/litellm-dashboard/src/components/molecules/models/columns.tsx @@ -1,10 +1,10 @@ import { ColumnDef } from "@tanstack/react-table"; import { Button, Badge, Icon } from "@tremor/react"; import { Tooltip } from "antd"; -import { getProviderLogoAndName } from "../provider_info_helpers"; -import { ModelData } from "./types"; -import { TrashIcon, PencilIcon, PencilAltIcon } from "@heroicons/react/outline"; -import DeleteModelButton from "../delete_model_button"; +import { getProviderLogoAndName } from "../../provider_info_helpers"; +import { ModelData } from "../../model_dashboard/types"; +import { TrashIcon, PencilIcon, PencilAltIcon, KeyIcon } from "@heroicons/react/outline"; +import DeleteModelButton from "../../delete_model_button"; import { useState } from "react"; export const columns = ( @@ -21,7 +21,7 @@ export const columns = ( setExpandedRows: (expandedRows: Set) => void, ): ColumnDef[] => [ { - header: "Model ID", + header: () => Model ID, accessorKey: "model_info.id", cell: ({ row }) => { const model = row.original; @@ -38,77 +38,117 @@ export const columns = ( }, }, { - header: "Public Model Name", + header: () => Model Information, accessorKey: "model_name", + size: 250, // Fixed column width cell: ({ row }) => { + const model = row.original; const displayName = getDisplayModelName(row.original) || "-"; + const tooltipContent = ( +
+
Provider: {model.provider || "-"}
+
Public Model Name: {displayName}
+
LiteLLM Model Name: {model.litellm_model_name || "-"}
+
+ ); + return ( - -
- {displayName} + +
+ {/* Provider Icon */} +
+ {model.provider ? ( + {`${model.provider} { + const target = e.target as HTMLImageElement; + const parent = target.parentElement; + if (parent) { + const fallbackDiv = document.createElement('div'); + fallbackDiv.className = 'w-4 h-4 rounded-full bg-gray-200 flex items-center justify-center text-xs'; + fallbackDiv.textContent = model.provider?.charAt(0) || '-'; + parent.replaceChild(fallbackDiv, target); + } + }} + /> + ) : ( +
+ - +
+ )} +
+ + {/* Model Names Container */} +
+ {/* Public Model Name */} +
+ {displayName} +
+ {/* LiteLLM Model Name */} +
+ {model.litellm_model_name || "-"} +
+
); }, }, { - header: "Provider", - accessorKey: "provider", + header: () => Credentials, + accessorKey: "litellm_credential_name", + size: 180, // Fixed column width cell: ({ row }) => { const model = row.original; - return ( -
- {model.provider && ( - {`${model.provider} { - const target = e.target as HTMLImageElement; - const parent = target.parentElement; - if (parent) { - const fallbackDiv = document.createElement('div'); - fallbackDiv.className = 'w-4 h-4 rounded-full bg-gray-200 flex items-center justify-center text-xs'; - fallbackDiv.textContent = model.provider?.charAt(0) || '-'; - parent.replaceChild(fallbackDiv, target); - } - }} - /> - )} -

{model.provider || "-"}

+ const credentialName = model.litellm_params?.litellm_credential_name; + + return credentialName ? ( + +
+ + + {credentialName} + +
+ + ) : ( +
+ + No credentials
); }, }, { - header: "LiteLLM Model Name", - accessorKey: "litellm_model_name", - cell: ({ row }) => { - const model = row.original; - return ( - -
- {model.litellm_model_name || "-"} -
- - ); - }, - }, - { - header: "Created At", - accessorKey: "model_info.created_at", + header: () => Created By, + accessorKey: "model_info.created_by", sortingFn: "datetime", + size: 160, // Fixed column width cell: ({ row }) => { const model = row.original; + const createdBy = model.model_info.created_by; + const createdAt = model.model_info.created_at + ? new Date(model.model_info.created_at).toLocaleDateString() + : null; + return ( - - {model.model_info.created_at ? new Date(model.model_info.created_at).toLocaleDateString() : "-"} - +
+ {/* Created By - Primary */} +
+ {createdBy || "Unknown"} +
+ {/* Created At - Secondary */} +
+ {createdAt || "Unknown date"} +
+
); }, }, { - header: "Updated At", + header: () => Updated At, accessorKey: "model_info.updated_at", sortingFn: "datetime", cell: ({ row }) => { @@ -121,51 +161,45 @@ export const columns = ( }, }, { - header: "Created By", - accessorKey: "model_info.created_by", - cell: ({ row }) => { - const model = row.original; - return ( - - {model.model_info.created_by || "-"} - - ); - }, - }, - { - header: () => ( - - Input Cost - - ), + header: () => Costs, accessorKey: "input_cost", + size: 120, // Fixed column width cell: ({ row }) => { const model = row.original; + const inputCost = model.input_cost; + const outputCost = model.output_cost; + + // If both costs are missing or undefined, show "-" + if (!inputCost && !outputCost) { + return ( +
+ - +
+ ); + } + return ( - 
-          {model.input_cost || "-"}
-
+ +
+ {/* Input Cost - Primary */} + {inputCost && ( +
+ In: ${inputCost} +
+ )} + {/* Output Cost - Secondary */} + {outputCost && ( +
+ Out: ${outputCost} +
+ )} +
+ ); }, }, { - header: () => ( - - Output Cost - - ), - accessorKey: "output_cost", - cell: ({ row }) => { - const model = row.original; - return ( - 
-          {model.output_cost || "-"}
-
- ); - }, - }, - { - header: "Team ID", + header: () => Team ID, accessorKey: "model_info.team_id", cell: ({ row }) => { const model = row.original; @@ -188,7 +222,7 @@ export const columns = ( }, }, { - header: "Model Access Group", + header: () => Model Access Group, accessorKey: "model_info.model_access_group", enableSorting: false, cell: ({ row }) => { @@ -252,23 +286,7 @@ export const columns = ( }, }, { - header: "Credentials", - accessorKey: "litellm_credential_name", - cell: ({ row }) => { - const model = row.original; - return model.litellm_params && model.litellm_params.litellm_credential_name ? ( -
- - {model.litellm_params.litellm_credential_name.slice(0, 7)}... - -
- ) : ( - - - ); - }, - }, - { - header: "Status", + header: () => Status, accessorKey: "model_info.db_model", cell: ({ row }) => { const model = row.original; @@ -292,17 +310,6 @@ export const columns = ( const canEditModel = userRole === "Admin" || model.model_info?.created_by === userID; return (
- { - if (canEditModel) { - setSelectedModelId(model.model_info.id); - setEditModel(true); - } - }} - className={!canEditModel ? "opacity-50 cursor-not-allowed" : "cursor-pointer"} - /> = ({ addForm.validateFields().then((values) => { // Call API to add the callback let payload; - if (values.callback === "langfuse") { + if (values.callback === "langfuse" || values.callback === "langfuse_otel") { payload = { environment_variables: { LANGFUSE_PUBLIC_KEY: values.langfusePublicKey, diff --git a/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx b/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx index bfea1e7968..012e4493fa 100644 --- a/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx +++ b/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx @@ -164,18 +164,21 @@ const LoggingSettings: React.FC = ({ > {allCallbacks.map((callbackName) => { const logo = callbackInfo[callbackName]?.logo; + const description = callbackInfo[callbackName]?.description; return ( ); })} @@ -257,18 +260,21 @@ const LoggingSettings: React.FC = ({ > {supportedCallbacks.map((callbackName) => { const logo = callbackInfo[callbackName]?.logo; + const description = callbackInfo[callbackName]?.description; return ( ); })} diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/templates/model_dashboard.tsx similarity index 52% rename from ui/litellm-dashboard/src/components/model_dashboard.tsx rename to ui/litellm-dashboard/src/components/templates/model_dashboard.tsx index 3734486a4e..ce303a12a8 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/templates/model_dashboard.tsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect, useRef } from "react"; +import React, { useState, useEffect, useRef } from "react" import { Card, Title, @@ -14,27 +14,16 @@ import { Col, DateRangePicker, TextInput, -} from "@tremor/react"; -import { - CredentialItem, - credentialListCall, - CredentialsResponse, -} from "./networking"; +} from "@tremor/react" +import { CredentialItem, credentialListCall, CredentialsResponse } from "../networking" -import { handleAddModelSubmit } from "./add_model/handle_add_model_submit"; +import { handleAddModelSubmit } from "../add_model/handle_add_model_submit" -import CredentialsPanel from "@/components/model_add/credentials"; -import { getDisplayModelName } from "./view_model/model_name_display"; -import { - TabPanel, - TabPanels, - TabGroup, - TabList, - Tab, - Icon, -} from "@tremor/react"; -import { Select, SelectItem, DateRangePickerValue } from "@tremor/react"; -import UsageDatePicker from "./shared/usage_date_picker"; +import CredentialsPanel from "@/components/model_add/credentials" +import { getDisplayModelName } from "../view_model/model_name_display" +import { TabPanel, TabPanels, TabGroup, TabList, Tab, Icon } from "@tremor/react" +import { Select, SelectItem, DateRangePickerValue } from "@tremor/react" +import UsageDatePicker from "../shared/usage_date_picker" import { modelInfoCall, Model, @@ -50,73 +39,68 @@ import { adminGlobalActivityExceptions, adminGlobalActivityExceptionsPerDeployment, allEndUsersCall, -} from "./networking"; -import { BarChart, AreaChart } from "@tremor/react"; -import { Popover, Form, InputNumber, message } from "antd"; -import { Button } from "@tremor/react"; -import { Typography } from "antd"; -import { RefreshIcon, FilterIcon } from "@heroicons/react/outline"; -import { InfoCircleOutlined } from "@ant-design/icons"; -import type { UploadProps } from "antd"; -import TimeToFirstToken from "./model_metrics/time_to_first_token"; -import { Team } from "./key_team_helpers/key_list"; -import TeamInfoView from "./team/team_info"; -import { - Providers, - provider_map, - getPlaceholder, - getProviderModels, -} from "./provider_info_helpers"; -import ModelInfoView from "./model_info_view"; -import AddModelTab from "./add_model/add_model_tab"; +} from "../networking" +import { BarChart, AreaChart } from "@tremor/react" +import { Popover, Form, InputNumber, message } from "antd" +import { Button } from "@tremor/react" +import { Typography } from "antd" +import { RefreshIcon, FilterIcon } from "@heroicons/react/outline" +import { InfoCircleOutlined } from "@ant-design/icons" +import type { UploadProps } from "antd" +import TimeToFirstToken from "../model_metrics/time_to_first_token" +import { Team } from "../key_team_helpers/key_list" +import TeamInfoView from "../team/team_info" +import { Providers, provider_map, getPlaceholder, getProviderModels } from "../provider_info_helpers" +import ModelInfoView from "../model_info_view" +import AddModelTab from "../add_model/add_model_tab" -import { ModelDataTable } from "./model_dashboard/table"; -import { columns } from "./model_dashboard/columns"; -import PriceDataReload from "./price_data_reload"; -import HealthCheckComponent from "./model_dashboard/HealthCheckComponent"; -import PassThroughSettings from "./pass_through_settings"; -import ModelGroupAliasSettings from "./model_group_alias_settings"; -import { all_admin_roles } from "@/utils/roles"; -import { Table as TableInstance } from "@tanstack/react-table"; -import NotificationManager from "./molecules/notifications_manager"; +import { ModelDataTable } from "../model_dashboard/table" +import { columns } from "../molecules/models/columns" +import PriceDataReload from "../price_data_reload" +import HealthCheckComponent from "../model_dashboard/HealthCheckComponent" +import PassThroughSettings from "../pass_through_settings" +import ModelGroupAliasSettings from "../model_group_alias_settings" +import { all_admin_roles } from "@/utils/roles" +import { Table as TableInstance } from "@tanstack/react-table" +import NotificationManager from "../molecules/notifications_manager" interface ModelDashboardProps { - accessToken: string | null; - token: string | null; - userRole: string | null; - userID: string | null; - modelData: any; - keys: any[] | null; - setModelData: any; - premiumUser: boolean; - teams: Team[] | null; + accessToken: string | null + token: string | null + userRole: string | null + userID: string | null + modelData: any + keys: any[] | null + setModelData: any + premiumUser: boolean + teams: Team[] | null } interface RetryPolicyObject { - [key: string]: { [retryPolicyKey: string]: number } | undefined; + [key: string]: { [retryPolicyKey: string]: number } | undefined } interface GlobalRetryPolicyObject { - [retryPolicyKey: string]: number; + [retryPolicyKey: string]: number } interface GlobalExceptionActivityData { - sum_num_rate_limit_exceptions: number; - daily_data: { date: string; num_rate_limit_exceptions: number }[]; + sum_num_rate_limit_exceptions: number + daily_data: { date: string; num_rate_limit_exceptions: number }[] } //["OpenAI", "Azure OpenAI", "Anthropic", "Gemini (Google AI Studio)", "Amazon Bedrock", "OpenAI-Compatible Endpoints (Groq, Together AI, Mistral AI, etc.)"] interface ProviderFields { - field_name: string; - field_type: string; - field_description: string; - field_value: string; + field_name: string + field_type: string + field_description: string + field_value: string } interface ProviderSettings { - name: string; - fields: ProviderFields[]; + name: string + fields: ProviderFields[] } const retry_policy_map: Record = { @@ -126,7 +110,7 @@ const retry_policy_map: Record = { "RateLimitError (429)": "RateLimitErrorRetries", "ContentPolicyViolationError (400)": "ContentPolicyViolationErrorRetries", "InternalServerError (500)": "InternalServerErrorRetries", -}; +} const ModelDashboard: React.FC = ({ accessToken, @@ -139,142 +123,115 @@ const ModelDashboard: React.FC = ({ premiumUser, teams, }) => { - const [addModelForm] = Form.useForm(); - const [autoRouterForm] = Form.useForm(); - const [modelMap, setModelMap] = useState(null); - const [lastRefreshed, setLastRefreshed] = useState(""); + const [addModelForm] = Form.useForm() + const [autoRouterForm] = Form.useForm() + const [modelMap, setModelMap] = useState(null) + const [lastRefreshed, setLastRefreshed] = useState("") - const [providerModels, setProviderModels] = useState>([]); // Explicitly typing providerModels as a string array + const [providerModels, setProviderModels] = useState>([]) // Explicitly typing providerModels as a string array - const [providerSettings, setProviderSettings] = useState( - [] - ); - const [selectedProvider, setSelectedProvider] = useState( - Providers.OpenAI - ); - const [healthCheckResponse, setHealthCheckResponse] = useState(null); - const [isHealthCheckLoading, setIsHealthCheckLoading] = - useState(false); - const [editModalVisible, setEditModalVisible] = useState(false); + const [providerSettings, setProviderSettings] = useState([]) + const [selectedProvider, setSelectedProvider] = useState(Providers.OpenAI) + const [healthCheckResponse, setHealthCheckResponse] = useState(null) + const [isHealthCheckLoading, setIsHealthCheckLoading] = useState(false) + const [editModalVisible, setEditModalVisible] = useState(false) - const [selectedModel, setSelectedModel] = useState(null); - const [availableModelGroups, setAvailableModelGroups] = useState< - Array - >([]); - const [availableModelAccessGroups, setAvailableModelAccessGroups] = useState< - Array - >([]); - const [selectedModelGroup, setSelectedModelGroup] = useState( - null - ); - const [modelMetrics, setModelMetrics] = useState([]); - const [modelMetricsCategories, setModelMetricsCategories] = useState( - [] - ); - const [streamingModelMetrics, setStreamingModelMetrics] = useState([]); - const [streamingModelMetricsCategories, setStreamingModelMetricsCategories] = - useState([]); - const [modelExceptions, setModelExceptions] = useState([]); - const [allExceptions, setAllExceptions] = useState([]); - const [slowResponsesData, setSlowResponsesData] = useState([]); + const [selectedModel, setSelectedModel] = useState(null) + const [availableModelGroups, setAvailableModelGroups] = useState>([]) + const [availableModelAccessGroups, setAvailableModelAccessGroups] = useState>([]) + const [selectedModelGroup, setSelectedModelGroup] = useState(null) + const [modelMetrics, setModelMetrics] = useState([]) + const [modelMetricsCategories, setModelMetricsCategories] = useState([]) + const [streamingModelMetrics, setStreamingModelMetrics] = useState([]) + const [streamingModelMetricsCategories, setStreamingModelMetricsCategories] = useState([]) + const [modelExceptions, setModelExceptions] = useState([]) + const [allExceptions, setAllExceptions] = useState([]) + const [slowResponsesData, setSlowResponsesData] = useState([]) const [dateValue, setDateValue] = useState({ from: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), to: new Date(), - }); + }) - const [modelGroupRetryPolicy, setModelGroupRetryPolicy] = - useState(null); - const [globalRetryPolicy, setGlobalRetryPolicy] = useState(null); - const [defaultRetry, setDefaultRetry] = useState(0); + const [modelGroupRetryPolicy, setModelGroupRetryPolicy] = useState(null) + const [globalRetryPolicy, setGlobalRetryPolicy] = useState(null) + const [defaultRetry, setDefaultRetry] = useState(0) - const [globalExceptionData, setGlobalExceptionData] = - useState({} as GlobalExceptionActivityData); - const [globalExceptionPerDeployment, setGlobalExceptionPerDeployment] = - useState([]); + const [globalExceptionData, setGlobalExceptionData] = useState( + {} as GlobalExceptionActivityData, + ) + const [globalExceptionPerDeployment, setGlobalExceptionPerDeployment] = useState([]) - const [showAdvancedFilters, setShowAdvancedFilters] = - useState(false); - const [selectedAPIKey, setSelectedAPIKey] = useState(null); - const [selectedCustomer, setSelectedCustomer] = useState(null); + const [showAdvancedFilters, setShowAdvancedFilters] = useState(false) + const [selectedAPIKey, setSelectedAPIKey] = useState(null) + const [selectedCustomer, setSelectedCustomer] = useState(null) - const [allEndUsers, setAllEndUsers] = useState([]); + const [allEndUsers, setAllEndUsers] = useState([]) - const [credentialsList, setCredentialsList] = useState([]); + const [credentialsList, setCredentialsList] = useState([]) // Model Group Alias state - const [modelGroupAlias, setModelGroupAlias] = useState<{[key: string]: string}>({}); + const [modelGroupAlias, setModelGroupAlias] = useState<{ [key: string]: string }>({}) // Add state for advanced settings visibility - const [showAdvancedSettings, setShowAdvancedSettings] = - useState(false); + const [showAdvancedSettings, setShowAdvancedSettings] = useState(false) // Add these state variables - const [selectedModelId, setSelectedModelId] = useState(null); - const [editModel, setEditModel] = useState(false); + const [selectedModelId, setSelectedModelId] = useState(null) + const [editModel, setEditModel] = useState(false) - const [selectedTeamId, setSelectedTeamId] = useState(null); - const [selectedTeam, setSelectedTeam] = useState(null); + const [selectedTeamId, setSelectedTeamId] = useState(null) + const [selectedTeam, setSelectedTeam] = useState(null) - const [selectedTeamFilter, setSelectedTeamFilter] = useState( - null - ); - const [selectedModelAccessGroupFilter, setSelectedModelAccessGroupFilter] = - useState(null); + const [selectedTeamFilter, setSelectedTeamFilter] = useState(null) + const [selectedModelAccessGroupFilter, setSelectedModelAccessGroupFilter] = useState(null) - const [modelNameSearch, setModelNameSearch] = useState(""); + const [modelNameSearch, setModelNameSearch] = useState("") // Add new state for current team and model view mode - const [currentTeam, setCurrentTeam] = useState("personal"); // 'personal' or team_id - const [modelViewMode, setModelViewMode] = useState<"current_team" | "all">( - "current_team" - ); + const [currentTeam, setCurrentTeam] = useState("personal") // 'personal' or team_id + const [modelViewMode, setModelViewMode] = useState<"current_team" | "all">("current_team") - const [showColumnDropdown, setShowColumnDropdown] = useState(false); + const [showColumnDropdown, setShowColumnDropdown] = useState(false) - const [isDropdownOpen, setIsDropdownOpen] = useState(false); - const [expandedRows, setExpandedRows] = useState>(new Set()); - const dropdownRef = useRef(null); - const tableRef = useRef>(null); - const [selectedTabIndex, setSelectedTabIndex] = useState(0); + const [isDropdownOpen, setIsDropdownOpen] = useState(false) + const [expandedRows, setExpandedRows] = useState>(new Set()) + const dropdownRef = useRef(null) + const tableRef = useRef>(null) + const [selectedTabIndex, setSelectedTabIndex] = useState(0) const handleCreateNewModelClick = () => { if (selectedModelId) { - setSelectedModelId(null); + setSelectedModelId(null) } - setSelectedTabIndex(1); - }; + setSelectedTabIndex(1) + } const setProviderModelsFn = (provider: Providers) => { - const _providerModels = getProviderModels(provider, modelMap); - setProviderModels(_providerModels); - console.log(`providerModels: ${_providerModels}`); - }; + const _providerModels = getProviderModels(provider, modelMap) + setProviderModels(_providerModels) + console.log(`providerModels: ${_providerModels}`) + } const updateModelMetrics = async ( modelGroup: string | null, startTime: Date | undefined, - endTime: Date | undefined + endTime: Date | undefined, ) => { - console.log("Updating model metrics for group:", modelGroup); + console.log("Updating model metrics for group:", modelGroup) if (!accessToken || !userID || !userRole || !startTime || !endTime) { - return; + return } - console.log( - "inside updateModelMetrics - startTime:", - startTime, - "endTime:", - endTime - ); - setSelectedModelGroup(modelGroup); + console.log("inside updateModelMetrics - startTime:", startTime, "endTime:", endTime) + setSelectedModelGroup(modelGroup) - let selected_token = selectedAPIKey?.token; + let selected_token = selectedAPIKey?.token if (selected_token === undefined) { - selected_token = null; + selected_token = null } - let selected_customer = selectedCustomer; + let selected_customer = selectedCustomer if (selected_customer === undefined) { - selected_customer = null; + selected_customer = null } try { @@ -286,26 +243,24 @@ const ModelDashboard: React.FC = ({ startTime.toISOString(), endTime.toISOString(), selected_token, - selected_customer - ); - console.log("Model metrics response:", modelMetricsResponse); + selected_customer, + ) + console.log("Model metrics response:", modelMetricsResponse) // Assuming modelMetricsResponse now contains the metric data for the specified model group - setModelMetrics(modelMetricsResponse.data); - setModelMetricsCategories(modelMetricsResponse.all_api_bases); + setModelMetrics(modelMetricsResponse.data) + setModelMetricsCategories(modelMetricsResponse.all_api_bases) const streamingModelMetricsResponse = await streamingModelMetricsCall( accessToken, modelGroup, startTime.toISOString(), - endTime.toISOString() - ); + endTime.toISOString(), + ) // Assuming modelMetricsResponse now contains the metric data for the specified model group - setStreamingModelMetrics(streamingModelMetricsResponse.data); - setStreamingModelMetricsCategories( - streamingModelMetricsResponse.all_api_bases - ); + setStreamingModelMetrics(streamingModelMetricsResponse.data) + setStreamingModelMetricsCategories(streamingModelMetricsResponse.all_api_bases) const modelExceptionsResponse = await modelExceptionsCall( accessToken, @@ -315,11 +270,11 @@ const ModelDashboard: React.FC = ({ startTime.toISOString(), endTime.toISOString(), selected_token, - selected_customer - ); - console.log("Model exceptions response:", modelExceptionsResponse); - setModelExceptions(modelExceptionsResponse.data); - setAllExceptions(modelExceptionsResponse.exception_types); + selected_customer, + ) + console.log("Model exceptions response:", modelExceptionsResponse) + setModelExceptions(modelExceptionsResponse.data) + setAllExceptions(modelExceptionsResponse.exception_types) const slowResponses = await modelMetricsSlowResponsesCall( accessToken, @@ -329,226 +284,209 @@ const ModelDashboard: React.FC = ({ startTime.toISOString(), endTime.toISOString(), selected_token, - selected_customer - ); + selected_customer, + ) - console.log("slowResponses:", slowResponses); + console.log("slowResponses:", slowResponses) - setSlowResponsesData(slowResponses); + setSlowResponsesData(slowResponses) if (modelGroup) { const dailyExceptions = await adminGlobalActivityExceptions( accessToken, startTime?.toISOString().split("T")[0], endTime?.toISOString().split("T")[0], - modelGroup - ); + modelGroup, + ) - setGlobalExceptionData(dailyExceptions); + setGlobalExceptionData(dailyExceptions) - const dailyExceptionsPerDeplyment = - await adminGlobalActivityExceptionsPerDeployment( - accessToken, - startTime?.toISOString().split("T")[0], - endTime?.toISOString().split("T")[0], - modelGroup - ); + const dailyExceptionsPerDeplyment = await adminGlobalActivityExceptionsPerDeployment( + accessToken, + startTime?.toISOString().split("T")[0], + endTime?.toISOString().split("T")[0], + modelGroup, + ) - setGlobalExceptionPerDeployment(dailyExceptionsPerDeplyment); + setGlobalExceptionPerDeployment(dailyExceptionsPerDeplyment) } } catch (error) { - console.error("Failed to fetch model metrics", error); + console.error("Failed to fetch model metrics", error) } - }; + } const fetchCredentials = async (accessToken: string) => { try { - const response: CredentialsResponse = - await credentialListCall(accessToken); - console.log(`credentials: ${JSON.stringify(response)}`); - setCredentialsList(response.credentials); + const response: CredentialsResponse = await credentialListCall(accessToken) + console.log(`credentials: ${JSON.stringify(response)}`) + setCredentialsList(response.credentials) } catch (error) { - console.error("Error fetching credentials:", error); + console.error("Error fetching credentials:", error) } - }; + } useEffect(() => { - updateModelMetrics(selectedModelGroup, dateValue.from, dateValue.to); - }, [selectedAPIKey, selectedCustomer, selectedTeam]); + updateModelMetrics(selectedModelGroup, dateValue.from, dateValue.to) + }, [selectedAPIKey, selectedCustomer, selectedTeam]) useEffect(() => { const handleClickOutside = (event: MouseEvent) => { - if ( - dropdownRef.current && - !dropdownRef.current.contains(event.target as Node) - ) { - setIsDropdownOpen(false); + if (dropdownRef.current && !dropdownRef.current.contains(event.target as Node)) { + setIsDropdownOpen(false) } - }; + } - document.addEventListener("mousedown", handleClickOutside); - return () => document.removeEventListener("mousedown", handleClickOutside); - }, []); + document.addEventListener("mousedown", handleClickOutside) + return () => document.removeEventListener("mousedown", handleClickOutside) + }, []) function formatCreatedAt(createdAt: string | null) { if (createdAt) { - const date = new Date(createdAt); - const options = { month: "long", day: "numeric", year: "numeric" }; - return date.toLocaleDateString("en-US"); + const date = new Date(createdAt) + const options = { month: "long", day: "numeric", year: "numeric" } + return date.toLocaleDateString("en-US") } - return null; + return null } const handleEditClick = (model: any) => { - setSelectedModel(model); - setEditModalVisible(true); - }; + setSelectedModel(model) + setEditModalVisible(true) + } const handleEditCancel = () => { - setEditModalVisible(false); - setSelectedModel(null); - }; + setEditModalVisible(false) + setSelectedModel(null) + } const uploadProps: UploadProps = { name: "file", accept: ".json", beforeUpload: (file) => { if (file.type === "application/json") { - const reader = new FileReader(); + const reader = new FileReader() reader.onload = (e) => { if (e.target) { - const jsonStr = e.target.result as string; - console.log( - `Resetting vertex_credentials to JSON; jsonStr: ${jsonStr}` - ); - addModelForm.setFieldsValue({ vertex_credentials: jsonStr }); - console.log( - "Form values right after setting:", - addModelForm.getFieldsValue() - ); + const jsonStr = e.target.result as string + console.log(`Resetting vertex_credentials to JSON; jsonStr: ${jsonStr}`) + addModelForm.setFieldsValue({ vertex_credentials: jsonStr }) + console.log("Form values right after setting:", addModelForm.getFieldsValue()) } - }; - reader.readAsText(file); + } + reader.readAsText(file) } // Prevent upload - return false; + return false }, onChange(info) { - console.log("Upload onChange triggered with values:", info); - console.log("Current form values:", addModelForm.getFieldsValue()); + console.log("Upload onChange triggered with values:", info) + console.log("Current form values:", addModelForm.getFieldsValue()) if (info.file.status !== "uploading") { - console.log(info.file, info.fileList); + console.log(info.file, info.fileList) } if (info.file.status === "done") { - message.success(`${info.file.name} file uploaded successfully`); + message.success(`${info.file.name} file uploaded successfully`) } else if (info.file.status === "error") { - NotificationManager.fromBackend(`${info.file.name} file upload failed.`); + NotificationManager.fromBackend(`${info.file.name} file upload failed.`) } }, - }; + } const handleRefreshClick = () => { // Update the 'lastRefreshed' state to the current date and time - const currentDate = new Date(); - setLastRefreshed(currentDate.toLocaleString()); - }; + const currentDate = new Date() + setLastRefreshed(currentDate.toLocaleString()) + } const handleSaveRetrySettings = async () => { if (!accessToken) { - console.error("Access token is missing"); - return; + console.error("Access token is missing") + return } try { const payload: any = { router_settings: {}, - }; + } if (selectedModelGroup === "global") { // Only update global retry policy - console.log("Saving global retry policy:", globalRetryPolicy); + console.log("Saving global retry policy:", globalRetryPolicy) if (globalRetryPolicy) { - payload.router_settings.retry_policy = globalRetryPolicy; + payload.router_settings.retry_policy = globalRetryPolicy } - message.success("Global retry settings saved successfully"); + message.success("Global retry settings saved successfully") } else { // Only update model group retry policy - console.log("Saving model group retry policy for", selectedModelGroup, ":", modelGroupRetryPolicy); + console.log("Saving model group retry policy for", selectedModelGroup, ":", modelGroupRetryPolicy) if (modelGroupRetryPolicy) { - payload.router_settings.model_group_retry_policy = modelGroupRetryPolicy; + payload.router_settings.model_group_retry_policy = modelGroupRetryPolicy } - message.success(`Retry settings saved successfully for ${selectedModelGroup}`); + message.success(`Retry settings saved successfully for ${selectedModelGroup}`) } - await setCallbacksCall(accessToken, payload); + await setCallbacksCall(accessToken, payload) } catch (error) { - console.error("Failed to save retry settings:", error); - NotificationManager.fromBackend("Failed to save retry settings"); + console.error("Failed to save retry settings:", error) + NotificationManager.fromBackend("Failed to save retry settings") } - }; - - + } useEffect(() => { if (!accessToken || !token || !userRole || !userID) { - return; + return } const fetchData = async () => { try { // Replace with your actual API call for model data - const modelDataResponse = await modelInfoCall( - accessToken, - userID, - userRole - ); - console.log("Model data response:", modelDataResponse.data); - setModelData(modelDataResponse); - const _providerSettings = await modelSettingsCall(accessToken); + const modelDataResponse = await modelInfoCall(accessToken, userID, userRole) + console.log("Model data response:", modelDataResponse.data) + setModelData(modelDataResponse) + const _providerSettings = await modelSettingsCall(accessToken) if (_providerSettings) { - setProviderSettings(_providerSettings); + setProviderSettings(_providerSettings) } // loop through modelDataResponse and get all`model_name` values - let all_model_groups: Set = new Set(); + let all_model_groups: Set = new Set() for (let i = 0; i < modelDataResponse.data.length; i++) { - const model = modelDataResponse.data[i]; - all_model_groups.add(model.model_name); + const model = modelDataResponse.data[i] + all_model_groups.add(model.model_name) } - console.log("all_model_groups:", all_model_groups); - let _array_model_groups = Array.from(all_model_groups); + console.log("all_model_groups:", all_model_groups) + let _array_model_groups = Array.from(all_model_groups) // sort _array_model_groups alphabetically - _array_model_groups = _array_model_groups.sort(); + _array_model_groups = _array_model_groups.sort() - setAvailableModelGroups(_array_model_groups); + setAvailableModelGroups(_array_model_groups) - let all_model_access_groups: Set = new Set(); + let all_model_access_groups: Set = new Set() for (let i = 0; i < modelDataResponse.data.length; i++) { - const model = modelDataResponse.data[i]; - let model_info: any | null = model.model_info; + const model = modelDataResponse.data[i] + let model_info: any | null = model.model_info if (model_info) { - let access_groups = model_info.access_groups; + let access_groups = model_info.access_groups if (access_groups) { for (let j = 0; j < access_groups.length; j++) { - all_model_access_groups.add(access_groups[j]); + all_model_access_groups.add(access_groups[j]) } } } } - setAvailableModelAccessGroups(Array.from(all_model_access_groups)); + setAvailableModelAccessGroups(Array.from(all_model_access_groups)) - console.log("array_model_groups:", _array_model_groups); - let _initial_model_group = "all"; + console.log("array_model_groups:", _array_model_groups) + let _initial_model_group = "all" if (_array_model_groups.length > 0) { // set selectedModelGroup to the last model group - _initial_model_group = - _array_model_groups[_array_model_groups.length - 1]; - console.log("_initial_model_group:", _initial_model_group); + _initial_model_group = _array_model_groups[_array_model_groups.length - 1] + console.log("_initial_model_group:", _initial_model_group) //setSelectedModelGroup(_initial_model_group); } - console.log("selectedModelGroup:", selectedModelGroup); + console.log("selectedModelGroup:", selectedModelGroup) const modelMetricsResponse = await modelMetricsCall( accessToken, @@ -558,27 +496,25 @@ const ModelDashboard: React.FC = ({ dateValue.from?.toISOString(), dateValue.to?.toISOString(), selectedAPIKey?.token, - selectedCustomer - ); + selectedCustomer, + ) - console.log("Model metrics response:", modelMetricsResponse); + console.log("Model metrics response:", modelMetricsResponse) // Sort by latency (avg_latency_per_token) - setModelMetrics(modelMetricsResponse.data); - setModelMetricsCategories(modelMetricsResponse.all_api_bases); + setModelMetrics(modelMetricsResponse.data) + setModelMetricsCategories(modelMetricsResponse.all_api_bases) const streamingModelMetricsResponse = await streamingModelMetricsCall( accessToken, _initial_model_group, dateValue.from?.toISOString(), - dateValue.to?.toISOString() - ); + dateValue.to?.toISOString(), + ) // Assuming modelMetricsResponse now contains the metric data for the specified model group - setStreamingModelMetrics(streamingModelMetricsResponse.data); - setStreamingModelMetricsCategories( - streamingModelMetricsResponse.all_api_bases - ); + setStreamingModelMetrics(streamingModelMetricsResponse.data) + setStreamingModelMetricsCategories(streamingModelMetricsResponse.all_api_bases) const modelExceptionsResponse = await modelExceptionsCall( accessToken, @@ -588,11 +524,11 @@ const ModelDashboard: React.FC = ({ dateValue.from?.toISOString(), dateValue.to?.toISOString(), selectedAPIKey?.token, - selectedCustomer - ); - console.log("Model exceptions response:", modelExceptionsResponse); - setModelExceptions(modelExceptionsResponse.data); - setAllExceptions(modelExceptionsResponse.exception_types); + selectedCustomer, + ) + console.log("Model exceptions response:", modelExceptionsResponse) + setModelExceptions(modelExceptionsResponse.data) + setAllExceptions(modelExceptionsResponse.exception_types) const slowResponses = await modelMetricsSlowResponsesCall( accessToken, @@ -602,118 +538,102 @@ const ModelDashboard: React.FC = ({ dateValue.from?.toISOString(), dateValue.to?.toISOString(), selectedAPIKey?.token, - selectedCustomer - ); + selectedCustomer, + ) const dailyExceptions = await adminGlobalActivityExceptions( accessToken, dateValue.from?.toISOString().split("T")[0], dateValue.to?.toISOString().split("T")[0], - _initial_model_group - ); + _initial_model_group, + ) - setGlobalExceptionData(dailyExceptions); + setGlobalExceptionData(dailyExceptions) - const dailyExceptionsPerDeplyment = - await adminGlobalActivityExceptionsPerDeployment( - accessToken, - dateValue.from?.toISOString().split("T")[0], - dateValue.to?.toISOString().split("T")[0], - _initial_model_group - ); - - setGlobalExceptionPerDeployment(dailyExceptionsPerDeplyment); - - console.log("dailyExceptions:", dailyExceptions); - - console.log( - "dailyExceptionsPerDeplyment:", - dailyExceptionsPerDeplyment - ); - - console.log("slowResponses:", slowResponses); - - setSlowResponsesData(slowResponses); - - let all_end_users_data = await allEndUsersCall(accessToken); - - setAllEndUsers(all_end_users_data?.end_users); - - const routerSettingsInfo = await getCallbacksCall( + const dailyExceptionsPerDeplyment = await adminGlobalActivityExceptionsPerDeployment( accessToken, - userID, - userRole - ); + dateValue.from?.toISOString().split("T")[0], + dateValue.to?.toISOString().split("T")[0], + _initial_model_group, + ) - let router_settings = routerSettingsInfo.router_settings; + setGlobalExceptionPerDeployment(dailyExceptionsPerDeplyment) - console.log("routerSettingsInfo:", router_settings); + console.log("dailyExceptions:", dailyExceptions) - let model_group_retry_policy = router_settings.model_group_retry_policy; - let default_retries = router_settings.num_retries; + console.log("dailyExceptionsPerDeplyment:", dailyExceptionsPerDeplyment) + + console.log("slowResponses:", slowResponses) + + setSlowResponsesData(slowResponses) + + let all_end_users_data = await allEndUsersCall(accessToken) + + setAllEndUsers(all_end_users_data?.end_users) + + const routerSettingsInfo = await getCallbacksCall(accessToken, userID, userRole) + + let router_settings = routerSettingsInfo.router_settings + + console.log("routerSettingsInfo:", router_settings) + + let model_group_retry_policy = router_settings.model_group_retry_policy + let default_retries = router_settings.num_retries + + console.log("model_group_retry_policy:", model_group_retry_policy) + console.log("default_retries:", default_retries) + setModelGroupRetryPolicy(model_group_retry_policy) + setGlobalRetryPolicy(router_settings.retry_policy) + setDefaultRetry(default_retries) - console.log("model_group_retry_policy:", model_group_retry_policy); - console.log("default_retries:", default_retries); - setModelGroupRetryPolicy(model_group_retry_policy); - setGlobalRetryPolicy(router_settings.retry_policy); - setDefaultRetry(default_retries); - // Set model group alias - const model_group_alias = router_settings.model_group_alias || {}; - setModelGroupAlias(model_group_alias); + const model_group_alias = router_settings.model_group_alias || {} + setModelGroupAlias(model_group_alias) } catch (error) { - console.error("There was an error fetching the model data", error); + console.error("There was an error fetching the model data", error) } - }; + } if (accessToken && token && userRole && userID) { - fetchData(); + fetchData() } const fetchModelMap = async () => { - const data = await modelCostMap(accessToken); - console.log(`received model cost map data: ${Object.keys(data)}`); - setModelMap(data); - }; + const data = await modelCostMap(accessToken) + console.log(`received model cost map data: ${Object.keys(data)}`) + setModelMap(data) + } if (modelMap == null) { - fetchModelMap(); + fetchModelMap() } - handleRefreshClick(); - }, [ - accessToken, - token, - userRole, - userID, - modelMap, - lastRefreshed, - selectedTeam, - ]); + handleRefreshClick() + }, [accessToken, token, userRole, userID, modelMap, lastRefreshed, selectedTeam]) if (!modelData) { - return
Loading...
; + return
Loading...
} if (!accessToken || !token || !userRole || !userID) { - return
Loading...
; + return
Loading...
} - let all_models_on_proxy: any[] = []; - let all_providers: string[] = []; + let all_models_on_proxy: any[] = [] + let all_providers: string[] = [] // loop through model data and edit each row for (let i = 0; i < modelData.data.length; i++) { - let curr_model = modelData.data[i]; - let litellm_model_name = curr_model?.litellm_params?.model; - let custom_llm_provider = curr_model?.litellm_params?.custom_llm_provider; - let model_info = curr_model?.model_info; + let curr_model = modelData.data[i] + let litellm_model_name = curr_model?.litellm_params?.model + let custom_llm_provider = curr_model?.litellm_params?.custom_llm_provider + let model_info = curr_model?.model_info - let defaultProvider = "openai"; - let provider = ""; - let input_cost = "Undefined"; - let output_cost = "Undefined"; - let max_tokens = "Undefined"; - let max_input_tokens = "Undefined"; - let cleanedLitellmParams = {}; + let defaultProvider = "openai" + let provider = "" + let input_cost = "Undefined" + let output_cost = "Undefined" + let max_tokens = "Undefined" + let max_input_tokens = "Undefined" + let cleanedLitellmParams = {} const getProviderFromModel = (model: string) => { /** @@ -721,107 +641,96 @@ const ModelDashboard: React.FC = ({ * - check if model in model map * - return it's litellm_provider, if so */ - console.log(`GET PROVIDER CALLED! - ${modelMap}`); + console.log(`GET PROVIDER CALLED! - ${modelMap}`) if (modelMap !== null && modelMap !== undefined) { if (typeof modelMap == "object" && model in modelMap) { - return modelMap[model]["litellm_provider"]; + return modelMap[model]["litellm_provider"] } } - return "openai"; - }; + return "openai" + } // Check if litellm_model_name is null or undefined if (litellm_model_name) { // Split litellm_model_name based on "/" - let splitModel = litellm_model_name.split("/"); + let splitModel = litellm_model_name.split("/") // Get the first element in the split - let firstElement = splitModel[0]; + let firstElement = splitModel[0] // If there is only one element, default provider to openai - provider = custom_llm_provider; + provider = custom_llm_provider if (!provider) { - provider = - splitModel.length === 1 - ? getProviderFromModel(litellm_model_name) - : firstElement; + provider = splitModel.length === 1 ? getProviderFromModel(litellm_model_name) : firstElement } } else { // litellm_model_name is null or undefined, default provider to openai - provider = "-"; + provider = "-" } if (model_info) { - input_cost = model_info?.input_cost_per_token; - output_cost = model_info?.output_cost_per_token; - max_tokens = model_info?.max_tokens; - max_input_tokens = model_info?.max_input_tokens; + input_cost = model_info?.input_cost_per_token + output_cost = model_info?.output_cost_per_token + max_tokens = model_info?.max_tokens + max_input_tokens = model_info?.max_input_tokens } if (curr_model?.litellm_params) { cleanedLitellmParams = Object.fromEntries( - Object.entries(curr_model?.litellm_params).filter( - ([key]) => key !== "model" && key !== "api_base" - ) - ); + Object.entries(curr_model?.litellm_params).filter(([key]) => key !== "model" && key !== "api_base"), + ) } - modelData.data[i].provider = provider; - modelData.data[i].input_cost = input_cost; - modelData.data[i].output_cost = output_cost; - modelData.data[i].litellm_model_name = litellm_model_name; - all_providers.push(provider); + modelData.data[i].provider = provider + modelData.data[i].input_cost = input_cost + modelData.data[i].output_cost = output_cost + modelData.data[i].litellm_model_name = litellm_model_name + all_providers.push(provider) // Convert Cost in terms of Cost per 1M tokens if (modelData.data[i].input_cost) { - modelData.data[i].input_cost = ( - Number(modelData.data[i].input_cost) * 1000000 - ).toFixed(2); + modelData.data[i].input_cost = (Number(modelData.data[i].input_cost) * 1000000).toFixed(2) } if (modelData.data[i].output_cost) { - modelData.data[i].output_cost = ( - Number(modelData.data[i].output_cost) * 1000000 - ).toFixed(2); + modelData.data[i].output_cost = (Number(modelData.data[i].output_cost) * 1000000).toFixed(2) } - modelData.data[i].max_tokens = max_tokens; - modelData.data[i].max_input_tokens = max_input_tokens; - modelData.data[i].api_base = curr_model?.litellm_params?.api_base; - modelData.data[i].cleanedLitellmParams = cleanedLitellmParams; + modelData.data[i].max_tokens = max_tokens + modelData.data[i].max_input_tokens = max_input_tokens + modelData.data[i].api_base = curr_model?.litellm_params?.api_base + modelData.data[i].cleanedLitellmParams = cleanedLitellmParams - all_models_on_proxy.push(curr_model.model_name); + all_models_on_proxy.push(curr_model.model_name) - console.log(modelData.data[i]); + console.log(modelData.data[i]) } // when users click request access show pop up to allow them to request access if (userRole && userRole == "Admin Viewer") { - const { Title, Paragraph } = Typography; + const { Title, Paragraph } = Typography return (
Access Denied - - Ask your proxy admin for access to view all models - + Ask your proxy admin for access to view all models
- ); + ) } const runHealthCheck = async () => { try { - message.info("Running health check..."); - setIsHealthCheckLoading(true); - setHealthCheckResponse(null); - const response = await healthCheckCall(accessToken); - setHealthCheckResponse(response); + message.info("Running health check...") + setIsHealthCheckLoading(true) + setHealthCheckResponse(null) + const response = await healthCheckCall(accessToken) + setHealthCheckResponse(response) } catch (error) { - console.error("Error running health check:", error); - setHealthCheckResponse("Error running health check"); + console.error("Error running health check:", error) + setHealthCheckResponse("Error running health check") } finally { - setIsHealthCheckLoading(false); + setIsHealthCheckLoading(false) } - }; + } const FilterByContent = (
@@ -834,30 +743,26 @@ const ModelDashboard: React.FC = ({ key="all-keys" value="all-keys" onClick={() => { - setSelectedAPIKey(null); + setSelectedAPIKey(null) }} > All Keys {keys?.map((key: any, index: number) => { - if ( - key && - key["key_alias"] !== null && - key["key_alias"].length > 0 - ) { + if (key && key["key_alias"] !== null && key["key_alias"].length > 0) { return ( { - setSelectedAPIKey(key); + setSelectedAPIKey(key) }} > {key["key_alias"]} - ); + ) } - return null; + return null })} @@ -868,7 +773,7 @@ const ModelDashboard: React.FC = ({ key="all-customers" value="all-customers" onClick={() => { - setSelectedCustomer(null); + setSelectedCustomer(null) }} > All Customers @@ -879,12 +784,12 @@ const ModelDashboard: React.FC = ({ key={index} value={user} onClick={() => { - setSelectedCustomer(user); + setSelectedCustomer(user) }} > {user} - ); + ) })} @@ -894,9 +799,7 @@ const ModelDashboard: React.FC = ({ className="w-64 relative z-50" defaultValue="all" value={selectedTeamFilter ?? "all"} - onValueChange={(value) => - setSelectedTeamFilter(value === "all" ? null : value) - } + onValueChange={(value) => setSelectedTeamFilter(value === "all" ? null : value)} > All Teams {teams @@ -919,9 +822,7 @@ const ModelDashboard: React.FC = ({ className="w-64 relative z-50" defaultValue="all" value={selectedTeamFilter ?? "all"} - onValueChange={(value) => - setSelectedTeamFilter(value === "all" ? null : value) - } + onValueChange={(value) => setSelectedTeamFilter(value === "all" ? null : value)} > All Teams {teams @@ -937,94 +838,82 @@ const ModelDashboard: React.FC = ({
)}
- ); + ) const customTooltip = (props: any) => { - const { payload, active } = props; - if (!active || !payload) return null; + const { payload, active } = props + if (!active || !payload) return null // Extract the date from the first item in the payload array - const date = payload[0]?.payload?.date; + const date = payload[0]?.payload?.date // Sort the payload array by category.value in descending order - let sortedPayload = payload.sort((a: any, b: any) => b.value - a.value); + let sortedPayload = payload.sort((a: any, b: any) => b.value - a.value) // Only show the top 5, the 6th one should be called "X other categories" depending on how many categories were not shown if (sortedPayload.length > 5) { - let remainingItems = sortedPayload.length - 5; - sortedPayload = sortedPayload.slice(0, 5); + let remainingItems = sortedPayload.length - 5 + sortedPayload = sortedPayload.slice(0, 5) sortedPayload.push({ dataKey: `${remainingItems} other deployments`, - value: payload - .slice(5) - .reduce((acc: number, curr: any) => acc + curr.value, 0), + value: payload.slice(5).reduce((acc: number, curr: any) => acc + curr.value, 0), color: "gray", - }); + }) } return (
- {date && ( -

Date: {date}

- )} + {date &&

Date: {date}

} {sortedPayload.map((category: any, idx: number) => { - const roundedValue = parseFloat(category.value.toFixed(5)); - const displayValue = - roundedValue === 0 && category.value > 0 - ? "<0.00001" - : roundedValue.toFixed(5); + const roundedValue = parseFloat(category.value.toFixed(5)) + const displayValue = roundedValue === 0 && category.value > 0 ? "<0.00001" : roundedValue.toFixed(5) return (
-
+

{category.dataKey}

-

- {displayValue} -

+

{displayValue}

- ); + ) })}
- ); - }; + ) + } const handleOk = () => { - console.log("🚀 handleOk called from model dashboard!"); - console.log("Current form values:", addModelForm.getFieldsValue()); - + console.log("🚀 handleOk called from model dashboard!") + console.log("Current form values:", addModelForm.getFieldsValue()) + addModelForm .validateFields() .then((values: any) => { - console.log("✅ Validation passed, submitting:", values); - handleAddModelSubmit(values, accessToken, addModelForm, handleRefreshClick); + console.log("✅ Validation passed, submitting:", values) + handleAddModelSubmit(values, accessToken, addModelForm, handleRefreshClick) }) .catch((error: any) => { - console.error("❌ Validation failed:", error); - console.error("Form errors:", error.errorFields); - const errorMessages = error.errorFields?.map((field: any) => { - return `${field.name.join('.')}: ${field.errors.join(', ')}`; - }).join(' | ') || 'Unknown validation error'; - NotificationManager.fromBackend(`Please fill in the following required fields: ${errorMessages}`); - }); - }; + console.error("❌ Validation failed:", error) + console.error("Form errors:", error.errorFields) + const errorMessages = + error.errorFields + ?.map((field: any) => { + return `${field.name.join(".")}: ${field.errors.join(", ")}` + }) + .join(" | ") || "Unknown validation error" + NotificationManager.fromBackend(`Please fill in the following required fields: ${errorMessages}`) + }) + } - - - console.log(`selectedProvider: ${selectedProvider}`); - console.log(`providerModels.length: ${providerModels.length}`); + console.log(`selectedProvider: ${selectedProvider}`) + console.log(`providerModels.length: ${providerModels.length}`) const providerKey = Object.keys(Providers).find( - (key) => (Providers as { [index: string]: any })[key] === selectedProvider - ); + (key) => (Providers as { [index: string]: any })[key] === selectedProvider, + ) - let dynamicProviderForm: ProviderSettings | undefined = undefined; + let dynamicProviderForm: ProviderSettings | undefined = undefined if (providerKey && providerSettings) { - dynamicProviderForm = providerSettings.find( - (provider) => provider.name === provider_map[providerKey] - ); + dynamicProviderForm = providerSettings.find((provider) => provider.name === provider_map[providerKey]) } // If a team is selected, render TeamInfoView in full page layout @@ -1042,7 +931,7 @@ const ModelDashboard: React.FC = ({ onUpdate={handleRefreshClick} />
- ); + ) } return ( @@ -1053,9 +942,11 @@ const ModelDashboard: React.FC = ({

Model Management

-

- Manage your models and configurations -

+ {!all_admin_roles.includes(userRole) ? ( +

Add models for teams you are an admin for.

+ ) : ( +

Add and manage models for the proxy

+ )}
{selectedModelId ? ( @@ -1063,12 +954,10 @@ const ModelDashboard: React.FC = ({ modelId={selectedModelId} editModel={true} onClose={() => { - setSelectedModelId(null); - setEditModel(false); + setSelectedModelId(null) + setEditModel(false) }} - modelData={modelData.data.find( - (model: any) => model.model_info.id === selectedModelId - )} + modelData={modelData.data.find((model: any) => model.model_info.id === selectedModelId)} accessToken={accessToken} userID={userID} userRole={userRole} @@ -1079,58 +968,32 @@ const ModelDashboard: React.FC = ({ const updatedModelData = { ...modelData, data: modelData.data.map((model: any) => - model.model_info.id === updatedModel.model_info.id - ? updatedModel - : model + model.model_info.id === updatedModel.model_info.id ? updatedModel : model, ), - }; - setModelData(updatedModelData); + } + setModelData(updatedModelData) // Trigger a refresh to update UI - handleRefreshClick(); + handleRefreshClick() }} modelAccessGroups={availableModelAccessGroups} /> ) : ( - +
- {all_admin_roles.includes(userRole) ? ( - All Models - ) : ( - Your Models - )} + {all_admin_roles.includes(userRole) ? All Models : Your Models} Add Model - {all_admin_roles.includes(userRole) && ( - LLM Credentials - )} - {all_admin_roles.includes(userRole) && ( - Pass-Through Endpoints - )} - {all_admin_roles.includes(userRole) && ( - Health Status - )} - {all_admin_roles.includes(userRole) && ( - Model Analytics - )} - {all_admin_roles.includes(userRole) && ( - Model Retry Settings - )} - {all_admin_roles.includes(userRole) && ( - Model Group Alias - )} - {all_admin_roles.includes(userRole) && ( - Price Data Reload - )} + {all_admin_roles.includes(userRole) && LLM Credentials} + {all_admin_roles.includes(userRole) && Pass-Through Endpoints} + {all_admin_roles.includes(userRole) && Health Status} + {all_admin_roles.includes(userRole) && Model Analytics} + {all_admin_roles.includes(userRole) && Model Retry Settings} + {all_admin_roles.includes(userRole) && Model Group Alias} + {all_admin_roles.includes(userRole) && Price Data Reload}
- {lastRefreshed && ( - Last Refreshed: {lastRefreshed} - )} + {lastRefreshed && Last Refreshed: {lastRefreshed}} = ({
-
-
- Model Management - {!all_admin_roles.includes(userRole) ? ( - - Add models for teams you are an admin for. - - ) : ( - - Add and manage models for the proxy - - )} -
-
@@ -1166,32 +1015,23 @@ const ModelDashboard: React.FC = ({
- - Current Team: - + Current Team: - setModelViewMode( - value as "current_team" | "all" - ) - } + onValueChange={(value) => setModelViewMode(value as "current_team" | "all")} >
- - Current Team Models - + Current Team Models
- - All Available Models - + All Available Models
@@ -1286,25 +1113,19 @@ const ModelDashboard: React.FC = ({ onValueChange={setModelNameSearch} />
- + {/* Model Name Filter */}
Filter by Public Model Name: - setSelectedModelAccessGroupFilter( - value === "all" ? null : value - ) + setSelectedModelAccessGroupFilter(value === "all" ? null : value) } > - - All Model Access Groups - - {availableModelAccessGroups.map( - (accessGroup, idx) => ( - - {accessGroup} - - ) - )} + All Model Access Groups + {availableModelAccessGroups.map((accessGroup, idx) => ( + + {accessGroup} + + ))}
@@ -1350,43 +1160,29 @@ const ModelDashboard: React.FC = ({ Showing{" "} {modelData && modelData.data.length > 0 ? modelData.data.filter((model: any) => { - const searchMatch = modelNameSearch === "" || - model.model_name.toLowerCase().includes(modelNameSearch.toLowerCase()); + const searchMatch = + modelNameSearch === "" || + model.model_name.toLowerCase().includes(modelNameSearch.toLowerCase()) const modelNameMatch = selectedModelGroup === "all" || - model.model_name === - selectedModelGroup || - !selectedModelGroup; + model.model_name === selectedModelGroup || + !selectedModelGroup const accessGroupMatch = - selectedModelAccessGroupFilter === - "all" || - model.model_info[ - "access_groups" - ]?.includes( - selectedModelAccessGroupFilter - ) || - !selectedModelAccessGroupFilter; - let teamAccessMatch = true; + selectedModelAccessGroupFilter === "all" || + model.model_info["access_groups"]?.includes(selectedModelAccessGroupFilter) || + !selectedModelAccessGroupFilter + let teamAccessMatch = true if (modelViewMode === "current_team") { if (currentTeam === "personal") { - teamAccessMatch = - model.model_info?.direct_access === - true; + teamAccessMatch = model.model_info?.direct_access === true } else { teamAccessMatch = - model.model_info?.access_via_team_ids?.includes( - currentTeam - ) === true; + model.model_info?.access_via_team_ids?.includes(currentTeam) === true } } - return ( - searchMatch && - modelNameMatch && - accessGroupMatch && - teamAccessMatch - ); + return searchMatch && modelNameMatch && accessGroupMatch && teamAccessMatch }).length : 0}{" "} results @@ -1395,88 +1191,79 @@ const ModelDashboard: React.FC = ({
- { - // Model name search filter - const searchMatch = modelNameSearch === "" || - model.model_name.toLowerCase().includes(modelNameSearch.toLowerCase()); + { + // Model name search filter + const searchMatch = + modelNameSearch === "" || + model.model_name.toLowerCase().includes(modelNameSearch.toLowerCase()) - // Model name filter - const modelNameMatch = selectedModelGroup === "all" || - model.model_name === selectedModelGroup || - !selectedModelGroup || (selectedModelGroup === "wildcard" && model.model_name?.includes('*')); - // Model access group filter - const accessGroupMatch = - selectedModelAccessGroupFilter === "all" || - model.model_info["access_groups"]?.includes( - selectedModelAccessGroupFilter - ) || - !selectedModelAccessGroupFilter; - // Team access filter based on current team and view mode - let teamAccessMatch = true; - if (modelViewMode === "current_team") { - if (currentTeam === "personal") { - // Show only models with direct access - teamAccessMatch = - model.model_info?.direct_access === true; - } else { - // Show only models accessible by the current team - teamAccessMatch = - model.model_info?.access_via_team_ids?.includes( - currentTeam - ) === true; + // Model name filter + const modelNameMatch = + selectedModelGroup === "all" || + model.model_name === selectedModelGroup || + !selectedModelGroup || + (selectedModelGroup === "wildcard" && model.model_name?.includes("*")) + // Model access group filter + const accessGroupMatch = + selectedModelAccessGroupFilter === "all" || + model.model_info["access_groups"]?.includes(selectedModelAccessGroupFilter) || + !selectedModelAccessGroupFilter + // Team access filter based on current team and view mode + let teamAccessMatch = true + if (modelViewMode === "current_team") { + if (currentTeam === "personal") { + // Show only models with direct access + teamAccessMatch = model.model_info?.direct_access === true + } else { + // Show only models accessible by the current team + teamAccessMatch = model.model_info?.access_via_team_ids?.includes(currentTeam) === true + } } - } - // For 'all' mode, show all models (teamAccessMatch remains true) + // For 'all' mode, show all models (teamAccessMatch remains true) - return ( - searchMatch && - modelNameMatch && - accessGroupMatch && - teamAccessMatch - ); - })} - isLoading={false} - table={tableRef} - /> -
-
- - - - - - + return searchMatch && modelNameMatch && accessGroupMatch && teamAccessMatch + })} + isLoading={false} + table={tableRef} + /> +
+
+ + + + + + = ({ value={dateValue} className="mr-2" onValueChange={(value) => { - setDateValue(value); - updateModelMetrics( - selectedModelGroup, - value.from, - value.to - ); + setDateValue(value) + updateModelMetrics(selectedModelGroup, value.from, value.to) }} /> Select Model Group - - - - - - + + + + + @@ -1581,13 +1349,9 @@ const ModelDashboard: React.FC = ({ -

- {" "} - (seconds/token) -

+

(seconds/token)

- average Latency for successfull requests divided - by the total tokens + average Latency for successfull requests divided by the total tokens {modelMetrics && modelMetricsCategories && ( = ({ @@ -1622,12 +1384,9 @@ const ModelDashboard: React.FC = ({ Deployment + Success Responses - Success Responses - - - Slow Responses{" "} -

Success Responses taking 600+s

+ Slow Responses

Success Responses taking 600+s

 @@ -1661,9 +1420,7 @@ const ModelDashboard: React.FC = ({ - - All Up Rate Limit Errors (429) for {selectedModelGroup} - + All Up Rate Limit Errors (429) for {selectedModelGroup} = ({ color: "#535452", }} > - Num Rate Limit Errors{" "} - {globalExceptionData.sum_num_rate_limit_exceptions} + Num Rate Limit Errors {globalExceptionData.sum_num_rate_limit_exceptions} = ({ {premiumUser ? ( <> - {globalExceptionPerDeployment.map( - (globalActivity, index) => ( - - - {globalActivity.api_base - ? globalActivity.api_base - : "Unknown API Base"} - - - - - Num Rate Limit Errors (429){" "} - { - globalActivity.sum_num_rate_limit_exceptions - } - - console.log(v)} - /> - - - - ) - )} + {globalExceptionPerDeployment.map((globalActivity, index) => ( + + {globalActivity.api_base ? globalActivity.api_base : "Unknown API Base"} + + + + Num Rate Limit Errors (429) {globalActivity.sum_num_rate_limit_exceptions} + + console.log(v)} + /> + + + + ))} ) : ( <> {globalExceptionPerDeployment && globalExceptionPerDeployment.length > 0 && - globalExceptionPerDeployment - .slice(0, 1) - .map((globalActivity, index) => ( - - - ✨ Rate Limit Errors by Deployment - -

- Upgrade to see exceptions for all deployments -

- - - {globalActivity.api_base} - - - - Num Rate Limit Errors{" "} - { - globalActivity.sum_num_rate_limit_exceptions - } - - console.log(v)} - /> - - - + globalExceptionPerDeployment.slice(0, 1).map((globalActivity, index) => ( + + ✨ Rate Limit Errors by Deployment +

+ Upgrade to see exceptions for all deployments +

+ + + {globalActivity.api_base} + + + + Num Rate Limit Errors {globalActivity.sum_num_rate_limit_exceptions} + + console.log(v)} + /> + + - ))} + + ))} )} @@ -1791,16 +1526,14 @@ const ModelDashboard: React.FC = ({