From f88e6afbb7251393bd296307e408766673c60c57 Mon Sep 17 00:00:00 2001 From: tanjiro <56165694+NANDINI-star@users.noreply.github.com> Date: Fri, 15 Aug 2025 01:42:32 +0900 Subject: [PATCH 01/21] fix query param --- ui/litellm-dashboard/src/components/networking.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 93ae040c86..a046d32488 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -4731,7 +4731,7 @@ export const deletePassThroughEndpointsCall = async ( try { let url = proxyBaseUrl ? `${proxyBaseUrl}/config/pass_through_endpoint?endpoint_id=${endpointId}` - : `/config/pass_through_endpoint${endpointId}`; + : `/config/pass_through_endpoint?endpoint_id=${endpointId}`; //message.info("Requesting model data"); const response = await fetch(url, { From 4b51e5787c781f43c221167126388593fd3580da Mon Sep 17 00:00:00 2001 From: Tasmay Pankaj Tibrewal <85983760+Tasmay-Tibrewal@users.noreply.github.com> Date: Fri, 15 Aug 2025 04:33:49 +0530 Subject: [PATCH 02/21] added qwen3, deepseek r1 0528 throughput, glm 4.5 and gpt oss models --- model_prices_and_context_window.json | 88 ++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9668053888..5e43113113 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -14649,6 +14649,50 @@ "mode": "chat", "supports_tool_choice": true }, + "deepseek-ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-06, + "max_input_tokens": 262000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "supports_tool_choice": false + "source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8" + }, + "deepseek-ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 2e-06, + "max_input_tokens": 256000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "supports_tool_choice": false + "source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct" + }, + "deepseek-ai/Qwen/Qwen3-235B-A22B-Thinking-2507": { + "input_cost_per_token": 6.5e-07, + "output_cost_per_token": 3e-06, + "max_input_tokens": 256000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "supports_tool_choice": false + "source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507" + }, + "deepseek-ai/Qwen/Qwen3-235B-A22B-fp8-tput": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-07, + "max_input_tokens": 40000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "supports_tool_choice": false + "source": "https://www.together.ai/models/qwen3-235b-a22b-fp8-tput" + }, "together_ai/deepseek-ai/DeepSeek-V3": { "input_cost_per_token": 1.25e-06, "output_cost_per_token": 1.25e-06, @@ -14673,6 +14717,17 @@ "mode": "chat", "supports_tool_choice": true }, + "deepseek-ai/DeepSeek-R1-0528-tput": { + "input_cost_per_token": 5.5e-07, + "output_cost_per_token": 2.19e-06, + "max_input_tokens": 128000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "supports_tool_choice": false + "source": "https://www.together.ai/models/deepseek-r1-0528-throughput" + }, "together_ai/mistralai/Mistral-Small-24B-Instruct-2501": { "litellm_provider": "together_ai", "supports_function_calling": true, @@ -14690,6 +14745,39 @@ "mode": "chat", "source": "https://www.together.ai/models/kimi-k2-instruct" }, + "together_ai/openai/gpt-oss-120b": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "max_input_tokens": 128000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "source": "https://www.together.ai/models/gpt-oss-120b" + }, + "together_ai/OpenAI/gpt-oss-20B": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2e-07, + "max_input_tokens": 128000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "source": "https://www.together.ai/models/gpt-oss-20b" + }, + "together_ai/zai-org/GLM-4.5-Air-FP8": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 1.1e-06, + "max_input_tokens": 128000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "source": "https://www.together.ai/models/glm-4-5-air" + }, "ollama/codegemma": { "max_tokens": 8192, "max_input_tokens": 8192, From a85ab9d2044e34514982396210ea95ad8a91e99a Mon Sep 17 00:00:00 2001 From: Tasmay Pankaj Tibrewal <85983760+Tasmay-Tibrewal@users.noreply.github.com> Date: Fri, 15 Aug 2025 04:34:55 +0530 Subject: [PATCH 03/21] added qwen3, deepseek r1 0528 throughput, glm 4.5 and gpt oss models --- ...odel_prices_and_context_window_backup.json | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9668053888..5e43113113 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -14649,6 +14649,50 @@ "mode": "chat", "supports_tool_choice": true }, + "deepseek-ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-06, + "max_input_tokens": 262000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "supports_tool_choice": false + "source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8" + }, + "deepseek-ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 2e-06, + "max_input_tokens": 256000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "supports_tool_choice": false + "source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct" + }, + "deepseek-ai/Qwen/Qwen3-235B-A22B-Thinking-2507": { + "input_cost_per_token": 6.5e-07, + "output_cost_per_token": 3e-06, + "max_input_tokens": 256000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "supports_tool_choice": false + "source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507" + }, + "deepseek-ai/Qwen/Qwen3-235B-A22B-fp8-tput": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 6e-07, + "max_input_tokens": 40000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "supports_tool_choice": false + "source": "https://www.together.ai/models/qwen3-235b-a22b-fp8-tput" + }, "together_ai/deepseek-ai/DeepSeek-V3": { "input_cost_per_token": 1.25e-06, "output_cost_per_token": 1.25e-06, @@ -14673,6 +14717,17 @@ "mode": "chat", "supports_tool_choice": true }, + "deepseek-ai/DeepSeek-R1-0528-tput": { + "input_cost_per_token": 5.5e-07, + "output_cost_per_token": 2.19e-06, + "max_input_tokens": 128000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "supports_tool_choice": false + "source": "https://www.together.ai/models/deepseek-r1-0528-throughput" + }, "together_ai/mistralai/Mistral-Small-24B-Instruct-2501": { "litellm_provider": "together_ai", "supports_function_calling": true, @@ -14690,6 +14745,39 @@ "mode": "chat", "source": "https://www.together.ai/models/kimi-k2-instruct" }, + "together_ai/openai/gpt-oss-120b": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "max_input_tokens": 128000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "source": "https://www.together.ai/models/gpt-oss-120b" + }, + "together_ai/OpenAI/gpt-oss-20B": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2e-07, + "max_input_tokens": 128000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "source": "https://www.together.ai/models/gpt-oss-20b" + }, + "together_ai/zai-org/GLM-4.5-Air-FP8": { + "input_cost_per_token": 2e-07, + "output_cost_per_token": 1.1e-06, + "max_input_tokens": 128000, + "litellm_provider": "together_ai", + "supports_function_calling": false, + "supports_tool_choice": false, + "supports_parallel_function_calling": false, + "mode": "chat", + "source": "https://www.together.ai/models/glm-4-5-air" + }, "ollama/codegemma": { "max_tokens": 8192, "max_input_tokens": 8192, From d20391101b3a5024ee1aeb33d1218fce9f194830 Mon Sep 17 00:00:00 2001 From: Tasmay Pankaj Tibrewal <85983760+Tasmay-Tibrewal@users.noreply.github.com> Date: Fri, 15 Aug 2025 04:43:52 +0530 Subject: [PATCH 04/21] fixed together ai provider name mistake --- model_prices_and_context_window.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5e43113113..071ade6e5a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -14649,7 +14649,7 @@ "mode": "chat", "supports_tool_choice": true }, - "deepseek-ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": { + "together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": { "input_cost_per_token": 2e-07, "output_cost_per_token": 6e-06, "max_input_tokens": 262000, @@ -14660,7 +14660,7 @@ "supports_tool_choice": false "source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8" }, - "deepseek-ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { + "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { "input_cost_per_token": 2e-06, "output_cost_per_token": 2e-06, "max_input_tokens": 256000, @@ -14671,7 +14671,7 @@ "supports_tool_choice": false "source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct" }, - "deepseek-ai/Qwen/Qwen3-235B-A22B-Thinking-2507": { + "together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": { "input_cost_per_token": 6.5e-07, "output_cost_per_token": 3e-06, "max_input_tokens": 256000, @@ -14682,7 +14682,7 @@ "supports_tool_choice": false "source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507" }, - "deepseek-ai/Qwen/Qwen3-235B-A22B-fp8-tput": { + "together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": { "input_cost_per_token": 2e-07, "output_cost_per_token": 6e-07, "max_input_tokens": 40000, @@ -14717,7 +14717,7 @@ "mode": "chat", "supports_tool_choice": true }, - "deepseek-ai/DeepSeek-R1-0528-tput": { + "together_ai/deepseek-ai/DeepSeek-R1-0528-tput": { "input_cost_per_token": 5.5e-07, "output_cost_per_token": 2.19e-06, "max_input_tokens": 128000, From 0a83aecb5c334a514ca87181dcb7682cf16a8f00 Mon Sep 17 00:00:00 2001 From: Tasmay Pankaj Tibrewal <85983760+Tasmay-Tibrewal@users.noreply.github.com> Date: Fri, 15 Aug 2025 04:44:43 +0530 Subject: [PATCH 05/21] fixed together ai provider name mistake --- litellm/model_prices_and_context_window_backup.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5e43113113..071ade6e5a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -14649,7 +14649,7 @@ "mode": "chat", "supports_tool_choice": true }, - "deepseek-ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": { + "together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": { "input_cost_per_token": 2e-07, "output_cost_per_token": 6e-06, "max_input_tokens": 262000, @@ -14660,7 +14660,7 @@ "supports_tool_choice": false "source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8" }, - "deepseek-ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { + "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { "input_cost_per_token": 2e-06, "output_cost_per_token": 2e-06, "max_input_tokens": 256000, @@ -14671,7 +14671,7 @@ "supports_tool_choice": false "source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct" }, - "deepseek-ai/Qwen/Qwen3-235B-A22B-Thinking-2507": { + "together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": { "input_cost_per_token": 6.5e-07, "output_cost_per_token": 3e-06, "max_input_tokens": 256000, @@ -14682,7 +14682,7 @@ "supports_tool_choice": false "source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507" }, - "deepseek-ai/Qwen/Qwen3-235B-A22B-fp8-tput": { + "together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": { "input_cost_per_token": 2e-07, "output_cost_per_token": 6e-07, "max_input_tokens": 40000, @@ -14717,7 +14717,7 @@ "mode": "chat", "supports_tool_choice": true }, - "deepseek-ai/DeepSeek-R1-0528-tput": { + "together_ai/deepseek-ai/DeepSeek-R1-0528-tput": { "input_cost_per_token": 5.5e-07, "output_cost_per_token": 2.19e-06, "max_input_tokens": 128000, From d9105a99abb7dab774b5c3858aebf0407cab83cb Mon Sep 17 00:00:00 2001 From: Tasmay Pankaj Tibrewal <85983760+Tasmay-Tibrewal@users.noreply.github.com> Date: Fri, 15 Aug 2025 05:07:46 +0530 Subject: [PATCH 06/21] fixed comma delimeter issue --- model_prices_and_context_window.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 071ade6e5a..a39f81897b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -14657,7 +14657,7 @@ "supports_function_calling": false, "supports_parallel_function_calling": false, "mode": "chat", - "supports_tool_choice": false + "supports_tool_choice": false, "source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8" }, "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { @@ -14668,7 +14668,7 @@ "supports_function_calling": false, "supports_parallel_function_calling": false, "mode": "chat", - "supports_tool_choice": false + "supports_tool_choice": false, "source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct" }, "together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": { @@ -14679,7 +14679,7 @@ "supports_function_calling": false, "supports_parallel_function_calling": false, "mode": "chat", - "supports_tool_choice": false + "supports_tool_choice": false, "source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507" }, "together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": { @@ -14690,7 +14690,7 @@ "supports_function_calling": false, "supports_parallel_function_calling": false, "mode": "chat", - "supports_tool_choice": false + "supports_tool_choice": false, "source": "https://www.together.ai/models/qwen3-235b-a22b-fp8-tput" }, "together_ai/deepseek-ai/DeepSeek-V3": { @@ -14725,7 +14725,7 @@ "supports_function_calling": false, "supports_parallel_function_calling": false, "mode": "chat", - "supports_tool_choice": false + "supports_tool_choice": false, "source": "https://www.together.ai/models/deepseek-r1-0528-throughput" }, "together_ai/mistralai/Mistral-Small-24B-Instruct-2501": { From d8a9509890e5ed6c7d2e095946e14f939d90a1b5 Mon Sep 17 00:00:00 2001 From: Tasmay Pankaj Tibrewal <85983760+Tasmay-Tibrewal@users.noreply.github.com> Date: Fri, 15 Aug 2025 05:08:20 +0530 Subject: [PATCH 07/21] fixed comma delimeter issue --- litellm/model_prices_and_context_window_backup.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 071ade6e5a..a39f81897b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -14657,7 +14657,7 @@ "supports_function_calling": false, "supports_parallel_function_calling": false, "mode": "chat", - "supports_tool_choice": false + "supports_tool_choice": false, "source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8" }, "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { @@ -14668,7 +14668,7 @@ "supports_function_calling": false, "supports_parallel_function_calling": false, "mode": "chat", - "supports_tool_choice": false + "supports_tool_choice": false, "source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct" }, "together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": { @@ -14679,7 +14679,7 @@ "supports_function_calling": false, "supports_parallel_function_calling": false, "mode": "chat", - "supports_tool_choice": false + "supports_tool_choice": false, "source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507" }, "together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": { @@ -14690,7 +14690,7 @@ "supports_function_calling": false, "supports_parallel_function_calling": false, "mode": "chat", - "supports_tool_choice": false + "supports_tool_choice": false, "source": "https://www.together.ai/models/qwen3-235b-a22b-fp8-tput" }, "together_ai/deepseek-ai/DeepSeek-V3": { @@ -14725,7 +14725,7 @@ "supports_function_calling": false, "supports_parallel_function_calling": false, "mode": "chat", - "supports_tool_choice": false + "supports_tool_choice": false, "source": "https://www.together.ai/models/deepseek-r1-0528-throughput" }, "together_ai/mistralai/Mistral-Small-24B-Instruct-2501": { From 790d2d08306189f97b5595aae8b59626ffcfd333 Mon Sep 17 00:00:00 2001 From: Jugal Bhatt Date: Fri, 15 Aug 2025 10:57:39 -0700 Subject: [PATCH 08/21] [Update] Adjust max_input_tokens for azure/gpt-5-chat models in JSON configuration * Reduced max_input_tokens from 400000 to 272000 for both azure/gpt-5-chat and azure/gpt-5-chat-latest to optimize resource usage and align with updated model specifications. --- litellm/model_prices_and_context_window_backup.json | 4 ++-- model_prices_and_context_window.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9668053888..3874549760 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2457,7 +2457,7 @@ }, "azure/gpt-5-chat": { "max_tokens": 128000, - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "input_cost_per_token": 1.25e-06, "output_cost_per_token": 1e-05, @@ -2490,7 +2490,7 @@ }, "azure/gpt-5-chat-latest": { "max_tokens": 128000, - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "input_cost_per_token": 1.25e-06, "output_cost_per_token": 1e-05, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9668053888..3874549760 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -2457,7 +2457,7 @@ }, "azure/gpt-5-chat": { "max_tokens": 128000, - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "input_cost_per_token": 1.25e-06, "output_cost_per_token": 1e-05, @@ -2490,7 +2490,7 @@ }, "azure/gpt-5-chat-latest": { "max_tokens": 128000, - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "input_cost_per_token": 1.25e-06, "output_cost_per_token": 1e-05, From de59691c4beeee153f77a78dcff18c6f19778f51 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Sat, 15 Feb 2025 15:48:32 -0800 Subject: [PATCH 09/21] Enable update/delete org members on UI (#8560) * feat(organization_endpoints.py): expose new `/organization/delete` endpoint. Cascade org deletion to member, teams and keys Ensures any org deletion is handled correctly * test(test_organizations.py): add simple test to ensure org deletion works * feat(organization_endpoints.py): expose /organization/update endpoint, and define response models for org delete + update * fix(organizations.tsx): support org delete on UI + move org/delete endpoint to use DELETE * feat(organization_endpoints.py): support `/organization/member_update` endpoint Allow admin to update member's role within org * feat(organization_endpoints.py): support deleting member from org * test(test_organizations.py): add e2e test to ensure org member flow works * fix(organization_endpoints.py): fix code qa check * fix(schema.prisma): don't introduce ondelete:cascade - breaking change * docs(organization_endpoints.py): document missing params From 6b78ade91883daadb7d0a3fd912ed5dfe07d2939 Mon Sep 17 00:00:00 2001 From: drorbaron Date: Tue, 19 Aug 2025 12:20:19 +0300 Subject: [PATCH 10/21] migrate to use new aim FW API --- litellm/proxy/guardrails/guardrail_hooks/aim/aim.py | 9 ++++++--- tests/local_testing/test_aim_guardrails.py | 7 ++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py b/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py index 621defa2d4..e40bafcc46 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py +++ b/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py @@ -118,7 +118,7 @@ class AimGuardrail(CustomGuardrail): litellm_call_id=call_id, ) response = await self.async_handler.post( - f"{self.api_base}/detect/openai/v2", + f"{self.api_base}/fw/v1/analyze", headers=headers, json={"messages": data.get("messages", [])}, ) @@ -183,14 +183,17 @@ class AimGuardrail(CustomGuardrail): ) call_id = request_data.get("litellm_call_id") response = await self.async_handler.post( - f"{self.api_base}/detect/output/v2", + f"{self.api_base}/fw/v1/analyze", headers=self._build_aim_headers( hook=hook, key_alias=key_alias, user_email=user_email, litellm_call_id=call_id, ), - json={"output": output, "messages": request_data.get("messages", [])}, + json={ + "messages": request_data.get("messages", []) + + [{"role": "assistant", "content": output}] + }, ) response.raise_for_status() res = response.json() diff --git a/tests/local_testing/test_aim_guardrails.py b/tests/local_testing/test_aim_guardrails.py index b11a424ac3..3a9b6e9a3d 100644 --- a/tests/local_testing/test_aim_guardrails.py +++ b/tests/local_testing/test_aim_guardrails.py @@ -216,12 +216,13 @@ async def test_post_call__with_anonymized_entities__it_deanonymizes_output(): ) as mock_post: def mock_post_detect_side_effect(url, *args, **kwargs): - if url.endswith("/detect/openai/v2"): + request_body = kwargs.get("json", {}) + if request_body["messages"][-1]["role"] == "user": return response_with_detections - elif url.endswith("/detect/output/v2"): + elif request_body["messages"][-1]["role"] == "assistant": return response_without_detections else: - raise ValueError("Unexpected URL: {}".format(url)) + raise ValueError("Unexpected request: {}".format(request_body)) mock_post.side_effect = mock_post_detect_side_effect From 7fedcf1ea96fc7852c4c43487c38427b0b6ea836 Mon Sep 17 00:00:00 2001 From: drorbaron Date: Tue, 19 Aug 2025 12:36:29 +0300 Subject: [PATCH 11/21] migrate stream ws --- litellm/proxy/guardrails/guardrail_hooks/aim/aim.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py b/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py index e40bafcc46..3c03d70cef 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py +++ b/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py @@ -300,7 +300,7 @@ class AimGuardrail(CustomGuardrail): ) call_id = request_data.get("litellm_call_id") async with connect( - f"{self.ws_api_base}/detect/output/ws", + f"{self.ws_api_base}/fw/v1/analyze/stream", additional_headers=self._build_aim_headers( hook="output", key_alias=user_api_key_dict.key_alias, From 4f38a6315251fa1843ee8782c6018d5e2e2d54a4 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 19 Aug 2025 16:40:12 +0530 Subject: [PATCH 12/21] Add long context support for claude-4-sonnet --- .../litellm_core_utils/llm_cost_calc/utils.py | 44 ++++++++++++------- ...odel_prices_and_context_window_backup.json | 10 +++-- model_prices_and_context_window.json | 10 +++-- 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 4b6cffd06c..fe74778824 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -113,15 +113,20 @@ def _generic_cost_per_character( return prompt_cost, completion_cost -def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, float]: +def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, float, float, float]: """ - Return prompt cost for a given model and usage. + Return prompt cost, completion cost, and cache costs for a given model and usage. If input_tokens > threshold and `input_cost_per_token_above_[x]k_tokens` or `input_cost_per_token_above_[x]_tokens` is set, - then we use the corresponding threshold cost. + then we use the corresponding threshold cost for all token types. + + Returns: + Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost) """ prompt_base_cost = cast(float, _get_cost_per_unit(model_info, "input_cost_per_token")) completion_base_cost = cast(float, _get_cost_per_unit(model_info, "output_cost_per_token")) + cache_creation_cost = cast(float, _get_cost_per_unit(model_info, "cache_creation_input_token_cost")) + cache_read_cost = cast(float, _get_cost_per_unit(model_info, "cache_read_input_token_cost")) ## CHECK IF ABOVE THRESHOLD threshold: Optional[float] = None @@ -141,13 +146,28 @@ def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, fl f"output_cost_per_token_above_{threshold_str}_tokens", completion_base_cost, )) + + # Apply tiered pricing to cache costs + cache_creation_tiered_key = f"cache_creation_input_token_cost_above_{threshold_str}_tokens" + cache_read_tiered_key = f"cache_read_input_token_cost_above_{threshold_str}_tokens" + + if cache_creation_tiered_key in model_info: + cache_creation_cost = cast(float, _get_cost_per_unit( + model_info, cache_creation_tiered_key, cache_creation_cost + )) + + if cache_read_tiered_key in model_info: + cache_read_cost = cast(float, _get_cost_per_unit( + model_info, cache_read_tiered_key, cache_read_cost + )) + break except (IndexError, ValueError): continue except Exception: continue - return prompt_base_cost, completion_base_cost + return prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost def calculate_cost_component( @@ -262,28 +282,22 @@ def generic_cost_per_token( if text_tokens == 0: text_tokens = usage.prompt_tokens - cache_hit_tokens - audio_tokens - prompt_base_cost, completion_base_cost = _get_token_base_cost( + prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost = _get_token_base_cost( model_info=model_info, usage=usage ) prompt_cost = float(text_tokens) * prompt_base_cost - ### CACHE READ COST - prompt_cost += calculate_cost_component( - model_info, "cache_read_input_token_cost", cache_hit_tokens - ) + ### CACHE READ COST - Now uses tiered pricing + prompt_cost += float(cache_hit_tokens) * cache_read_cost ### AUDIO COST prompt_cost += calculate_cost_component( model_info, "input_cost_per_audio_token", audio_tokens ) - ### CACHE WRITING COST - prompt_cost += calculate_cost_component( - model_info, - "cache_creation_input_token_cost", - usage._cache_creation_input_tokens, - ) + ### CACHE WRITING COST - Now uses tiered pricing + prompt_cost += float(usage._cache_creation_input_tokens or 0) * cache_creation_cost ### CHARACTER COST diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9668053888..4c6bff53c4 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6437,11 +6437,13 @@ "supports_computer_use": true }, "claude-4-sonnet-20250514": { - "max_tokens": 64000, - "max_input_tokens": 200000, - "max_output_tokens": 64000, + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, + "input_cost_per_token_above_200k_tokens": 6e-06, + "output_cost_per_token_above_200k_tokens": 2.25e-05, "search_context_cost_per_query": { "search_context_size_low": 0.01, "search_context_size_medium": 0.01, @@ -6449,6 +6451,8 @@ }, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, + "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, "litellm_provider": "anthropic", "mode": "chat", "supports_function_calling": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9668053888..4c6bff53c4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6437,11 +6437,13 @@ "supports_computer_use": true }, "claude-4-sonnet-20250514": { - "max_tokens": 64000, - "max_input_tokens": 200000, - "max_output_tokens": 64000, + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, + "input_cost_per_token_above_200k_tokens": 6e-06, + "output_cost_per_token_above_200k_tokens": 2.25e-05, "search_context_cost_per_query": { "search_context_size_low": 0.01, "search_context_size_medium": 0.01, @@ -6449,6 +6451,8 @@ }, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, + "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, "litellm_provider": "anthropic", "mode": "chat", "supports_function_calling": true, From d39b2e8888820966574ffc80a301be4e9fffe485 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 19 Aug 2025 16:56:13 +0530 Subject: [PATCH 13/21] Add test for long context cost calculation --- tests/test_litellm/test_cost_calculator.py | 119 +++++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 4efba133f1..e229effb5f 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -480,3 +480,122 @@ def test_gemini_25_implicit_caching_cost(): ), f"Expected cost {expected_cost}, but got {result}" print(f"✓ Gemini 2.5 implicit caching cost calculation is correct: ${result:.8f}") + + +def test_log_context_cost_calculation(): + """ + Test that log context cost calculation works correctly with tiered pricing. + + This test verifies that when using extended context (above 200k tokens), + the log context costs are calculated using the appropriate tiered rates. + """ + from litellm import completion_cost + from litellm.types.utils import ( + Choices, + Message, + ModelResponse, + PromptTokensDetailsWrapper, + Usage, + ) + + # Create a mock response with extended context usage + extended_context_response = ModelResponse( + id="test-extended-context-response", + created=1750733889, + model="claude-4-sonnet-20250514", + object="chat.completion", + system_fingerprint=None, + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message( + content="This is a test response for extended context cost calculation.", + role="assistant", + tool_calls=None, + function_call=None, + ), + ) + ], + usage=Usage( + total_tokens=350000, # Above 200k threshold + prompt_tokens=300000, # Above 200k threshold + completion_tokens=50000, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=300000, + cached_tokens=0, # No cache hits + audio_tokens=None, + image_tokens=None, + character_count=None, + video_length_seconds=None, + ), + completion_tokens_details=None, + _cache_creation_input_tokens=1000, # Some tokens added to cache + ), + ) + + # Calculate the cost using the extended context model + result = completion_cost( + completion_response=extended_context_response, + model="claude-4-sonnet-20250514", + custom_llm_provider="anthropic", + ) + + # Debug: Print the actual result + print(f"DEBUG: Actual cost result: ${result:.6f}") + + # Get model info to understand the pricing + from litellm import get_model_info + model_info = get_model_info(model="claude-4-sonnet-20250514", custom_llm_provider="anthropic") + + # Calculate expected cost based on actual model pricing + input_cost_per_token = model_info.get("input_cost_per_token", 0) + output_cost_per_token = model_info.get("output_cost_per_token", 0) + cache_creation_cost_per_token = model_info.get("cache_creation_input_token_cost", 0) + + # Check if tiered pricing is applied + input_cost_above_200k = model_info.get("input_cost_per_token_above_200k_tokens", input_cost_per_token) + output_cost_above_200k = model_info.get("output_cost_per_token_above_200k_tokens", output_cost_per_token) + cache_creation_above_200k = model_info.get("cache_creation_input_token_cost_above_200k_tokens", cache_creation_cost_per_token) + + print(f"DEBUG: Base input cost per token: ${input_cost_per_token:.2e}") + print(f"DEBUG: Base output cost per token: ${output_cost_per_token:.2e}") + print(f"DEBUG: Base cache creation cost per token: ${cache_creation_cost_per_token:.2e}") + + # Handle tiered pricing - if not available, use base pricing + if input_cost_above_200k is not None: + print(f"DEBUG: Tiered input cost per token (>200k): ${input_cost_above_200k:.2e}") + else: + print(f"DEBUG: No tiered input pricing available, using base pricing") + input_cost_above_200k = input_cost_per_token + + if output_cost_above_200k is not None: + print(f"DEBUG: Tiered output cost per token (>200k): ${output_cost_above_200k:.2e}") + else: + print(f"DEBUG: No tiered output pricing available, using base pricing") + output_cost_above_200k = output_cost_per_token + + if cache_creation_above_200k is not None: + print(f"DEBUG: Tiered cache creation cost per token (>200k): ${cache_creation_above_200k:.2e}") + else: + print(f"DEBUG: No tiered cache creation pricing available, using base pricing") + cache_creation_above_200k = cache_creation_cost_per_token + + # Since we're above 200k tokens, we should use tiered pricing if available + expected_input_cost = 300000 * input_cost_above_200k + expected_output_cost = 50000 * output_cost_above_200k + expected_cache_cost = 1000 * cache_creation_above_200k + expected_total = expected_input_cost + expected_output_cost + expected_cache_cost + + print(f"DEBUG: Expected total: ${expected_total:.6f}") + + # Allow for small floating point differences + assert ( + abs(result - expected_total) < 1e-6 + ), f"Expected cost ${expected_total:.6f}, but got ${result:.6f}" + + print(f"✓ Log context cost calculation with tiered pricing is correct: ${result:.6f}") + print(f" - Input tokens (300k): ${expected_input_cost:.6f}") + print(f" - Output tokens (50k): ${expected_output_cost:.6f}") + print(f" - Cache creation (1k): ${expected_cache_cost:.6f}") + print(f" - Total: ${result:.6f}") From 195ea6515e568e4379ae48b019b96effcd0a870c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 19 Aug 2025 10:26:25 -0700 Subject: [PATCH 14/21] [Feat] Datadog LLM Observability - Add support for tracing guardrail input/output (#13767) * add guardrail information on DD LLM Obs * test_guardrail_information_in_metadata --- .../integrations/datadog/datadog_llm_obs.py | 1 + .../datadog/test_datadog_llm_observability.py | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index 8b1838c1ed..4f9c640977 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -422,6 +422,7 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger): "cache_hit": standard_logging_payload.get("cache_hit", "unknown"), "cache_key": standard_logging_payload.get("cache_key", "unknown"), "saved_cache_cost": standard_logging_payload.get("saved_cache_cost", 0), + "guardrail_information": standard_logging_payload.get("guardrail_information", None), } ######################################################### diff --git a/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py b/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py index 25263e31f9..b4575a7ebd 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py @@ -435,6 +435,8 @@ def create_standard_logging_payload_with_latency_metrics() -> StandardLoggingPay start_time=1234567890.0, end_time=1234567890.5, duration=0.5, # 500ms + guardrail_request={"input": "test input message", "user_id": "test_user"}, + guardrail_response={"output": "filtered output", "flagged": False, "score": 0.1}, ) hidden_params = StandardLoggingHiddenParams( @@ -567,3 +569,46 @@ def test_latency_metrics_edge_cases(mock_env_vars): ) metadata = logger._get_dd_llm_obs_payload_metadata(standard_payload) assert "guardrail_overhead_time_ms" not in metadata + + +def test_guardrail_information_in_metadata(mock_env_vars): + """Test that guardrail_information is included in metadata with input/output fields""" + with patch('litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client'), \ + patch('asyncio.create_task'): + logger = DataDogLLMObsLogger() + + # Create a standard payload with guardrail information + standard_payload = create_standard_logging_payload_with_latency_metrics() + + kwargs = { + "standard_logging_object": standard_payload, + "litellm_params": {"metadata": {}} + } + + start_time = datetime.now() + end_time = datetime.now() + + # Create the payload and verify guardrail_information is in metadata + payload = logger.create_llm_obs_payload(kwargs, start_time, end_time) + metadata = payload["meta"]["metadata"] + + # Verify guardrail_information is present in metadata + assert "guardrail_information" in metadata + assert metadata["guardrail_information"] is not None + + # Verify the guardrail information structure + guardrail_info = metadata["guardrail_information"] + assert guardrail_info["guardrail_name"] == "test_guardrail" + assert guardrail_info["guardrail_status"] == "success" + assert guardrail_info["duration"] == 0.5 + + # Verify input/output fields are present + assert "guardrail_request" in guardrail_info + assert "guardrail_response" in guardrail_info + + # Validate the input/output content + assert guardrail_info["guardrail_request"]["input"] == "test input message" + assert guardrail_info["guardrail_request"]["user_id"] == "test_user" + assert guardrail_info["guardrail_response"]["output"] == "filtered output" + assert guardrail_info["guardrail_response"]["flagged"] == False + assert guardrail_info["guardrail_response"]["score"] == 0.1 From fa81b6c63929663c4676b097641d4893fcbbd609 Mon Sep 17 00:00:00 2001 From: Sameer Kankute <135028480+kankute-sameer@users.noreply.github.com> Date: Tue, 19 Aug 2025 23:14:22 +0530 Subject: [PATCH 15/21] Update test_cost_calculator.py --- tests/test_litellm/test_cost_calculator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 87a98dec6c..5d9e7876cf 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -665,4 +665,3 @@ def test_gemini_25_explicit_caching_cost_direct_usage(): print(f"Expected actual cost: {expected_actual_cost}") assert expected_actual_cost == total_cost - From 48622a4ee71eea76eeddf73b7e3a9004943dc2e2 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 20 Aug 2025 01:47:34 +0530 Subject: [PATCH 16/21] add cache above 200k keys in INTENDED_SCHEMA --- tests/test_litellm/test_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index cb53638d83..da570e5d48 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -497,7 +497,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_computer_use": {"type": "boolean"}, "cache_creation_input_audio_token_cost": {"type": "number"}, "cache_creation_input_token_cost": {"type": "number"}, + "cache_creation_input_token_cost_above_200k_tokens": {"type": "number"}, "cache_read_input_token_cost": {"type": "number"}, + "cache_read_input_token_cost_above_200k_tokens": {"type": "number"}, "cache_read_input_audio_token_cost": {"type": "number"}, "deprecation_date": {"type": "string"}, "input_cost_per_audio_per_second": {"type": "number"}, From b0e34692023ff5d12bdb79d7e7b296d5c50a4851 Mon Sep 17 00:00:00 2001 From: tanjiro <56165694+NANDINI-star@users.noreply.github.com> Date: Wed, 20 Aug 2025 06:00:58 +0900 Subject: [PATCH 17/21] Models page row UI restructure (#13771) * remove unused model dashboard * move model_dashboard to templates folder * moving columns to molecules directory * group model names and provider icon * increase table column name font size. cleanup extra title and description inside the tab * fix column width and truncate string * moved credentials column * combined created by and created at * combine in and out costs * remove edit button * remove 2nd extra tooltip --- ui/litellm-dashboard/src/app/page.tsx | 2 +- .../model_dashboard/model_dashboard.tsx | 63 - .../models}/columns.tsx | 241 +-- .../{ => templates}/model_dashboard.tsx | 1655 +++++++---------- 4 files changed, 811 insertions(+), 1150 deletions(-) delete mode 100644 ui/litellm-dashboard/src/components/model_dashboard/model_dashboard.tsx rename ui/litellm-dashboard/src/components/{model_dashboard => molecules/models}/columns.tsx (52%) rename ui/litellm-dashboard/src/components/{ => templates}/model_dashboard.tsx (52%) diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 22f0e29d11..a6bb6245eb 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -8,7 +8,7 @@ import { Team } from "@/components/key_team_helpers/key_list" import Navbar from "@/components/navbar" import { ThemeProvider } from "@/contexts/ThemeContext" import UserDashboard from "@/components/user_dashboard" -import ModelDashboard from "@/components/model_dashboard" +import ModelDashboard from "@/components/templates/model_dashboard" import ViewUserDashboard from "@/components/view_users" import Teams from "@/components/teams" import Organizations from "@/components/organizations" diff --git a/ui/litellm-dashboard/src/components/model_dashboard/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard/model_dashboard.tsx deleted file mode 100644 index 1f8146aa10..0000000000 --- a/ui/litellm-dashboard/src/components/model_dashboard/model_dashboard.tsx +++ /dev/null @@ -1,63 +0,0 @@ -import { useState } from 'react'; -import { Select, SelectItem, Text } from "@tremor/react"; - -interface ModelData { - team_id: string; - team_name: string; - // Add other properties as needed -} - -interface ModelDashboardProps { - modelData: ModelData[]; -} - -export default function ModelDashboard({ modelData }: ModelDashboardProps) { - const [selectedTeam, setSelectedTeam] = useState(null); - - const getTeamName = (teamId: string): string => { - const team = modelData.find(item => item.team_id === teamId); - return team?.team_name || 'Unknown Team'; - }; - - return ( -
-
-
- Model Management - Add and manage models for the proxy -
- -
-
- Filter by Public Model Name: - -
- -
- Filter by Team: - -
-
-
-
- ); -} \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/model_dashboard/columns.tsx b/ui/litellm-dashboard/src/components/molecules/models/columns.tsx similarity index 52% rename from ui/litellm-dashboard/src/components/model_dashboard/columns.tsx rename to ui/litellm-dashboard/src/components/molecules/models/columns.tsx index a2327dfe71..48034d6579 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/columns.tsx +++ b/ui/litellm-dashboard/src/components/molecules/models/columns.tsx @@ -1,10 +1,10 @@ import { ColumnDef } from "@tanstack/react-table"; import { Button, Badge, Icon } from "@tremor/react"; import { Tooltip } from "antd"; -import { getProviderLogoAndName } from "../provider_info_helpers"; -import { ModelData } from "./types"; -import { TrashIcon, PencilIcon, PencilAltIcon } from "@heroicons/react/outline"; -import DeleteModelButton from "../delete_model_button"; +import { getProviderLogoAndName } from "../../provider_info_helpers"; +import { ModelData } from "../../model_dashboard/types"; +import { TrashIcon, PencilIcon, PencilAltIcon, KeyIcon } from "@heroicons/react/outline"; +import DeleteModelButton from "../../delete_model_button"; import { useState } from "react"; export const columns = ( @@ -21,7 +21,7 @@ export const columns = ( setExpandedRows: (expandedRows: Set) => void, ): ColumnDef[] => [ { - header: "Model ID", + header: () => Model ID, accessorKey: "model_info.id", cell: ({ row }) => { const model = row.original; @@ -38,77 +38,117 @@ export const columns = ( }, }, { - header: "Public Model Name", + header: () => Model Information, accessorKey: "model_name", + size: 250, // Fixed column width cell: ({ row }) => { + const model = row.original; const displayName = getDisplayModelName(row.original) || "-"; + const tooltipContent = ( +
+
Provider: {model.provider || "-"}
+
Public Model Name: {displayName}
+
LiteLLM Model Name: {model.litellm_model_name || "-"}
+
+ ); + return ( - -
- {displayName} + +
+ {/* Provider Icon */} +
+ {model.provider ? ( + {`${model.provider} { + const target = e.target as HTMLImageElement; + const parent = target.parentElement; + if (parent) { + const fallbackDiv = document.createElement('div'); + fallbackDiv.className = 'w-4 h-4 rounded-full bg-gray-200 flex items-center justify-center text-xs'; + fallbackDiv.textContent = model.provider?.charAt(0) || '-'; + parent.replaceChild(fallbackDiv, target); + } + }} + /> + ) : ( +
+ - +
+ )} +
+ + {/* Model Names Container */} +
+ {/* Public Model Name */} +
+ {displayName} +
+ {/* LiteLLM Model Name */} +
+ {model.litellm_model_name || "-"} +
+
); }, }, { - header: "Provider", - accessorKey: "provider", + header: () => Credentials, + accessorKey: "litellm_credential_name", + size: 180, // Fixed column width cell: ({ row }) => { const model = row.original; - return ( -
- {model.provider && ( - {`${model.provider} { - const target = e.target as HTMLImageElement; - const parent = target.parentElement; - if (parent) { - const fallbackDiv = document.createElement('div'); - fallbackDiv.className = 'w-4 h-4 rounded-full bg-gray-200 flex items-center justify-center text-xs'; - fallbackDiv.textContent = model.provider?.charAt(0) || '-'; - parent.replaceChild(fallbackDiv, target); - } - }} - /> - )} -

{model.provider || "-"}

+ const credentialName = model.litellm_params?.litellm_credential_name; + + return credentialName ? ( + +
+ + + {credentialName} + +
+ + ) : ( +
+ + No credentials
); }, }, { - header: "LiteLLM Model Name", - accessorKey: "litellm_model_name", - cell: ({ row }) => { - const model = row.original; - return ( - -
- {model.litellm_model_name || "-"} -
- - ); - }, - }, - { - header: "Created At", - accessorKey: "model_info.created_at", + header: () => Created By, + accessorKey: "model_info.created_by", sortingFn: "datetime", + size: 160, // Fixed column width cell: ({ row }) => { const model = row.original; + const createdBy = model.model_info.created_by; + const createdAt = model.model_info.created_at + ? new Date(model.model_info.created_at).toLocaleDateString() + : null; + return ( - - {model.model_info.created_at ? new Date(model.model_info.created_at).toLocaleDateString() : "-"} - +
+ {/* Created By - Primary */} +
+ {createdBy || "Unknown"} +
+ {/* Created At - Secondary */} +
+ {createdAt || "Unknown date"} +
+
); }, }, { - header: "Updated At", + header: () => Updated At, accessorKey: "model_info.updated_at", sortingFn: "datetime", cell: ({ row }) => { @@ -121,51 +161,45 @@ export const columns = ( }, }, { - header: "Created By", - accessorKey: "model_info.created_by", - cell: ({ row }) => { - const model = row.original; - return ( - - {model.model_info.created_by || "-"} - - ); - }, - }, - { - header: () => ( - - Input Cost - - ), + header: () => Costs, accessorKey: "input_cost", + size: 120, // Fixed column width cell: ({ row }) => { const model = row.original; + const inputCost = model.input_cost; + const outputCost = model.output_cost; + + // If both costs are missing or undefined, show "-" + if (!inputCost && !outputCost) { + return ( +
+ - +
+ ); + } + return ( - 
-          {model.input_cost || "-"}
-
+ +
+ {/* Input Cost - Primary */} + {inputCost && ( +
+ In: ${inputCost} +
+ )} + {/* Output Cost - Secondary */} + {outputCost && ( +
+ Out: ${outputCost} +
+ )} +
+ ); }, }, { - header: () => ( - - Output Cost - - ), - accessorKey: "output_cost", - cell: ({ row }) => { - const model = row.original; - return ( - 
-          {model.output_cost || "-"}
-
- ); - }, - }, - { - header: "Team ID", + header: () => Team ID, accessorKey: "model_info.team_id", cell: ({ row }) => { const model = row.original; @@ -188,7 +222,7 @@ export const columns = ( }, }, { - header: "Model Access Group", + header: () => Model Access Group, accessorKey: "model_info.model_access_group", enableSorting: false, cell: ({ row }) => { @@ -252,23 +286,7 @@ export const columns = ( }, }, { - header: "Credentials", - accessorKey: "litellm_credential_name", - cell: ({ row }) => { - const model = row.original; - return model.litellm_params && model.litellm_params.litellm_credential_name ? ( -
- - {model.litellm_params.litellm_credential_name.slice(0, 7)}... - -
- ) : ( - - - ); - }, - }, - { - header: "Status", + header: () => Status, accessorKey: "model_info.db_model", cell: ({ row }) => { const model = row.original; @@ -292,17 +310,6 @@ export const columns = ( const canEditModel = userRole === "Admin" || model.model_info?.created_by === userID; return (
- { - if (canEditModel) { - setSelectedModelId(model.model_info.id); - setEditModel(true); - } - }} - className={!canEditModel ? "opacity-50 cursor-not-allowed" : "cursor-pointer"} - /> = { @@ -126,7 +110,7 @@ const retry_policy_map: Record = { "RateLimitError (429)": "RateLimitErrorRetries", "ContentPolicyViolationError (400)": "ContentPolicyViolationErrorRetries", "InternalServerError (500)": "InternalServerErrorRetries", -}; +} const ModelDashboard: React.FC = ({ accessToken, @@ -139,142 +123,115 @@ const ModelDashboard: React.FC = ({ premiumUser, teams, }) => { - const [addModelForm] = Form.useForm(); - const [autoRouterForm] = Form.useForm(); - const [modelMap, setModelMap] = useState(null); - const [lastRefreshed, setLastRefreshed] = useState(""); + const [addModelForm] = Form.useForm() + const [autoRouterForm] = Form.useForm() + const [modelMap, setModelMap] = useState(null) + const [lastRefreshed, setLastRefreshed] = useState("") - const [providerModels, setProviderModels] = useState>([]); // Explicitly typing providerModels as a string array + const [providerModels, setProviderModels] = useState>([]) // Explicitly typing providerModels as a string array - const [providerSettings, setProviderSettings] = useState( - [] - ); - const [selectedProvider, setSelectedProvider] = useState( - Providers.OpenAI - ); - const [healthCheckResponse, setHealthCheckResponse] = useState(null); - const [isHealthCheckLoading, setIsHealthCheckLoading] = - useState(false); - const [editModalVisible, setEditModalVisible] = useState(false); + const [providerSettings, setProviderSettings] = useState([]) + const [selectedProvider, setSelectedProvider] = useState(Providers.OpenAI) + const [healthCheckResponse, setHealthCheckResponse] = useState(null) + const [isHealthCheckLoading, setIsHealthCheckLoading] = useState(false) + const [editModalVisible, setEditModalVisible] = useState(false) - const [selectedModel, setSelectedModel] = useState(null); - const [availableModelGroups, setAvailableModelGroups] = useState< - Array - >([]); - const [availableModelAccessGroups, setAvailableModelAccessGroups] = useState< - Array - >([]); - const [selectedModelGroup, setSelectedModelGroup] = useState( - null - ); - const [modelMetrics, setModelMetrics] = useState([]); - const [modelMetricsCategories, setModelMetricsCategories] = useState( - [] - ); - const [streamingModelMetrics, setStreamingModelMetrics] = useState([]); - const [streamingModelMetricsCategories, setStreamingModelMetricsCategories] = - useState([]); - const [modelExceptions, setModelExceptions] = useState([]); - const [allExceptions, setAllExceptions] = useState([]); - const [slowResponsesData, setSlowResponsesData] = useState([]); + const [selectedModel, setSelectedModel] = useState(null) + const [availableModelGroups, setAvailableModelGroups] = useState>([]) + const [availableModelAccessGroups, setAvailableModelAccessGroups] = useState>([]) + const [selectedModelGroup, setSelectedModelGroup] = useState(null) + const [modelMetrics, setModelMetrics] = useState([]) + const [modelMetricsCategories, setModelMetricsCategories] = useState([]) + const [streamingModelMetrics, setStreamingModelMetrics] = useState([]) + const [streamingModelMetricsCategories, setStreamingModelMetricsCategories] = useState([]) + const [modelExceptions, setModelExceptions] = useState([]) + const [allExceptions, setAllExceptions] = useState([]) + const [slowResponsesData, setSlowResponsesData] = useState([]) const [dateValue, setDateValue] = useState({ from: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), to: new Date(), - }); + }) - const [modelGroupRetryPolicy, setModelGroupRetryPolicy] = - useState(null); - const [globalRetryPolicy, setGlobalRetryPolicy] = useState(null); - const [defaultRetry, setDefaultRetry] = useState(0); + const [modelGroupRetryPolicy, setModelGroupRetryPolicy] = useState(null) + const [globalRetryPolicy, setGlobalRetryPolicy] = useState(null) + const [defaultRetry, setDefaultRetry] = useState(0) - const [globalExceptionData, setGlobalExceptionData] = - useState({} as GlobalExceptionActivityData); - const [globalExceptionPerDeployment, setGlobalExceptionPerDeployment] = - useState([]); + const [globalExceptionData, setGlobalExceptionData] = useState( + {} as GlobalExceptionActivityData, + ) + const [globalExceptionPerDeployment, setGlobalExceptionPerDeployment] = useState([]) - const [showAdvancedFilters, setShowAdvancedFilters] = - useState(false); - const [selectedAPIKey, setSelectedAPIKey] = useState(null); - const [selectedCustomer, setSelectedCustomer] = useState(null); + const [showAdvancedFilters, setShowAdvancedFilters] = useState(false) + const [selectedAPIKey, setSelectedAPIKey] = useState(null) + const [selectedCustomer, setSelectedCustomer] = useState(null) - const [allEndUsers, setAllEndUsers] = useState([]); + const [allEndUsers, setAllEndUsers] = useState([]) - const [credentialsList, setCredentialsList] = useState([]); + const [credentialsList, setCredentialsList] = useState([]) // Model Group Alias state - const [modelGroupAlias, setModelGroupAlias] = useState<{[key: string]: string}>({}); + const [modelGroupAlias, setModelGroupAlias] = useState<{ [key: string]: string }>({}) // Add state for advanced settings visibility - const [showAdvancedSettings, setShowAdvancedSettings] = - useState(false); + const [showAdvancedSettings, setShowAdvancedSettings] = useState(false) // Add these state variables - const [selectedModelId, setSelectedModelId] = useState(null); - const [editModel, setEditModel] = useState(false); + const [selectedModelId, setSelectedModelId] = useState(null) + const [editModel, setEditModel] = useState(false) - const [selectedTeamId, setSelectedTeamId] = useState(null); - const [selectedTeam, setSelectedTeam] = useState(null); + const [selectedTeamId, setSelectedTeamId] = useState(null) + const [selectedTeam, setSelectedTeam] = useState(null) - const [selectedTeamFilter, setSelectedTeamFilter] = useState( - null - ); - const [selectedModelAccessGroupFilter, setSelectedModelAccessGroupFilter] = - useState(null); + const [selectedTeamFilter, setSelectedTeamFilter] = useState(null) + const [selectedModelAccessGroupFilter, setSelectedModelAccessGroupFilter] = useState(null) - const [modelNameSearch, setModelNameSearch] = useState(""); + const [modelNameSearch, setModelNameSearch] = useState("") // Add new state for current team and model view mode - const [currentTeam, setCurrentTeam] = useState("personal"); // 'personal' or team_id - const [modelViewMode, setModelViewMode] = useState<"current_team" | "all">( - "current_team" - ); + const [currentTeam, setCurrentTeam] = useState("personal") // 'personal' or team_id + const [modelViewMode, setModelViewMode] = useState<"current_team" | "all">("current_team") - const [showColumnDropdown, setShowColumnDropdown] = useState(false); + const [showColumnDropdown, setShowColumnDropdown] = useState(false) - const [isDropdownOpen, setIsDropdownOpen] = useState(false); - const [expandedRows, setExpandedRows] = useState>(new Set()); - const dropdownRef = useRef(null); - const tableRef = useRef>(null); - const [selectedTabIndex, setSelectedTabIndex] = useState(0); + const [isDropdownOpen, setIsDropdownOpen] = useState(false) + const [expandedRows, setExpandedRows] = useState>(new Set()) + const dropdownRef = useRef(null) + const tableRef = useRef>(null) + const [selectedTabIndex, setSelectedTabIndex] = useState(0) const handleCreateNewModelClick = () => { if (selectedModelId) { - setSelectedModelId(null); + setSelectedModelId(null) } - setSelectedTabIndex(1); - }; + setSelectedTabIndex(1) + } const setProviderModelsFn = (provider: Providers) => { - const _providerModels = getProviderModels(provider, modelMap); - setProviderModels(_providerModels); - console.log(`providerModels: ${_providerModels}`); - }; + const _providerModels = getProviderModels(provider, modelMap) + setProviderModels(_providerModels) + console.log(`providerModels: ${_providerModels}`) + } const updateModelMetrics = async ( modelGroup: string | null, startTime: Date | undefined, - endTime: Date | undefined + endTime: Date | undefined, ) => { - console.log("Updating model metrics for group:", modelGroup); + console.log("Updating model metrics for group:", modelGroup) if (!accessToken || !userID || !userRole || !startTime || !endTime) { - return; + return } - console.log( - "inside updateModelMetrics - startTime:", - startTime, - "endTime:", - endTime - ); - setSelectedModelGroup(modelGroup); + console.log("inside updateModelMetrics - startTime:", startTime, "endTime:", endTime) + setSelectedModelGroup(modelGroup) - let selected_token = selectedAPIKey?.token; + let selected_token = selectedAPIKey?.token if (selected_token === undefined) { - selected_token = null; + selected_token = null } - let selected_customer = selectedCustomer; + let selected_customer = selectedCustomer if (selected_customer === undefined) { - selected_customer = null; + selected_customer = null } try { @@ -286,26 +243,24 @@ const ModelDashboard: React.FC = ({ startTime.toISOString(), endTime.toISOString(), selected_token, - selected_customer - ); - console.log("Model metrics response:", modelMetricsResponse); + selected_customer, + ) + console.log("Model metrics response:", modelMetricsResponse) // Assuming modelMetricsResponse now contains the metric data for the specified model group - setModelMetrics(modelMetricsResponse.data); - setModelMetricsCategories(modelMetricsResponse.all_api_bases); + setModelMetrics(modelMetricsResponse.data) + setModelMetricsCategories(modelMetricsResponse.all_api_bases) const streamingModelMetricsResponse = await streamingModelMetricsCall( accessToken, modelGroup, startTime.toISOString(), - endTime.toISOString() - ); + endTime.toISOString(), + ) // Assuming modelMetricsResponse now contains the metric data for the specified model group - setStreamingModelMetrics(streamingModelMetricsResponse.data); - setStreamingModelMetricsCategories( - streamingModelMetricsResponse.all_api_bases - ); + setStreamingModelMetrics(streamingModelMetricsResponse.data) + setStreamingModelMetricsCategories(streamingModelMetricsResponse.all_api_bases) const modelExceptionsResponse = await modelExceptionsCall( accessToken, @@ -315,11 +270,11 @@ const ModelDashboard: React.FC = ({ startTime.toISOString(), endTime.toISOString(), selected_token, - selected_customer - ); - console.log("Model exceptions response:", modelExceptionsResponse); - setModelExceptions(modelExceptionsResponse.data); - setAllExceptions(modelExceptionsResponse.exception_types); + selected_customer, + ) + console.log("Model exceptions response:", modelExceptionsResponse) + setModelExceptions(modelExceptionsResponse.data) + setAllExceptions(modelExceptionsResponse.exception_types) const slowResponses = await modelMetricsSlowResponsesCall( accessToken, @@ -329,226 +284,209 @@ const ModelDashboard: React.FC = ({ startTime.toISOString(), endTime.toISOString(), selected_token, - selected_customer - ); + selected_customer, + ) - console.log("slowResponses:", slowResponses); + console.log("slowResponses:", slowResponses) - setSlowResponsesData(slowResponses); + setSlowResponsesData(slowResponses) if (modelGroup) { const dailyExceptions = await adminGlobalActivityExceptions( accessToken, startTime?.toISOString().split("T")[0], endTime?.toISOString().split("T")[0], - modelGroup - ); + modelGroup, + ) - setGlobalExceptionData(dailyExceptions); + setGlobalExceptionData(dailyExceptions) - const dailyExceptionsPerDeplyment = - await adminGlobalActivityExceptionsPerDeployment( - accessToken, - startTime?.toISOString().split("T")[0], - endTime?.toISOString().split("T")[0], - modelGroup - ); + const dailyExceptionsPerDeplyment = await adminGlobalActivityExceptionsPerDeployment( + accessToken, + startTime?.toISOString().split("T")[0], + endTime?.toISOString().split("T")[0], + modelGroup, + ) - setGlobalExceptionPerDeployment(dailyExceptionsPerDeplyment); + setGlobalExceptionPerDeployment(dailyExceptionsPerDeplyment) } } catch (error) { - console.error("Failed to fetch model metrics", error); + console.error("Failed to fetch model metrics", error) } - }; + } const fetchCredentials = async (accessToken: string) => { try { - const response: CredentialsResponse = - await credentialListCall(accessToken); - console.log(`credentials: ${JSON.stringify(response)}`); - setCredentialsList(response.credentials); + const response: CredentialsResponse = await credentialListCall(accessToken) + console.log(`credentials: ${JSON.stringify(response)}`) + setCredentialsList(response.credentials) } catch (error) { - console.error("Error fetching credentials:", error); + console.error("Error fetching credentials:", error) } - }; + } useEffect(() => { - updateModelMetrics(selectedModelGroup, dateValue.from, dateValue.to); - }, [selectedAPIKey, selectedCustomer, selectedTeam]); + updateModelMetrics(selectedModelGroup, dateValue.from, dateValue.to) + }, [selectedAPIKey, selectedCustomer, selectedTeam]) useEffect(() => { const handleClickOutside = (event: MouseEvent) => { - if ( - dropdownRef.current && - !dropdownRef.current.contains(event.target as Node) - ) { - setIsDropdownOpen(false); + if (dropdownRef.current && !dropdownRef.current.contains(event.target as Node)) { + setIsDropdownOpen(false) } - }; + } - document.addEventListener("mousedown", handleClickOutside); - return () => document.removeEventListener("mousedown", handleClickOutside); - }, []); + document.addEventListener("mousedown", handleClickOutside) + return () => document.removeEventListener("mousedown", handleClickOutside) + }, []) function formatCreatedAt(createdAt: string | null) { if (createdAt) { - const date = new Date(createdAt); - const options = { month: "long", day: "numeric", year: "numeric" }; - return date.toLocaleDateString("en-US"); + const date = new Date(createdAt) + const options = { month: "long", day: "numeric", year: "numeric" } + return date.toLocaleDateString("en-US") } - return null; + return null } const handleEditClick = (model: any) => { - setSelectedModel(model); - setEditModalVisible(true); - }; + setSelectedModel(model) + setEditModalVisible(true) + } const handleEditCancel = () => { - setEditModalVisible(false); - setSelectedModel(null); - }; + setEditModalVisible(false) + setSelectedModel(null) + } const uploadProps: UploadProps = { name: "file", accept: ".json", beforeUpload: (file) => { if (file.type === "application/json") { - const reader = new FileReader(); + const reader = new FileReader() reader.onload = (e) => { if (e.target) { - const jsonStr = e.target.result as string; - console.log( - `Resetting vertex_credentials to JSON; jsonStr: ${jsonStr}` - ); - addModelForm.setFieldsValue({ vertex_credentials: jsonStr }); - console.log( - "Form values right after setting:", - addModelForm.getFieldsValue() - ); + const jsonStr = e.target.result as string + console.log(`Resetting vertex_credentials to JSON; jsonStr: ${jsonStr}`) + addModelForm.setFieldsValue({ vertex_credentials: jsonStr }) + console.log("Form values right after setting:", addModelForm.getFieldsValue()) } - }; - reader.readAsText(file); + } + reader.readAsText(file) } // Prevent upload - return false; + return false }, onChange(info) { - console.log("Upload onChange triggered with values:", info); - console.log("Current form values:", addModelForm.getFieldsValue()); + console.log("Upload onChange triggered with values:", info) + console.log("Current form values:", addModelForm.getFieldsValue()) if (info.file.status !== "uploading") { - console.log(info.file, info.fileList); + console.log(info.file, info.fileList) } if (info.file.status === "done") { - message.success(`${info.file.name} file uploaded successfully`); + message.success(`${info.file.name} file uploaded successfully`) } else if (info.file.status === "error") { - NotificationManager.fromBackend(`${info.file.name} file upload failed.`); + NotificationManager.fromBackend(`${info.file.name} file upload failed.`) } }, - }; + } const handleRefreshClick = () => { // Update the 'lastRefreshed' state to the current date and time - const currentDate = new Date(); - setLastRefreshed(currentDate.toLocaleString()); - }; + const currentDate = new Date() + setLastRefreshed(currentDate.toLocaleString()) + } const handleSaveRetrySettings = async () => { if (!accessToken) { - console.error("Access token is missing"); - return; + console.error("Access token is missing") + return } try { const payload: any = { router_settings: {}, - }; + } if (selectedModelGroup === "global") { // Only update global retry policy - console.log("Saving global retry policy:", globalRetryPolicy); + console.log("Saving global retry policy:", globalRetryPolicy) if (globalRetryPolicy) { - payload.router_settings.retry_policy = globalRetryPolicy; + payload.router_settings.retry_policy = globalRetryPolicy } - message.success("Global retry settings saved successfully"); + message.success("Global retry settings saved successfully") } else { // Only update model group retry policy - console.log("Saving model group retry policy for", selectedModelGroup, ":", modelGroupRetryPolicy); + console.log("Saving model group retry policy for", selectedModelGroup, ":", modelGroupRetryPolicy) if (modelGroupRetryPolicy) { - payload.router_settings.model_group_retry_policy = modelGroupRetryPolicy; + payload.router_settings.model_group_retry_policy = modelGroupRetryPolicy } - message.success(`Retry settings saved successfully for ${selectedModelGroup}`); + message.success(`Retry settings saved successfully for ${selectedModelGroup}`) } - await setCallbacksCall(accessToken, payload); + await setCallbacksCall(accessToken, payload) } catch (error) { - console.error("Failed to save retry settings:", error); - NotificationManager.fromBackend("Failed to save retry settings"); + console.error("Failed to save retry settings:", error) + NotificationManager.fromBackend("Failed to save retry settings") } - }; - - + } useEffect(() => { if (!accessToken || !token || !userRole || !userID) { - return; + return } const fetchData = async () => { try { // Replace with your actual API call for model data - const modelDataResponse = await modelInfoCall( - accessToken, - userID, - userRole - ); - console.log("Model data response:", modelDataResponse.data); - setModelData(modelDataResponse); - const _providerSettings = await modelSettingsCall(accessToken); + const modelDataResponse = await modelInfoCall(accessToken, userID, userRole) + console.log("Model data response:", modelDataResponse.data) + setModelData(modelDataResponse) + const _providerSettings = await modelSettingsCall(accessToken) if (_providerSettings) { - setProviderSettings(_providerSettings); + setProviderSettings(_providerSettings) } // loop through modelDataResponse and get all`model_name` values - let all_model_groups: Set = new Set(); + let all_model_groups: Set = new Set() for (let i = 0; i < modelDataResponse.data.length; i++) { - const model = modelDataResponse.data[i]; - all_model_groups.add(model.model_name); + const model = modelDataResponse.data[i] + all_model_groups.add(model.model_name) } - console.log("all_model_groups:", all_model_groups); - let _array_model_groups = Array.from(all_model_groups); + console.log("all_model_groups:", all_model_groups) + let _array_model_groups = Array.from(all_model_groups) // sort _array_model_groups alphabetically - _array_model_groups = _array_model_groups.sort(); + _array_model_groups = _array_model_groups.sort() - setAvailableModelGroups(_array_model_groups); + setAvailableModelGroups(_array_model_groups) - let all_model_access_groups: Set = new Set(); + let all_model_access_groups: Set = new Set() for (let i = 0; i < modelDataResponse.data.length; i++) { - const model = modelDataResponse.data[i]; - let model_info: any | null = model.model_info; + const model = modelDataResponse.data[i] + let model_info: any | null = model.model_info if (model_info) { - let access_groups = model_info.access_groups; + let access_groups = model_info.access_groups if (access_groups) { for (let j = 0; j < access_groups.length; j++) { - all_model_access_groups.add(access_groups[j]); + all_model_access_groups.add(access_groups[j]) } } } } - setAvailableModelAccessGroups(Array.from(all_model_access_groups)); + setAvailableModelAccessGroups(Array.from(all_model_access_groups)) - console.log("array_model_groups:", _array_model_groups); - let _initial_model_group = "all"; + console.log("array_model_groups:", _array_model_groups) + let _initial_model_group = "all" if (_array_model_groups.length > 0) { // set selectedModelGroup to the last model group - _initial_model_group = - _array_model_groups[_array_model_groups.length - 1]; - console.log("_initial_model_group:", _initial_model_group); + _initial_model_group = _array_model_groups[_array_model_groups.length - 1] + console.log("_initial_model_group:", _initial_model_group) //setSelectedModelGroup(_initial_model_group); } - console.log("selectedModelGroup:", selectedModelGroup); + console.log("selectedModelGroup:", selectedModelGroup) const modelMetricsResponse = await modelMetricsCall( accessToken, @@ -558,27 +496,25 @@ const ModelDashboard: React.FC = ({ dateValue.from?.toISOString(), dateValue.to?.toISOString(), selectedAPIKey?.token, - selectedCustomer - ); + selectedCustomer, + ) - console.log("Model metrics response:", modelMetricsResponse); + console.log("Model metrics response:", modelMetricsResponse) // Sort by latency (avg_latency_per_token) - setModelMetrics(modelMetricsResponse.data); - setModelMetricsCategories(modelMetricsResponse.all_api_bases); + setModelMetrics(modelMetricsResponse.data) + setModelMetricsCategories(modelMetricsResponse.all_api_bases) const streamingModelMetricsResponse = await streamingModelMetricsCall( accessToken, _initial_model_group, dateValue.from?.toISOString(), - dateValue.to?.toISOString() - ); + dateValue.to?.toISOString(), + ) // Assuming modelMetricsResponse now contains the metric data for the specified model group - setStreamingModelMetrics(streamingModelMetricsResponse.data); - setStreamingModelMetricsCategories( - streamingModelMetricsResponse.all_api_bases - ); + setStreamingModelMetrics(streamingModelMetricsResponse.data) + setStreamingModelMetricsCategories(streamingModelMetricsResponse.all_api_bases) const modelExceptionsResponse = await modelExceptionsCall( accessToken, @@ -588,11 +524,11 @@ const ModelDashboard: React.FC = ({ dateValue.from?.toISOString(), dateValue.to?.toISOString(), selectedAPIKey?.token, - selectedCustomer - ); - console.log("Model exceptions response:", modelExceptionsResponse); - setModelExceptions(modelExceptionsResponse.data); - setAllExceptions(modelExceptionsResponse.exception_types); + selectedCustomer, + ) + console.log("Model exceptions response:", modelExceptionsResponse) + setModelExceptions(modelExceptionsResponse.data) + setAllExceptions(modelExceptionsResponse.exception_types) const slowResponses = await modelMetricsSlowResponsesCall( accessToken, @@ -602,118 +538,102 @@ const ModelDashboard: React.FC = ({ dateValue.from?.toISOString(), dateValue.to?.toISOString(), selectedAPIKey?.token, - selectedCustomer - ); + selectedCustomer, + ) const dailyExceptions = await adminGlobalActivityExceptions( accessToken, dateValue.from?.toISOString().split("T")[0], dateValue.to?.toISOString().split("T")[0], - _initial_model_group - ); + _initial_model_group, + ) - setGlobalExceptionData(dailyExceptions); + setGlobalExceptionData(dailyExceptions) - const dailyExceptionsPerDeplyment = - await adminGlobalActivityExceptionsPerDeployment( - accessToken, - dateValue.from?.toISOString().split("T")[0], - dateValue.to?.toISOString().split("T")[0], - _initial_model_group - ); - - setGlobalExceptionPerDeployment(dailyExceptionsPerDeplyment); - - console.log("dailyExceptions:", dailyExceptions); - - console.log( - "dailyExceptionsPerDeplyment:", - dailyExceptionsPerDeplyment - ); - - console.log("slowResponses:", slowResponses); - - setSlowResponsesData(slowResponses); - - let all_end_users_data = await allEndUsersCall(accessToken); - - setAllEndUsers(all_end_users_data?.end_users); - - const routerSettingsInfo = await getCallbacksCall( + const dailyExceptionsPerDeplyment = await adminGlobalActivityExceptionsPerDeployment( accessToken, - userID, - userRole - ); + dateValue.from?.toISOString().split("T")[0], + dateValue.to?.toISOString().split("T")[0], + _initial_model_group, + ) - let router_settings = routerSettingsInfo.router_settings; + setGlobalExceptionPerDeployment(dailyExceptionsPerDeplyment) - console.log("routerSettingsInfo:", router_settings); + console.log("dailyExceptions:", dailyExceptions) - let model_group_retry_policy = router_settings.model_group_retry_policy; - let default_retries = router_settings.num_retries; + console.log("dailyExceptionsPerDeplyment:", dailyExceptionsPerDeplyment) + + console.log("slowResponses:", slowResponses) + + setSlowResponsesData(slowResponses) + + let all_end_users_data = await allEndUsersCall(accessToken) + + setAllEndUsers(all_end_users_data?.end_users) + + const routerSettingsInfo = await getCallbacksCall(accessToken, userID, userRole) + + let router_settings = routerSettingsInfo.router_settings + + console.log("routerSettingsInfo:", router_settings) + + let model_group_retry_policy = router_settings.model_group_retry_policy + let default_retries = router_settings.num_retries + + console.log("model_group_retry_policy:", model_group_retry_policy) + console.log("default_retries:", default_retries) + setModelGroupRetryPolicy(model_group_retry_policy) + setGlobalRetryPolicy(router_settings.retry_policy) + setDefaultRetry(default_retries) - console.log("model_group_retry_policy:", model_group_retry_policy); - console.log("default_retries:", default_retries); - setModelGroupRetryPolicy(model_group_retry_policy); - setGlobalRetryPolicy(router_settings.retry_policy); - setDefaultRetry(default_retries); - // Set model group alias - const model_group_alias = router_settings.model_group_alias || {}; - setModelGroupAlias(model_group_alias); + const model_group_alias = router_settings.model_group_alias || {} + setModelGroupAlias(model_group_alias) } catch (error) { - console.error("There was an error fetching the model data", error); + console.error("There was an error fetching the model data", error) } - }; + } if (accessToken && token && userRole && userID) { - fetchData(); + fetchData() } const fetchModelMap = async () => { - const data = await modelCostMap(accessToken); - console.log(`received model cost map data: ${Object.keys(data)}`); - setModelMap(data); - }; + const data = await modelCostMap(accessToken) + console.log(`received model cost map data: ${Object.keys(data)}`) + setModelMap(data) + } if (modelMap == null) { - fetchModelMap(); + fetchModelMap() } - handleRefreshClick(); - }, [ - accessToken, - token, - userRole, - userID, - modelMap, - lastRefreshed, - selectedTeam, - ]); + handleRefreshClick() + }, [accessToken, token, userRole, userID, modelMap, lastRefreshed, selectedTeam]) if (!modelData) { - return
Loading...
; + return
Loading...
} if (!accessToken || !token || !userRole || !userID) { - return
Loading...
; + return
Loading...
} - let all_models_on_proxy: any[] = []; - let all_providers: string[] = []; + let all_models_on_proxy: any[] = [] + let all_providers: string[] = [] // loop through model data and edit each row for (let i = 0; i < modelData.data.length; i++) { - let curr_model = modelData.data[i]; - let litellm_model_name = curr_model?.litellm_params?.model; - let custom_llm_provider = curr_model?.litellm_params?.custom_llm_provider; - let model_info = curr_model?.model_info; + let curr_model = modelData.data[i] + let litellm_model_name = curr_model?.litellm_params?.model + let custom_llm_provider = curr_model?.litellm_params?.custom_llm_provider + let model_info = curr_model?.model_info - let defaultProvider = "openai"; - let provider = ""; - let input_cost = "Undefined"; - let output_cost = "Undefined"; - let max_tokens = "Undefined"; - let max_input_tokens = "Undefined"; - let cleanedLitellmParams = {}; + let defaultProvider = "openai" + let provider = "" + let input_cost = "Undefined" + let output_cost = "Undefined" + let max_tokens = "Undefined" + let max_input_tokens = "Undefined" + let cleanedLitellmParams = {} const getProviderFromModel = (model: string) => { /** @@ -721,107 +641,96 @@ const ModelDashboard: React.FC = ({ * - check if model in model map * - return it's litellm_provider, if so */ - console.log(`GET PROVIDER CALLED! - ${modelMap}`); + console.log(`GET PROVIDER CALLED! - ${modelMap}`) if (modelMap !== null && modelMap !== undefined) { if (typeof modelMap == "object" && model in modelMap) { - return modelMap[model]["litellm_provider"]; + return modelMap[model]["litellm_provider"] } } - return "openai"; - }; + return "openai" + } // Check if litellm_model_name is null or undefined if (litellm_model_name) { // Split litellm_model_name based on "/" - let splitModel = litellm_model_name.split("/"); + let splitModel = litellm_model_name.split("/") // Get the first element in the split - let firstElement = splitModel[0]; + let firstElement = splitModel[0] // If there is only one element, default provider to openai - provider = custom_llm_provider; + provider = custom_llm_provider if (!provider) { - provider = - splitModel.length === 1 - ? getProviderFromModel(litellm_model_name) - : firstElement; + provider = splitModel.length === 1 ? getProviderFromModel(litellm_model_name) : firstElement } } else { // litellm_model_name is null or undefined, default provider to openai - provider = "-"; + provider = "-" } if (model_info) { - input_cost = model_info?.input_cost_per_token; - output_cost = model_info?.output_cost_per_token; - max_tokens = model_info?.max_tokens; - max_input_tokens = model_info?.max_input_tokens; + input_cost = model_info?.input_cost_per_token + output_cost = model_info?.output_cost_per_token + max_tokens = model_info?.max_tokens + max_input_tokens = model_info?.max_input_tokens } if (curr_model?.litellm_params) { cleanedLitellmParams = Object.fromEntries( - Object.entries(curr_model?.litellm_params).filter( - ([key]) => key !== "model" && key !== "api_base" - ) - ); + Object.entries(curr_model?.litellm_params).filter(([key]) => key !== "model" && key !== "api_base"), + ) } - modelData.data[i].provider = provider; - modelData.data[i].input_cost = input_cost; - modelData.data[i].output_cost = output_cost; - modelData.data[i].litellm_model_name = litellm_model_name; - all_providers.push(provider); + modelData.data[i].provider = provider + modelData.data[i].input_cost = input_cost + modelData.data[i].output_cost = output_cost + modelData.data[i].litellm_model_name = litellm_model_name + all_providers.push(provider) // Convert Cost in terms of Cost per 1M tokens if (modelData.data[i].input_cost) { - modelData.data[i].input_cost = ( - Number(modelData.data[i].input_cost) * 1000000 - ).toFixed(2); + modelData.data[i].input_cost = (Number(modelData.data[i].input_cost) * 1000000).toFixed(2) } if (modelData.data[i].output_cost) { - modelData.data[i].output_cost = ( - Number(modelData.data[i].output_cost) * 1000000 - ).toFixed(2); + modelData.data[i].output_cost = (Number(modelData.data[i].output_cost) * 1000000).toFixed(2) } - modelData.data[i].max_tokens = max_tokens; - modelData.data[i].max_input_tokens = max_input_tokens; - modelData.data[i].api_base = curr_model?.litellm_params?.api_base; - modelData.data[i].cleanedLitellmParams = cleanedLitellmParams; + modelData.data[i].max_tokens = max_tokens + modelData.data[i].max_input_tokens = max_input_tokens + modelData.data[i].api_base = curr_model?.litellm_params?.api_base + modelData.data[i].cleanedLitellmParams = cleanedLitellmParams - all_models_on_proxy.push(curr_model.model_name); + all_models_on_proxy.push(curr_model.model_name) - console.log(modelData.data[i]); + console.log(modelData.data[i]) } // when users click request access show pop up to allow them to request access if (userRole && userRole == "Admin Viewer") { - const { Title, Paragraph } = Typography; + const { Title, Paragraph } = Typography return (
Access Denied - - Ask your proxy admin for access to view all models - + Ask your proxy admin for access to view all models
- ); + ) } const runHealthCheck = async () => { try { - message.info("Running health check..."); - setIsHealthCheckLoading(true); - setHealthCheckResponse(null); - const response = await healthCheckCall(accessToken); - setHealthCheckResponse(response); + message.info("Running health check...") + setIsHealthCheckLoading(true) + setHealthCheckResponse(null) + const response = await healthCheckCall(accessToken) + setHealthCheckResponse(response) } catch (error) { - console.error("Error running health check:", error); - setHealthCheckResponse("Error running health check"); + console.error("Error running health check:", error) + setHealthCheckResponse("Error running health check") } finally { - setIsHealthCheckLoading(false); + setIsHealthCheckLoading(false) } - }; + } const FilterByContent = (
@@ -834,30 +743,26 @@ const ModelDashboard: React.FC = ({ key="all-keys" value="all-keys" onClick={() => { - setSelectedAPIKey(null); + setSelectedAPIKey(null) }} > All Keys {keys?.map((key: any, index: number) => { - if ( - key && - key["key_alias"] !== null && - key["key_alias"].length > 0 - ) { + if (key && key["key_alias"] !== null && key["key_alias"].length > 0) { return ( { - setSelectedAPIKey(key); + setSelectedAPIKey(key) }} > {key["key_alias"]} - ); + ) } - return null; + return null })} @@ -868,7 +773,7 @@ const ModelDashboard: React.FC = ({ key="all-customers" value="all-customers" onClick={() => { - setSelectedCustomer(null); + setSelectedCustomer(null) }} > All Customers @@ -879,12 +784,12 @@ const ModelDashboard: React.FC = ({ key={index} value={user} onClick={() => { - setSelectedCustomer(user); + setSelectedCustomer(user) }} > {user} - ); + ) })} @@ -894,9 +799,7 @@ const ModelDashboard: React.FC = ({ className="w-64 relative z-50" defaultValue="all" value={selectedTeamFilter ?? "all"} - onValueChange={(value) => - setSelectedTeamFilter(value === "all" ? null : value) - } + onValueChange={(value) => setSelectedTeamFilter(value === "all" ? null : value)} > All Teams {teams @@ -919,9 +822,7 @@ const ModelDashboard: React.FC = ({ className="w-64 relative z-50" defaultValue="all" value={selectedTeamFilter ?? "all"} - onValueChange={(value) => - setSelectedTeamFilter(value === "all" ? null : value) - } + onValueChange={(value) => setSelectedTeamFilter(value === "all" ? null : value)} > All Teams {teams @@ -937,94 +838,82 @@ const ModelDashboard: React.FC = ({
)}
- ); + ) const customTooltip = (props: any) => { - const { payload, active } = props; - if (!active || !payload) return null; + const { payload, active } = props + if (!active || !payload) return null // Extract the date from the first item in the payload array - const date = payload[0]?.payload?.date; + const date = payload[0]?.payload?.date // Sort the payload array by category.value in descending order - let sortedPayload = payload.sort((a: any, b: any) => b.value - a.value); + let sortedPayload = payload.sort((a: any, b: any) => b.value - a.value) // Only show the top 5, the 6th one should be called "X other categories" depending on how many categories were not shown if (sortedPayload.length > 5) { - let remainingItems = sortedPayload.length - 5; - sortedPayload = sortedPayload.slice(0, 5); + let remainingItems = sortedPayload.length - 5 + sortedPayload = sortedPayload.slice(0, 5) sortedPayload.push({ dataKey: `${remainingItems} other deployments`, - value: payload - .slice(5) - .reduce((acc: number, curr: any) => acc + curr.value, 0), + value: payload.slice(5).reduce((acc: number, curr: any) => acc + curr.value, 0), color: "gray", - }); + }) } return (
- {date && ( -

Date: {date}

- )} + {date &&

Date: {date}

} {sortedPayload.map((category: any, idx: number) => { - const roundedValue = parseFloat(category.value.toFixed(5)); - const displayValue = - roundedValue === 0 && category.value > 0 - ? "<0.00001" - : roundedValue.toFixed(5); + const roundedValue = parseFloat(category.value.toFixed(5)) + const displayValue = roundedValue === 0 && category.value > 0 ? "<0.00001" : roundedValue.toFixed(5) return (
-
+

{category.dataKey}

-

- {displayValue} -

+

{displayValue}

- ); + ) })}
- ); - }; + ) + } const handleOk = () => { - console.log("🚀 handleOk called from model dashboard!"); - console.log("Current form values:", addModelForm.getFieldsValue()); - + console.log("🚀 handleOk called from model dashboard!") + console.log("Current form values:", addModelForm.getFieldsValue()) + addModelForm .validateFields() .then((values: any) => { - console.log("✅ Validation passed, submitting:", values); - handleAddModelSubmit(values, accessToken, addModelForm, handleRefreshClick); + console.log("✅ Validation passed, submitting:", values) + handleAddModelSubmit(values, accessToken, addModelForm, handleRefreshClick) }) .catch((error: any) => { - console.error("❌ Validation failed:", error); - console.error("Form errors:", error.errorFields); - const errorMessages = error.errorFields?.map((field: any) => { - return `${field.name.join('.')}: ${field.errors.join(', ')}`; - }).join(' | ') || 'Unknown validation error'; - NotificationManager.fromBackend(`Please fill in the following required fields: ${errorMessages}`); - }); - }; + console.error("❌ Validation failed:", error) + console.error("Form errors:", error.errorFields) + const errorMessages = + error.errorFields + ?.map((field: any) => { + return `${field.name.join(".")}: ${field.errors.join(", ")}` + }) + .join(" | ") || "Unknown validation error" + NotificationManager.fromBackend(`Please fill in the following required fields: ${errorMessages}`) + }) + } - - - console.log(`selectedProvider: ${selectedProvider}`); - console.log(`providerModels.length: ${providerModels.length}`); + console.log(`selectedProvider: ${selectedProvider}`) + console.log(`providerModels.length: ${providerModels.length}`) const providerKey = Object.keys(Providers).find( - (key) => (Providers as { [index: string]: any })[key] === selectedProvider - ); + (key) => (Providers as { [index: string]: any })[key] === selectedProvider, + ) - let dynamicProviderForm: ProviderSettings | undefined = undefined; + let dynamicProviderForm: ProviderSettings | undefined = undefined if (providerKey && providerSettings) { - dynamicProviderForm = providerSettings.find( - (provider) => provider.name === provider_map[providerKey] - ); + dynamicProviderForm = providerSettings.find((provider) => provider.name === provider_map[providerKey]) } // If a team is selected, render TeamInfoView in full page layout @@ -1042,7 +931,7 @@ const ModelDashboard: React.FC = ({ onUpdate={handleRefreshClick} />
- ); + ) } return ( @@ -1053,9 +942,11 @@ const ModelDashboard: React.FC = ({

Model Management

-

- Manage your models and configurations -

+ {!all_admin_roles.includes(userRole) ? ( +

Add models for teams you are an admin for.

+ ) : ( +

Add and manage models for the proxy

+ )}
{selectedModelId ? ( @@ -1063,12 +954,10 @@ const ModelDashboard: React.FC = ({ modelId={selectedModelId} editModel={true} onClose={() => { - setSelectedModelId(null); - setEditModel(false); + setSelectedModelId(null) + setEditModel(false) }} - modelData={modelData.data.find( - (model: any) => model.model_info.id === selectedModelId - )} + modelData={modelData.data.find((model: any) => model.model_info.id === selectedModelId)} accessToken={accessToken} userID={userID} userRole={userRole} @@ -1079,58 +968,32 @@ const ModelDashboard: React.FC = ({ const updatedModelData = { ...modelData, data: modelData.data.map((model: any) => - model.model_info.id === updatedModel.model_info.id - ? updatedModel - : model + model.model_info.id === updatedModel.model_info.id ? updatedModel : model, ), - }; - setModelData(updatedModelData); + } + setModelData(updatedModelData) // Trigger a refresh to update UI - handleRefreshClick(); + handleRefreshClick() }} modelAccessGroups={availableModelAccessGroups} /> ) : ( - +
- {all_admin_roles.includes(userRole) ? ( - All Models - ) : ( - Your Models - )} + {all_admin_roles.includes(userRole) ? All Models : Your Models} Add Model - {all_admin_roles.includes(userRole) && ( - LLM Credentials - )} - {all_admin_roles.includes(userRole) && ( - Pass-Through Endpoints - )} - {all_admin_roles.includes(userRole) && ( - Health Status - )} - {all_admin_roles.includes(userRole) && ( - Model Analytics - )} - {all_admin_roles.includes(userRole) && ( - Model Retry Settings - )} - {all_admin_roles.includes(userRole) && ( - Model Group Alias - )} - {all_admin_roles.includes(userRole) && ( - Price Data Reload - )} + {all_admin_roles.includes(userRole) && LLM Credentials} + {all_admin_roles.includes(userRole) && Pass-Through Endpoints} + {all_admin_roles.includes(userRole) && Health Status} + {all_admin_roles.includes(userRole) && Model Analytics} + {all_admin_roles.includes(userRole) && Model Retry Settings} + {all_admin_roles.includes(userRole) && Model Group Alias} + {all_admin_roles.includes(userRole) && Price Data Reload}
- {lastRefreshed && ( - Last Refreshed: {lastRefreshed} - )} + {lastRefreshed && Last Refreshed: {lastRefreshed}} = ({
-
-
- Model Management - {!all_admin_roles.includes(userRole) ? ( - - Add models for teams you are an admin for. - - ) : ( - - Add and manage models for the proxy - - )} -
-
@@ -1166,32 +1015,23 @@ const ModelDashboard: React.FC = ({
- - Current Team: - + Current Team: - setModelViewMode( - value as "current_team" | "all" - ) - } + onValueChange={(value) => setModelViewMode(value as "current_team" | "all")} >
- - Current Team Models - + Current Team Models
- - All Available Models - + All Available Models
@@ -1286,25 +1113,19 @@ const ModelDashboard: React.FC = ({ onValueChange={setModelNameSearch} />
- + {/* Model Name Filter */}
Filter by Public Model Name: - setSelectedModelAccessGroupFilter( - value === "all" ? null : value - ) + setSelectedModelAccessGroupFilter(value === "all" ? null : value) } > - - All Model Access Groups - - {availableModelAccessGroups.map( - (accessGroup, idx) => ( - - {accessGroup} - - ) - )} + All Model Access Groups + {availableModelAccessGroups.map((accessGroup, idx) => ( + + {accessGroup} + + ))}
@@ -1350,43 +1160,29 @@ const ModelDashboard: React.FC = ({ Showing{" "} {modelData && modelData.data.length > 0 ? modelData.data.filter((model: any) => { - const searchMatch = modelNameSearch === "" || - model.model_name.toLowerCase().includes(modelNameSearch.toLowerCase()); + const searchMatch = + modelNameSearch === "" || + model.model_name.toLowerCase().includes(modelNameSearch.toLowerCase()) const modelNameMatch = selectedModelGroup === "all" || - model.model_name === - selectedModelGroup || - !selectedModelGroup; + model.model_name === selectedModelGroup || + !selectedModelGroup const accessGroupMatch = - selectedModelAccessGroupFilter === - "all" || - model.model_info[ - "access_groups" - ]?.includes( - selectedModelAccessGroupFilter - ) || - !selectedModelAccessGroupFilter; - let teamAccessMatch = true; + selectedModelAccessGroupFilter === "all" || + model.model_info["access_groups"]?.includes(selectedModelAccessGroupFilter) || + !selectedModelAccessGroupFilter + let teamAccessMatch = true if (modelViewMode === "current_team") { if (currentTeam === "personal") { - teamAccessMatch = - model.model_info?.direct_access === - true; + teamAccessMatch = model.model_info?.direct_access === true } else { teamAccessMatch = - model.model_info?.access_via_team_ids?.includes( - currentTeam - ) === true; + model.model_info?.access_via_team_ids?.includes(currentTeam) === true } } - return ( - searchMatch && - modelNameMatch && - accessGroupMatch && - teamAccessMatch - ); + return searchMatch && modelNameMatch && accessGroupMatch && teamAccessMatch }).length : 0}{" "} results @@ -1395,88 +1191,79 @@ const ModelDashboard: React.FC = ({
- { - // Model name search filter - const searchMatch = modelNameSearch === "" || - model.model_name.toLowerCase().includes(modelNameSearch.toLowerCase()); + { + // Model name search filter + const searchMatch = + modelNameSearch === "" || + model.model_name.toLowerCase().includes(modelNameSearch.toLowerCase()) - // Model name filter - const modelNameMatch = selectedModelGroup === "all" || - model.model_name === selectedModelGroup || - !selectedModelGroup || (selectedModelGroup === "wildcard" && model.model_name?.includes('*')); - // Model access group filter - const accessGroupMatch = - selectedModelAccessGroupFilter === "all" || - model.model_info["access_groups"]?.includes( - selectedModelAccessGroupFilter - ) || - !selectedModelAccessGroupFilter; - // Team access filter based on current team and view mode - let teamAccessMatch = true; - if (modelViewMode === "current_team") { - if (currentTeam === "personal") { - // Show only models with direct access - teamAccessMatch = - model.model_info?.direct_access === true; - } else { - // Show only models accessible by the current team - teamAccessMatch = - model.model_info?.access_via_team_ids?.includes( - currentTeam - ) === true; + // Model name filter + const modelNameMatch = + selectedModelGroup === "all" || + model.model_name === selectedModelGroup || + !selectedModelGroup || + (selectedModelGroup === "wildcard" && model.model_name?.includes("*")) + // Model access group filter + const accessGroupMatch = + selectedModelAccessGroupFilter === "all" || + model.model_info["access_groups"]?.includes(selectedModelAccessGroupFilter) || + !selectedModelAccessGroupFilter + // Team access filter based on current team and view mode + let teamAccessMatch = true + if (modelViewMode === "current_team") { + if (currentTeam === "personal") { + // Show only models with direct access + teamAccessMatch = model.model_info?.direct_access === true + } else { + // Show only models accessible by the current team + teamAccessMatch = model.model_info?.access_via_team_ids?.includes(currentTeam) === true + } } - } - // For 'all' mode, show all models (teamAccessMatch remains true) + // For 'all' mode, show all models (teamAccessMatch remains true) - return ( - searchMatch && - modelNameMatch && - accessGroupMatch && - teamAccessMatch - ); - })} - isLoading={false} - table={tableRef} - /> -
-
- - - - - - + return searchMatch && modelNameMatch && accessGroupMatch && teamAccessMatch + })} + isLoading={false} + table={tableRef} + /> +
+
+ + + + + + = ({ value={dateValue} className="mr-2" onValueChange={(value) => { - setDateValue(value); - updateModelMetrics( - selectedModelGroup, - value.from, - value.to - ); + setDateValue(value) + updateModelMetrics(selectedModelGroup, value.from, value.to) }} /> Select Model Group - - - - - - + + + + + @@ -1581,13 +1349,9 @@ const ModelDashboard: React.FC = ({ -

- {" "} - (seconds/token) -

+

(seconds/token)

- average Latency for successfull requests divided - by the total tokens + average Latency for successfull requests divided by the total tokens {modelMetrics && modelMetricsCategories && ( = ({ @@ -1622,12 +1384,9 @@ const ModelDashboard: React.FC = ({ Deployment + Success Responses - Success Responses - - - Slow Responses{" "} -

Success Responses taking 600+s

+ Slow Responses

Success Responses taking 600+s

 @@ -1661,9 +1420,7 @@ const ModelDashboard: React.FC = ({ - - All Up Rate Limit Errors (429) for {selectedModelGroup} - + All Up Rate Limit Errors (429) for {selectedModelGroup} = ({ color: "#535452", }} > - Num Rate Limit Errors{" "} - {globalExceptionData.sum_num_rate_limit_exceptions} + Num Rate Limit Errors {globalExceptionData.sum_num_rate_limit_exceptions} = ({ {premiumUser ? ( <> - {globalExceptionPerDeployment.map( - (globalActivity, index) => ( - - - {globalActivity.api_base - ? globalActivity.api_base - : "Unknown API Base"} - - - - - Num Rate Limit Errors (429){" "} - { - globalActivity.sum_num_rate_limit_exceptions - } - - console.log(v)} - /> - - - - ) - )} + {globalExceptionPerDeployment.map((globalActivity, index) => ( + + {globalActivity.api_base ? globalActivity.api_base : "Unknown API Base"} + + + + Num Rate Limit Errors (429) {globalActivity.sum_num_rate_limit_exceptions} + + console.log(v)} + /> + + + + ))} ) : ( <> {globalExceptionPerDeployment && globalExceptionPerDeployment.length > 0 && - globalExceptionPerDeployment - .slice(0, 1) - .map((globalActivity, index) => ( - - - ✨ Rate Limit Errors by Deployment - -

- Upgrade to see exceptions for all deployments -

- - - {globalActivity.api_base} - - - - Num Rate Limit Errors{" "} - { - globalActivity.sum_num_rate_limit_exceptions - } - - console.log(v)} - /> - - - + globalExceptionPerDeployment.slice(0, 1).map((globalActivity, index) => ( + + ✨ Rate Limit Errors by Deployment +

+ Upgrade to see exceptions for all deployments +

+ + + {globalActivity.api_base} + + + + Num Rate Limit Errors {globalActivity.sum_num_rate_limit_exceptions} + + console.log(v)} + /> + + - ))} + + ))} )} @@ -1791,16 +1526,14 @@ const ModelDashboard: React.FC = ({