From 6742d3cb10eac92109e835f68388dd2651b10ae8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 11:10:10 -0700 Subject: [PATCH 01/28] fix route llm request to allow non-router models --- litellm/proxy/route_llm_request.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index ac9332b219..d5c2e2c087 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -94,9 +94,7 @@ async def route_request( ) elif data["model"] not in router_model_names: - if llm_router.router_general_settings.pass_through_all_models: - return getattr(litellm, f"{route_type}")(**data) - elif ( + if ( llm_router.default_deployment is not None or len(llm_router.pattern_router.patterns) > 0 ): @@ -104,6 +102,8 @@ async def route_request( elif route_type == "amoderation": # moderation endpoint does not require `model` parameter return getattr(llm_router, f"{route_type}")(**data) + else: + return getattr(litellm, f"{route_type}")(**data) elif user_model is not None: return getattr(litellm, f"{route_type}")(**data) From 9be24adaca85fc4f70a9bd0e18417832f95c5e26 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 11:11:19 -0700 Subject: [PATCH 02/28] Add test connection --- .../src/components/add_model/add_model_tab.tsx | 15 ++++++++++++++- .../src/components/model_dashboard.tsx | 1 + 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx index c699c161e3..59f5ed5cbe 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx @@ -9,6 +9,8 @@ import AdvancedSettings from "./advanced_settings"; import { Providers, providerLogoMap, getPlaceholder } from "../provider_info_helpers"; import type { Team } from "../key_team_helpers/key_list"; import { CredentialItem } from "../networking"; +import { testModelConnection } from "./handle_add_model_submit"; + interface AddModelTabProps { form: FormInstance; handleOk: () => void; @@ -22,6 +24,7 @@ interface AddModelTabProps { setShowAdvancedSettings: (show: boolean) => void; teams: Team[] | null; credentials: CredentialItem[]; + accessToken: string; } const { Title, Link } = Typography; @@ -39,7 +42,14 @@ const AddModelTab: React.FC = ({ setShowAdvancedSettings, teams, credentials, + accessToken, }) => { + // Add a function to handle test connection + const handleTestConnection = async () => { + const formValues = form.getFieldsValue(); + await testModelConnection(formValues, accessToken); + }; + return ( <> Add new model @@ -184,7 +194,10 @@ const AddModelTab: React.FC = ({ Need Help? - +
+ + +
diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 0a9c26ac4a..77e422ce43 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -1140,6 +1140,7 @@ const ModelDashboard: React.FC = ({ setShowAdvancedSettings={setShowAdvancedSettings} teams={teams} credentials={credentialsList} + accessToken={accessToken} /> From c84717c9e08564d0119607688e6da797a3777c66 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 11:30:24 -0700 Subject: [PATCH 03/28] add health/test_connection --- .../health_endpoints/_health_endpoints.py | 89 ++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index f52ab15c4e..3383fa753b 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -19,7 +19,11 @@ from litellm.proxy._types import ( WebhookEvent, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.health_check import perform_health_check +from litellm.proxy.health_check import ( + _clean_endpoint_data, + perform_health_check, + run_with_timeout, +) #### Health ENDPOINTS #### @@ -600,3 +604,86 @@ async def health_liveliness_options(): "Access-Control-Allow-Headers": "*", } return Response(headers=response_headers, status_code=200) + + +@router.post( + "/health/test_connection", + tags=["health"], + dependencies=[Depends(user_api_key_auth)], +) +async def test_model_connection( + request: Request, + model: str = fastapi.Body(..., description="The model to test connection with"), + mode: Optional[ + Literal[ + "chat", + "completion", + "embedding", + "audio_speech", + "audio_transcription", + "image_generation", + "batch", + "rerank", + "realtime", + ] + ] = fastapi.Body("chat", description="The mode to test the model with"), + prompt: Optional[str] = fastapi.Body(None, description="Test prompt for the model"), + timeout: Optional[int] = fastapi.Body( + 30, description="Timeout in seconds for the health check" + ), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Test a direct connection to a specific model. + + This endpoint allows you to verify if your proxy can successfully connect to a specific model. + It's useful for troubleshooting model connectivity issues without going through the full proxy routing. + + Example: + ```bash + curl -X POST 'http://localhost:4000/health/test_connection' \\ + -H 'Authorization: Bearer sk-1234' \\ + -H 'Content-Type: application/json' \\ + -d '{ + "model": "openai/gpt-3.5-turbo", + "mode": "chat", + "prompt": "Hello, world!", + "timeout": 30 + }' + ``` + + Returns: + dict: A dictionary containing the health check result with either success information or error details. + """ + try: + # Create basic params for the model + model_params = {"model": model} + + # Run the health check with timeout + result = await run_with_timeout( + litellm.ahealth_check( + model_params, + mode=mode, + prompt=prompt, + input=[prompt] if prompt else ["test from litellm"], + ), + timeout, + ) + + # Clean the result for display + cleaned_result = _clean_endpoint_data({**model_params, **result}, details=True) + + return { + "status": "error" if "error" in result else "success", + "result": cleaned_result, + } + + except Exception as e: + verbose_proxy_logger.error( + f"litellm.proxy.health_endpoints.test_model_connection(): Exception occurred - {str(e)}" + ) + verbose_proxy_logger.debug(traceback.format_exc()) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"error": f"Failed to test connection: {str(e)}"}, + ) From 95c25ccb78feaa539c4b585fde912f428fdf2117 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 11:58:03 -0700 Subject: [PATCH 04/28] docs working test error display --- .../add_model/ConnectionErrorDisplay.tsx | 56 +++++++++++++++++++ .../src/components/networking.tsx | 46 +++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx diff --git a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx b/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx new file mode 100644 index 0000000000..de3a057c32 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx @@ -0,0 +1,56 @@ +import React from 'react'; +import { Typography, Space, Button, Divider } from 'antd'; +import { WarningOutlined, InfoCircleOutlined } from '@ant-design/icons'; +import { ErrorViewer } from '../view_logs/ErrorViewer'; + +const { Text } = Typography; + +interface ConnectionErrorDisplayProps { + error: Error | string; + modelName?: string; + onClose?: () => void; +} + +const ConnectionErrorDisplay: React.FC = ({ + error, + modelName = "this model", + onClose +}) => { + const errorMessage = typeof error === 'string' ? error : error.message; + + // Create an error info object compatible with ErrorViewer + const errorInfo = { + error_message: errorMessage.split('\n')[0], + traceback: errorMessage, + // We don't have these fields from the connection test error, but the component handles undefined + error_class: undefined, + llm_provider: undefined, + error_code: undefined + }; + + return ( + +
+ + + {/* Use the ErrorViewer component for consistent error display */} + + + + +
+ +
+
+ + ); +}; + +export default ConnectionErrorDisplay; \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 19589400a0..1603d2ae26 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -2274,6 +2274,52 @@ export const keyInfoCall = async (accessToken: String, keys: String[]) => { }; +export const testConnectionRequest = async ( + accessToken: string, + requestBody: Record +) => { + try { + console.log("Sending model connection test request:", JSON.stringify(requestBody)); + + // Construct the URL based on environment + const url = proxyBaseUrl ? `${proxyBaseUrl}/health/test_connection` : `/health/test_connection`; + + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + [globalLitellmHeaderName]: `Bearer ${accessToken}` + }, + body: JSON.stringify(requestBody) + }); + + // Check for non-JSON responses first + const contentType = response.headers.get('content-type'); + if (!contentType || !contentType.includes('application/json')) { + const text = await response.text(); + console.error("Received non-JSON response:", text); + throw new Error(`Received non-JSON response (${response.status}: ${response.statusText}). Check network tab for details.`); + } + + const data = await response.json(); + + if (!response.ok || data.status === "error") { + // Handle the specific error format you're receiving + if (data.status === "error" && data.result && data.result.error) { + throw new Error(data.result.error); + } else { + throw new Error(data.error?.message || `Connection test failed: ${response.status} ${response.statusText}`); + } + } + + return data; + } catch (error) { + console.error("Model connection test error:", error); + throw error; + } +}; + +// ... existing code ... export const keyInfoV1Call = async (accessToken: string, key: string) => { try { let url = proxyBaseUrl ? `${proxyBaseUrl}/key/info` : `/key/info`; From 414f6fc66427706263fb495dc740be28933d7a63 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 11:58:23 -0700 Subject: [PATCH 05/28] est Model Connectio --- .../components/add_model/add_model_tab.tsx | 345 ++++++++++-------- 1 file changed, 197 insertions(+), 148 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx index 59f5ed5cbe..362870b562 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx @@ -1,5 +1,5 @@ -import React from "react"; -import { Card, Form, Button, Tooltip, Typography, Select as AntdSelect } from "antd"; +import React, { useState } from "react"; +import { Card, Form, Button, Tooltip, Typography, Select as AntdSelect, Modal } from "antd"; import type { FormInstance } from "antd"; import type { UploadProps } from "antd/es/upload"; import LiteLLMModelNameField from "./litellm_model_name"; @@ -29,6 +29,17 @@ interface AddModelTabProps { const { Title, Link } = Typography; +// Define the available test modes +const TEST_MODES = [ + { value: "chat", label: "Chat" }, + { value: "completion", label: "Completion" }, + { value: "embedding", label: "Embedding" }, + { value: "audio_speech", label: "Audio Speech" }, + { value: "audio_transcription", label: "Audio Transcription" }, + { value: "image_generation", label: "Image Generation" }, + { value: "rerank", label: "Rerank" } +]; + const AddModelTab: React.FC = ({ form, handleOk, @@ -44,166 +55,204 @@ const AddModelTab: React.FC = ({ credentials, accessToken, }) => { + // Add state for test mode + const [testMode, setTestMode] = useState("chat"); + const [isTestModalVisible, setIsTestModalVisible] = useState(false); + // Add a function to handle test connection const handleTestConnection = async () => { const formValues = form.getFieldsValue(); - await testModelConnection(formValues, accessToken); + await testModelConnection(formValues, accessToken, testMode); + setIsTestModalVisible(false); + }; + + // Show test modal with mode selection + const showTestModal = () => { + setIsTestModalVisible(true); }; return ( <> Add new model -
+ <> + {/* Provider Selection */} + + { + setSelectedProvider(value); + setProviderModelsFn(value); + form.setFieldsValue({ + model: [], + model_name: undefined + }); + }} > - <> - {/* Provider Selection */} - ( + - { - setSelectedProvider(value); - setProviderModelsFn(value); - form.setFieldsValue({ - model: [], - model_name: undefined - }); - }} - > - {Object.entries(Providers).map(([providerEnum, providerDisplayName]) => ( - -
- {`${providerEnum} { - // Create a div with provider initial as fallback - const target = e.target as HTMLImageElement; - const parent = target.parentElement; - if (parent) { - const fallbackDiv = document.createElement('div'); - fallbackDiv.className = 'w-5 h-5 rounded-full bg-gray-200 flex items-center justify-center text-xs'; - fallbackDiv.textContent = providerDisplayName.charAt(0); - parent.replaceChild(fallbackDiv, target); - } - }} - /> - {providerDisplayName} -
- - ))} - - - - - {/* Conditionally Render "Public Model Name" */} - - - {/* Credentials */} -
- - Either select existing credentials OR enter new provider credentials below - -
- - - - (option?.label ?? '').toLowerCase().includes(input.toLowerCase()) - } - options={[ - { value: null, label: 'None' }, - ...credentials.map((credential) => ({ - value: credential.credential_name, - label: credential.credential_name - })) - ]} - allowClear - /> - - -
-
- OR -
-
- - - prevValues.litellm_credential_name !== currentValues.litellm_credential_name || - prevValues.provider !== currentValues.provider - } - > - {({ getFieldValue }) => { - const credentialName = getFieldValue('litellm_credential_name'); - console.log("🔑 Credential Name Changed:", credentialName); - // Only show provider specific fields if no credentials selected - if (!credentialName) { - return ( - - ); - } - return ( -
- Using existing credentials - no additional provider fields needed -
- ); - }} - - - - -
- - - Need Help? - - -
- - +
+ {`${providerEnum} { + // Create a div with provider initial as fallback + const target = e.target as HTMLImageElement; + const parent = target.parentElement; + if (parent) { + const fallbackDiv = document.createElement('div'); + fallbackDiv.className = 'w-5 h-5 rounded-full bg-gray-200 flex items-center justify-center text-xs'; + fallbackDiv.textContent = providerDisplayName.charAt(0); + parent.replaceChild(fallbackDiv, target); + } + }} + /> + {providerDisplayName}
+ + ))} + + + + + {/* Conditionally Render "Public Model Name" */} + + + {/* Credentials */} +
+ + Either select existing credentials OR enter new provider credentials below + +
+ + + + (option?.label ?? '').toLowerCase().includes(input.toLowerCase()) + } + options={[ + { value: null, label: 'None' }, + ...credentials.map((credential) => ({ + value: credential.credential_name, + label: credential.credential_name + })) + ]} + allowClear + /> + + +
+
+ OR +
+
+ + + prevValues.litellm_credential_name !== currentValues.litellm_credential_name || + prevValues.provider !== currentValues.provider + } + > + {({ getFieldValue }) => { + const credentialName = getFieldValue('litellm_credential_name'); + console.log("🔑 Credential Name Changed:", credentialName); + // Only show provider specific fields if no credentials selected + if (!credentialName) { + return ( + + ); + } + return ( +
+ Using existing credentials - no additional provider fields needed
- - - - + ); + }} + + + + +
+ + + Need Help? + + +
+ + +
+
+ + + + {/* Test Connection Modal */} + setIsTestModalVisible(false)} + footer={[ + , + + ]} + > +
+ Select the mode to test this model with: +
+ setTestMode(value)} + options={TEST_MODES} + /> +
+ + Different models support different modes. Choose the appropriate mode for your model. + +
+ ); }; From 33c85825def37f3e68ce5ab547238fb37c6fdc13 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 12:15:39 -0700 Subject: [PATCH 06/28] use prepareModelAddRequest --- .../add_model/handle_add_model_submit.tsx | 54 +++++++++++++------ 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx index 0fa08e48af..4a19653a6a 100644 --- a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx +++ b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx @@ -3,11 +3,10 @@ import { provider_map, Providers } from "../provider_info_helpers"; import { modelCreateCall, Model } from "../networking"; -export const handleAddModelSubmit = async ( +export const prepareModelAddRequest = async ( formValues: Record, accessToken: string, form: any, - callback?: ()=>void ) => { try { console.log("handling submit for formValues:", formValues); @@ -135,20 +134,45 @@ export const handleAddModelSubmit = async ( litellmParamsObj[key] = value; } } - - const new_model: Model = { - model_name: modelName, - litellm_params: litellmParamsObj, - model_info: modelInfoObj, - }; - - const response: any = await modelCreateCall(accessToken, new_model); - console.log(`response for model create call: ${response["data"]}`); - } - callback && callback() - form.resetFields(); + return { litellmParamsObj, modelInfoObj, modelName }; + } } catch (error) { message.error("Failed to create model: " + error, 10); } - }; \ No newline at end of file + }; + +export const handleAddModelSubmit = async ( + accessToken: string, + form: any, + callback?: () => void, + ) => { + try { + const formValues = form.getFieldsValue(); + const result = await prepareModelAddRequest(formValues, accessToken, form); + + if (!result) { + return; // Exit if preparation failed + } + + const { litellmParamsObj, modelInfoObj, modelName } = result; + + const new_model: Model = { + model_name: modelName, + litellm_params: litellmParamsObj, + model_info: modelInfoObj, + }; + + const response: any = await modelCreateCall(accessToken, new_model); + console.log(`response for model create call: ${response["data"]}`); + + callback && callback(); + form.resetFields(); + + message.success("Model added successfully"); + } catch (error) { + message.error("Failed to add model: " + error, 10); + } + }; + + From d67fc03e205616d342cfb14113580e155c73ab3f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 12:26:17 -0700 Subject: [PATCH 07/28] fix endpoint --- .../health_endpoints/_health_endpoints.py | 3 +- .../add_model/handle_add_model_submit.tsx | 61 ++++++++++++++++++- 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 3383fa753b..c9b954280d 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -613,7 +613,6 @@ async def health_liveliness_options(): ) async def test_model_connection( request: Request, - model: str = fastapi.Body(..., description="The model to test connection with"), mode: Optional[ Literal[ "chat", @@ -657,7 +656,7 @@ async def test_model_connection( """ try: # Create basic params for the model - model_params = {"model": model} + model_params = await request.json() # Run the health check with timeout result = await run_with_timeout( diff --git a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx index 4a19653a6a..e9ea366e87 100644 --- a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx +++ b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx @@ -1,7 +1,8 @@ import { message } from "antd"; import { provider_map, Providers } from "../provider_info_helpers"; -import { modelCreateCall, Model } from "../networking"; - +import { modelCreateCall, Model, testConnectionRequest } from "../networking"; +import React, { useState } from 'react'; +import ConnectionErrorDisplay from './ConnectionErrorDisplay'; export const prepareModelAddRequest = async ( formValues: Record, @@ -175,4 +176,60 @@ export const handleAddModelSubmit = async ( } }; +export const testModelConnection = async ( + formValues: Record, + accessToken: string, + testMode: string, + setConnectionError?: (error: Error | string | null) => void +) => { + try { + // Prepare the model data using the existing function + const result = await prepareModelAddRequest(formValues, accessToken, null); + + if (!result) { + throw new Error("Failed to prepare model data"); + } + + const { litellmParamsObj, modelInfoObj } = result; + + // Create the request body for the test connection + const requestBody = { + ...litellmParamsObj, // Unfurl the parameters directly + mode: testMode + }; + + // Call the test connection endpoint + const response = await testConnectionRequest(accessToken, requestBody); + + if (response.status === "success") { + message.success("Connection test successful!"); + // Clear any previous error when successful + if (setConnectionError) { + setConnectionError(null); + } + } else { + // Set the error for ConnectionErrorDisplay instead of showing a message + const errorMessage = response.message || "Unknown error"; + if (setConnectionError) { + setConnectionError(errorMessage); + } else { + message.error("Connection test failed: " + errorMessage); + } + } + + return response; + } catch (error) { + console.error("Test connection error:", error); + + // Set the error for ConnectionErrorDisplay + if (setConnectionError) { + setConnectionError(error); + } else { + message.error("Test connection failed: " + error, 10); + } + + throw error; + } +}; + From 8cfd2e65cf3f19d332c62efe6723670e038fdd6e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 12:26:41 -0700 Subject: [PATCH 08/28] ui add health/test_connection --- ui/litellm-dashboard/src/components/networking.tsx | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 1603d2ae26..d4de7aad82 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -2304,17 +2304,22 @@ export const testConnectionRequest = async ( const data = await response.json(); if (!response.ok || data.status === "error") { - // Handle the specific error format you're receiving - if (data.status === "error" && data.result && data.result.error) { - throw new Error(data.result.error); + // Return the error response instead of throwing an error + // This allows the caller to handle the error format properly + if (data.status === "error") { + return data; // Return the full error response } else { - throw new Error(data.error?.message || `Connection test failed: ${response.status} ${response.statusText}`); + return { + status: "error", + message: data.error?.message || `Connection test failed: ${response.status} ${response.statusText}` + }; } } return data; } catch (error) { console.error("Model connection test error:", error); + // For network errors or other exceptions, still throw throw error; } }; From caa589f08334514468344655703812c57cbedf60 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 12:50:16 -0700 Subject: [PATCH 09/28] working test connection --- .../components/add_model/add_model_tab.tsx | 47 ++++++++++++-- .../add_model/handle_add_model_submit.tsx | 56 ---------------- .../add_model/test_connection_handler.tsx | 65 +++++++++++++++++++ 3 files changed, 106 insertions(+), 62 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/add_model/test_connection_handler.tsx diff --git a/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx index 362870b562..521d84d6df 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx @@ -9,7 +9,8 @@ import AdvancedSettings from "./advanced_settings"; import { Providers, providerLogoMap, getPlaceholder } from "../provider_info_helpers"; import type { Team } from "../key_team_helpers/key_list"; import { CredentialItem } from "../networking"; -import { testModelConnection } from "./handle_add_model_submit"; +import { testModelConnection } from "./test_connection_handler"; +import ConnectionErrorDisplay from "./ConnectionErrorDisplay"; interface AddModelTabProps { form: FormInstance; @@ -55,19 +56,34 @@ const AddModelTab: React.FC = ({ credentials, accessToken, }) => { - // Add state for test mode + // Add state for test mode and connection error const [testMode, setTestMode] = useState("chat"); const [isTestModalVisible, setIsTestModalVisible] = useState(false); + const [connectionError, setConnectionError] = useState(null); // Add a function to handle test connection const handleTestConnection = async () => { - const formValues = form.getFieldsValue(); - await testModelConnection(formValues, accessToken, testMode); - setIsTestModalVisible(false); + // Clear any previous errors + setConnectionError(null); + + try { + const formValues = form.getFieldsValue(); + + // Call the existing testModelConnection function + const result = await testModelConnection(formValues, accessToken, testMode, setConnectionError); + + // Only close the modal on success + if (result && result.status === "success") { + setIsTestModalVisible(false); + } + } catch (error) { + console.error("Test connection failed:", error); + } }; // Show test modal with mode selection const showTestModal = () => { + setConnectionError(null); setIsTestModalVisible(true); }; @@ -233,10 +249,16 @@ const AddModelTab: React.FC = ({ , - ]} + width={connectionError ? 700 : 520} >
Select the mode to test this model with: @@ -252,6 +274,19 @@ const AddModelTab: React.FC = ({ Different models support different modes. Choose the appropriate mode for your model.
+ + {/* Render the ConnectionErrorDisplay when there's an error */} + {connectionError && ( +
+ Connection Test Failed +
+ +
+
+ )} ); diff --git a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx index e9ea366e87..dd2219af17 100644 --- a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx +++ b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx @@ -176,60 +176,4 @@ export const handleAddModelSubmit = async ( } }; -export const testModelConnection = async ( - formValues: Record, - accessToken: string, - testMode: string, - setConnectionError?: (error: Error | string | null) => void -) => { - try { - // Prepare the model data using the existing function - const result = await prepareModelAddRequest(formValues, accessToken, null); - - if (!result) { - throw new Error("Failed to prepare model data"); - } - - const { litellmParamsObj, modelInfoObj } = result; - - // Create the request body for the test connection - const requestBody = { - ...litellmParamsObj, // Unfurl the parameters directly - mode: testMode - }; - - // Call the test connection endpoint - const response = await testConnectionRequest(accessToken, requestBody); - - if (response.status === "success") { - message.success("Connection test successful!"); - // Clear any previous error when successful - if (setConnectionError) { - setConnectionError(null); - } - } else { - // Set the error for ConnectionErrorDisplay instead of showing a message - const errorMessage = response.message || "Unknown error"; - if (setConnectionError) { - setConnectionError(errorMessage); - } else { - message.error("Connection test failed: " + errorMessage); - } - } - - return response; - } catch (error) { - console.error("Test connection error:", error); - - // Set the error for ConnectionErrorDisplay - if (setConnectionError) { - setConnectionError(error); - } else { - message.error("Test connection failed: " + error, 10); - } - - throw error; - } -}; - diff --git a/ui/litellm-dashboard/src/components/add_model/test_connection_handler.tsx b/ui/litellm-dashboard/src/components/add_model/test_connection_handler.tsx new file mode 100644 index 0000000000..7376ee06c4 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/test_connection_handler.tsx @@ -0,0 +1,65 @@ +import { message } from "antd"; +import { testConnectionRequest } from "../networking"; +import { prepareModelAddRequest } from "./handle_add_model_submit"; + +export const testModelConnection = async ( + formValues: Record, + accessToken: string, + testMode: string, + setConnectionError?: (error: Error | string | null) => void +) => { + try { + // Prepare the model data using the existing function + const result = await prepareModelAddRequest(formValues, accessToken, null); + + if (!result) { + throw new Error("Failed to prepare model data"); + } + + const { litellmParamsObj, modelInfoObj } = result; + + // Create the request body for the test connection + const requestBody = { + ...litellmParamsObj, // Unfurl the parameters directly + mode: testMode + }; + + // Call the test connection endpoint + const response = await testConnectionRequest(accessToken, requestBody); + + if (response.status === "success") { + message.success("Connection test successful!"); + // Clear any previous error when successful + if (setConnectionError) { + setConnectionError(null); + } + } else { + // Extract the detailed error message from the response + let errorMessage = response.message || "Unknown error"; + + // Check if there's a more detailed error in the result + if (response.result && response.result.error) { + errorMessage = response.result.error; + } + + if (setConnectionError) { + setConnectionError(errorMessage); + } else { + message.error("Connection test failed: " + errorMessage); + } + } + + return response; + } catch (error) { + console.error("Test connection error:", error); + + // Set the error for ConnectionErrorDisplay + if (setConnectionError) { + setConnectionError(error); + } else { + message.error("Test connection failed: " + error, 10); + } + + return { status: "error", message: error instanceof Error ? error.message : String(error) }; + } +}; \ No newline at end of file From a1f5002638487f9a3794269e996fe43b1e777922 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 16:26:22 -0700 Subject: [PATCH 10/28] fix mode --- .../src/components/add_model/handle_add_model_submit.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx index dd2219af17..bbb226cc11 100644 --- a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx +++ b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx @@ -81,6 +81,9 @@ export const prepareModelAddRequest = async ( else if (key === "team_id") { modelInfoObj["team_id"] = value; } + else if (key == "mode") { + modelInfoObj["mode"] = value; + } else if (key === "custom_model_name") { litellmParamsObj["model"] = value; } else if (key == "litellm_extra_params") { From cdd625972c9e19ff8f2f12dc2f2c6d84c3b95126 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 16:39:48 -0700 Subject: [PATCH 11/28] working showing user the raw request / response --- .../add_model/ConnectionErrorDisplay.tsx | 136 +++++++++++++----- .../components/add_model/add_model_tab.tsx | 30 +--- .../add_model/test_connection_handler.tsx | 17 ++- 3 files changed, 115 insertions(+), 68 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx b/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx index de3a057c32..69f1becdc0 100644 --- a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx @@ -1,55 +1,121 @@ import React from 'react'; -import { Typography, Space, Button, Divider } from 'antd'; -import { WarningOutlined, InfoCircleOutlined } from '@ant-design/icons'; -import { ErrorViewer } from '../view_logs/ErrorViewer'; +import { Typography, Space, Button, Divider, message } from 'antd'; +import { WarningOutlined, InfoCircleOutlined, CopyOutlined } from '@ant-design/icons'; +import { testConnectionRequest } from "../networking"; +import { prepareModelAddRequest } from "./handle_add_model_submit"; const { Text } = Typography; interface ConnectionErrorDisplayProps { - error: Error | string; + formValues: Record; + accessToken: string; + testMode: string; modelName?: string; onClose?: () => void; } const ConnectionErrorDisplay: React.FC = ({ - error, - modelName = "this model", + formValues, + accessToken, + testMode, + modelName = "this model", onClose }) => { - const errorMessage = typeof error === 'string' ? error : error.message; - - // Create an error info object compatible with ErrorViewer - const errorInfo = { - error_message: errorMessage.split('\n')[0], - traceback: errorMessage, - // We don't have these fields from the connection test error, but the component handles undefined - error_class: undefined, - llm_provider: undefined, - error_code: undefined + const [error, setError] = React.useState(null); + const [rawRequest, setRawRequest] = React.useState(null); + const [rawResponse, setRawResponse] = React.useState(null); + + const testModelConnection = async () => { + try { + const result = await prepareModelAddRequest(formValues, accessToken, null); + if (!result) throw new Error("Failed to prepare model data"); + + const { litellmParamsObj } = result; + const requestBody = { ...litellmParamsObj, mode: testMode }; + + const response = await testConnectionRequest(accessToken, requestBody); + if (response.status === "success") { + message.success("Connection test successful!"); + setError(null); + } else { + const errorMessage = response.result?.error || response.message || "Unknown error"; + setError(errorMessage); + setRawRequest(requestBody); + setRawResponse(response.result?.raw_request_typed_dict); + } + } catch (error) { + console.error("Test connection error:", error); + setError(error instanceof Error ? error.message : String(error)); + } }; + React.useEffect(() => { + testModelConnection(); + }, []); + + const errorMessage = typeof error === 'string' ? error : error?.message; + + const formatCurlCommand = (apiBase: string, requestBody: Record, requestHeaders: Record) => { + const formattedBody = JSON.stringify(requestBody, null, 2) + .split('\n') + .map(line => ` ${line}`) + .join('\n'); + + const headerString = Object.entries(requestHeaders) + .map(([key, value]) => `-H '${key}: ${value}'`) + .join(' \\\n '); + + return `curl -X POST \\ + ${apiBase} \\ + ${headerString ? `${headerString} \\\n ` : ''}-H 'Content-Type: application/json' \\ + -d '{ +${formattedBody} + }'`; + }; + + const curlCommand = rawResponse ? formatCurlCommand( + rawResponse.raw_request_api_base, + rawResponse.raw_request_body, + rawResponse.raw_request_headers || {} + ) : ''; + return ( - -
- - - {/* Use the ErrorViewer component for consistent error display */} - - - - -
- +
+ + {error && ( +
+ {errorMessage} + +
+

Raw Request

+ 
+              {curlCommand || "No request data"}
+
+ +
+ )} + +
+
- +
); }; diff --git a/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx index 521d84d6df..0760afd4dd 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx @@ -9,7 +9,6 @@ import AdvancedSettings from "./advanced_settings"; import { Providers, providerLogoMap, getPlaceholder } from "../provider_info_helpers"; import type { Team } from "../key_team_helpers/key_list"; import { CredentialItem } from "../networking"; -import { testModelConnection } from "./test_connection_handler"; import ConnectionErrorDisplay from "./ConnectionErrorDisplay"; interface AddModelTabProps { @@ -61,26 +60,6 @@ const AddModelTab: React.FC = ({ const [isTestModalVisible, setIsTestModalVisible] = useState(false); const [connectionError, setConnectionError] = useState(null); - // Add a function to handle test connection - const handleTestConnection = async () => { - // Clear any previous errors - setConnectionError(null); - - try { - const formValues = form.getFieldsValue(); - - // Call the existing testModelConnection function - const result = await testModelConnection(formValues, accessToken, testMode, setConnectionError); - - // Only close the modal on success - if (result && result.status === "success") { - setIsTestModalVisible(false); - } - } catch (error) { - console.error("Test connection failed:", error); - } - }; - // Show test modal with mode selection const showTestModal = () => { setConnectionError(null); @@ -252,7 +231,7 @@ const AddModelTab: React.FC = ({
diff --git a/ui/litellm-dashboard/src/components/add_model/test_connection_handler.tsx b/ui/litellm-dashboard/src/components/add_model/test_connection_handler.tsx index 7376ee06c4..d0b7e85dd4 100644 --- a/ui/litellm-dashboard/src/components/add_model/test_connection_handler.tsx +++ b/ui/litellm-dashboard/src/components/add_model/test_connection_handler.tsx @@ -6,7 +6,7 @@ export const testModelConnection = async ( formValues: Record, accessToken: string, testMode: string, - setConnectionError?: (error: Error | string | null) => void + setConnectionError?: (error: Error | string | null, rawRequest?: any, rawResponse?: any) => void ) => { try { // Prepare the model data using the existing function @@ -20,30 +20,30 @@ export const testModelConnection = async ( // Create the request body for the test connection const requestBody = { - ...litellmParamsObj, // Unfurl the parameters directly + ...litellmParamsObj, mode: testMode }; + console.log("Request Body:", requestBody); // Debugging log + // Call the test connection endpoint const response = await testConnectionRequest(accessToken, requestBody); + console.log("Response:", response); // Debugging log + if (response.status === "success") { message.success("Connection test successful!"); - // Clear any previous error when successful if (setConnectionError) { setConnectionError(null); } } else { - // Extract the detailed error message from the response let errorMessage = response.message || "Unknown error"; - - // Check if there's a more detailed error in the result if (response.result && response.result.error) { errorMessage = response.result.error; } if (setConnectionError) { - setConnectionError(errorMessage); + setConnectionError(errorMessage, requestBody, response.result.raw_request_typed_dict); } else { message.error("Connection test failed: " + errorMessage); } @@ -53,9 +53,8 @@ export const testModelConnection = async ( } catch (error) { console.error("Test connection error:", error); - // Set the error for ConnectionErrorDisplay if (setConnectionError) { - setConnectionError(error); + setConnectionError(error, requestBody, null); } else { message.error("Test connection failed: " + error, 10); } From 4c9eba4b949f738e03a3ac7932b532164bba78b5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 16:47:54 -0700 Subject: [PATCH 12/28] delete bloat file --- .../add_model/test_connection_handler.tsx | 64 ------------------- 1 file changed, 64 deletions(-) delete mode 100644 ui/litellm-dashboard/src/components/add_model/test_connection_handler.tsx diff --git a/ui/litellm-dashboard/src/components/add_model/test_connection_handler.tsx b/ui/litellm-dashboard/src/components/add_model/test_connection_handler.tsx deleted file mode 100644 index d0b7e85dd4..0000000000 --- a/ui/litellm-dashboard/src/components/add_model/test_connection_handler.tsx +++ /dev/null @@ -1,64 +0,0 @@ -import { message } from "antd"; -import { testConnectionRequest } from "../networking"; -import { prepareModelAddRequest } from "./handle_add_model_submit"; - -export const testModelConnection = async ( - formValues: Record, - accessToken: string, - testMode: string, - setConnectionError?: (error: Error | string | null, rawRequest?: any, rawResponse?: any) => void -) => { - try { - // Prepare the model data using the existing function - const result = await prepareModelAddRequest(formValues, accessToken, null); - - if (!result) { - throw new Error("Failed to prepare model data"); - } - - const { litellmParamsObj, modelInfoObj } = result; - - // Create the request body for the test connection - const requestBody = { - ...litellmParamsObj, - mode: testMode - }; - - console.log("Request Body:", requestBody); // Debugging log - - // Call the test connection endpoint - const response = await testConnectionRequest(accessToken, requestBody); - - console.log("Response:", response); // Debugging log - - if (response.status === "success") { - message.success("Connection test successful!"); - if (setConnectionError) { - setConnectionError(null); - } - } else { - let errorMessage = response.message || "Unknown error"; - if (response.result && response.result.error) { - errorMessage = response.result.error; - } - - if (setConnectionError) { - setConnectionError(errorMessage, requestBody, response.result.raw_request_typed_dict); - } else { - message.error("Connection test failed: " + errorMessage); - } - } - - return response; - } catch (error) { - console.error("Test connection error:", error); - - if (setConnectionError) { - setConnectionError(error, requestBody, null); - } else { - message.error("Test connection failed: " + error, 10); - } - - return { status: "error", message: error instanceof Error ? error.message : String(error) }; - } -}; \ No newline at end of file From d7b37f498669b37334b3d4ff262195440695332f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 16:50:32 -0700 Subject: [PATCH 13/28] fix dont show extra modal --- .../add_model/ConnectionErrorDisplay.tsx | 64 +++++++++---- .../components/add_model/add_model_tab.tsx | 96 +++++++++---------- 2 files changed, 87 insertions(+), 73 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx b/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx index 69f1becdc0..acceb0c08c 100644 --- a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx @@ -24,8 +24,11 @@ const ConnectionErrorDisplay: React.FC = ({ const [error, setError] = React.useState(null); const [rawRequest, setRawRequest] = React.useState(null); const [rawResponse, setRawResponse] = React.useState(null); + const [isLoading, setIsLoading] = React.useState(true); + const [isSuccess, setIsSuccess] = React.useState(false); const testModelConnection = async () => { + setIsLoading(true); try { const result = await prepareModelAddRequest(formValues, accessToken, null); if (!result) throw new Error("Failed to prepare model data"); @@ -37,15 +40,20 @@ const ConnectionErrorDisplay: React.FC = ({ if (response.status === "success") { message.success("Connection test successful!"); setError(null); + setIsSuccess(true); } else { const errorMessage = response.result?.error || response.message || "Unknown error"; setError(errorMessage); setRawRequest(requestBody); setRawResponse(response.result?.raw_request_typed_dict); + setIsSuccess(false); } } catch (error) { console.error("Test connection error:", error); setError(error instanceof Error ? error.message : String(error)); + setIsSuccess(false); + } finally { + setIsLoading(false); } }; @@ -81,28 +89,42 @@ ${formattedBody} return (
- - {error && ( -
- {errorMessage} - -
-

Raw Request

- 
-              {curlCommand || "No request data"}
-
- -
+ {isLoading ? ( +
+
Testing connection to {modelName}...
+ {/* You could add a spinner here */}
+ ) : isSuccess ? ( +
+ + Connection to {modelName} successful! + +
+ ) : ( + <> +
+ Connection to {modelName} failed + + {errorMessage} + +
+

Raw Request

+ 
+                {curlCommand || "No request data"}
+
+ +
+
+ )}
diff --git a/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx index 0760afd4dd..66ecc33dc9 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx @@ -57,13 +57,14 @@ const AddModelTab: React.FC = ({ }) => { // Add state for test mode and connection error const [testMode, setTestMode] = useState("chat"); - const [isTestModalVisible, setIsTestModalVisible] = useState(false); - const [connectionError, setConnectionError] = useState(null); + const [isResultModalVisible, setIsResultModalVisible] = useState(false); + const [isTestingConnection, setIsTestingConnection] = useState(false); - // Show test modal with mode selection - const showTestModal = () => { - setConnectionError(null); - setIsTestModalVisible(true); + // Test connection directly when button is clicked + const handleTestConnection = async () => { + setIsTestingConnection(true); + setIsResultModalVisible(true); + // The actual testing is handled in ConnectionErrorDisplay component }; return ( @@ -135,6 +136,20 @@ const AddModelTab: React.FC = ({ {/* Conditionally Render "Public Model Name" */} + + {/* Select Mode */} + + setTestMode(value)} + options={TEST_MODES} + /> + {/* Credentials */}
@@ -202,7 +217,6 @@ const AddModelTab: React.FC = ({ setShowAdvancedSettings={setShowAdvancedSettings} teams={teams} /> -
@@ -211,7 +225,7 @@ const AddModelTab: React.FC = ({
- +
@@ -219,56 +233,34 @@ const AddModelTab: React.FC = ({ - {/* Test Connection Modal */} + {/* Test Connection Results Modal */} setIsTestModalVisible(false)} + title="Connection Test Results" + open={isResultModalVisible} + onCancel={() => { + setIsResultModalVisible(false); + setIsTestingConnection(false); + }} footer={[ - , - ]} - width={connectionError ? 700 : 520} + width={700} > -
- Select the mode to test this model with: -
- setTestMode(value)} - options={TEST_MODES} + { + setIsResultModalVisible(false); + setIsTestingConnection(false); + }} /> -
- - Different models support different modes. Choose the appropriate mode for your model. - -
- - {/* Render the ConnectionErrorDisplay when there's an error */} - {connectionError && ( -
- Connection Test Failed -
- setIsTestModalVisible(false)} - /> -
-
- )} ); From a9fd8de90b573d63e5be3674be7ba43d634f24b5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 17:02:13 -0700 Subject: [PATCH 14/28] cleaner view --- .../add_model/ConnectionErrorDisplay.tsx | 79 ++++++++++++++++--- 1 file changed, 70 insertions(+), 9 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx b/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx index acceb0c08c..96e6b83c36 100644 --- a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx @@ -26,6 +26,7 @@ const ConnectionErrorDisplay: React.FC = ({ const [rawResponse, setRawResponse] = React.useState(null); const [isLoading, setIsLoading] = React.useState(true); const [isSuccess, setIsSuccess] = React.useState(false); + const [showDetails, setShowDetails] = React.useState(false); const testModelConnection = async () => { setIsLoading(true); @@ -61,7 +62,19 @@ const ConnectionErrorDisplay: React.FC = ({ testModelConnection(); }, []); - const errorMessage = typeof error === 'string' ? error : error?.message; + const getCleanErrorMessage = (errorMsg: string) => { + if (!errorMsg) return "Unknown error"; + + const mainError = errorMsg.split('stack trace:')[0].trim(); + + const cleanedError = mainError.replace(/^litellm\.(.*?)Error: /, ''); + + return cleanedError; + }; + + const errorMessage = typeof error === 'string' + ? getCleanErrorMessage(error) + : error?.message ? getCleanErrorMessage(error.message) : "Unknown error"; const formatCurlCommand = (apiBase: string, requestBody: Record, requestHeaders: Record) => { const formattedBody = JSON.stringify(requestBody, null, 2) @@ -103,14 +116,61 @@ ${formattedBody} ) : ( <>
- Connection to {modelName} failed - - {errorMessage} - +
+ + Connection to {modelName} failed +
+ +
+ Error: + {errorMessage} + + {error && ( +
+ +
+ )} +
+ + {showDetails && ( +
+

Troubleshooting Details

+ 
+                  {typeof error === 'string' ? error : JSON.stringify(error, null, 2)}
+
+
+ )} +
-

Raw Request

- 
-                {curlCommand || "No request data"}
+              
API Request

+              
+                {curlCommand || "No request data available"}
               

               
+
); From 446422c8ce45e1dd3e1ce0cc91dae406b9126a5a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 17:08:45 -0700 Subject: [PATCH 15/28] decent test connection --- .../add_model/ConnectionErrorDisplay.tsx | 92 +++++++++++++------ 1 file changed, 62 insertions(+), 30 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx b/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx index 96e6b83c36..7eb7e2a190 100644 --- a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx @@ -101,42 +101,65 @@ ${formattedBody} ) : ''; return ( -
+
{isLoading ? ( -
-
Testing connection to {modelName}...
- {/* You could add a spinner here */} +
+
+ {/* Simple CSS spinner */} +
+
+ Testing connection to {modelName}... +
) : isSuccess ? ( -
- +
+
+ +
+ Connection to {modelName} successful!
) : ( <>
-
- - Connection to {modelName} failed +
+ + Connection to {modelName} failed
- Error: - {errorMessage} + Error: + {errorMessage} {error && ( -
+
@@ -145,15 +168,17 @@ ${formattedBody}
{showDetails && ( -
-

Troubleshooting Details

+
+ Troubleshooting Details 
                   {typeof error === 'string' ? error : JSON.stringify(error, null, 2)}
@@ -161,19 +186,21 @@ ${formattedBody} )}
-

API Request

+ API Request 
                 {curlCommand || "No request data available"}
)} - -
+ +
+ )}
); From 5a6da56058461c4b5b5497691748c0e1120d3511 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 17:21:01 -0700 Subject: [PATCH 16/28] fix endpoint_data --- litellm/main.py | 38 +++++++++++++++---- litellm/proxy/health_check.py | 1 + .../health_endpoints/_health_endpoints.py | 1 + 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 6ae2df517d..4e53f99258 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -74,6 +74,7 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.realtime_api.main import _realtime_health_check from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import RawRequestTypedDict from litellm.utils import ( CustomStreamWrapper, ProviderConfigManager, @@ -2986,9 +2987,11 @@ def completion( # type: ignore # noqa: PLR0915 ) return response response = model_response - elif custom_llm_provider == "snowflake" or model in litellm.snowflake_models: + elif custom_llm_provider == "snowflake" or model in litellm.snowflake_models: try: - client = HTTPHandler(timeout=timeout) if stream is False else None # Keep this here, otherwise, the httpx.client closes and streaming is impossible + client = ( + HTTPHandler(timeout=timeout) if stream is False else None + ) # Keep this here, otherwise, the httpx.client closes and streaming is impossible response = base_llm_http_handler.completion( model=model, messages=messages, @@ -3001,12 +3004,11 @@ def completion( # type: ignore # noqa: PLR0915 optional_params=optional_params, litellm_params=litellm_params, timeout=timeout, # type: ignore - client= client, + client=client, custom_llm_provider=custom_llm_provider, encoding=encoding, stream=stream, - ) - + ) except Exception as e: ## LOGGING - log the original exception returned @@ -3017,7 +3019,7 @@ def completion( # type: ignore # noqa: PLR0915 additional_args={"headers": headers}, ) raise e - + elif custom_llm_provider == "custom": url = litellm.api_base or api_base or "" if url is None or url == "": @@ -5449,6 +5451,17 @@ async def ahealth_check( "x-ms-region": str, } """ + # Map modes to their corresponding health check calls + litellm_logging_obj = Logging( + model="", + messages=[], + stream=False, + call_type="acompletion", + litellm_call_id="1234", + start_time=datetime.datetime.now(), + function_id="1234", + log_raw_request_response=True, + ) try: model: Optional[str] = model_params.get("model", None) if model is None: @@ -5471,7 +5484,8 @@ async def ahealth_check( custom_llm_provider=custom_llm_provider, model_params=model_params, ) - # Map modes to their corresponding health check calls + model_params["litellm_logging_obj"] = litellm_logging_obj + mode_handlers = { "chat": lambda: litellm.acompletion(**model_params), "completion": lambda: litellm.atext_completion( @@ -5536,7 +5550,15 @@ async def ahealth_check( + "\nstack trace: " + stack_trace ) - return {"error": error_to_return} + + raw_request_typed_dict = litellm_logging_obj.model_call_details.get( + "raw_request_typed_dict" + ) + + return { + "error": error_to_return, + "raw_request_typed_dict": raw_request_typed_dict, + } ####### HELPER FUNCTIONS ################ diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index f452dded62..f9455387cc 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -36,6 +36,7 @@ def _clean_endpoint_data(endpoint_data: dict, details: Optional[bool] = True): """ Clean the endpoint data for display to users. """ + endpoint_data.pop("litellm_logging_obj", None) return ( {k: v for k, v in endpoint_data.items() if k not in ILLEGAL_DISPLAY_PARAMS} if details is not False diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index c9b954280d..d269bcf84a 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -657,6 +657,7 @@ async def test_model_connection( try: # Create basic params for the model model_params = await request.json() + model_params.pop("mode") # Run the health check with timeout result = await run_with_timeout( From 9a016029790e42a0ef5045790340618a2c5fd414 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 17:21:34 -0700 Subject: [PATCH 17/28] fix Test Modes --- .../components/add_model/add_model_modes.tsx | 11 +++++++++++ .../src/components/add_model/add_model_tab.tsx | 18 ++++-------------- 2 files changed, 15 insertions(+), 14 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/add_model/add_model_modes.tsx diff --git a/ui/litellm-dashboard/src/components/add_model/add_model_modes.tsx b/ui/litellm-dashboard/src/components/add_model/add_model_modes.tsx new file mode 100644 index 0000000000..2b544fd190 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/add_model_modes.tsx @@ -0,0 +1,11 @@ +// Define the available test modes +export const TEST_MODES = [ + { value: "chat", label: "Chat - /chat/completions" }, + { value: "completion", label: "Completion - /completions" }, + { value: "embedding", label: "Embedding - /embeddings" }, + { value: "audio_speech", label: "Audio Speech - /audio/speech" }, + { value: "audio_transcription", label: "Audio Transcription - /audio/transcriptions" }, + { value: "image_generation", label: "Image Generation - /images/generations" }, + { value: "rerank", label: "Rerank - /rerank" }, + { value: "realtime", label: "Realtime - /realtime"} +]; \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx index 66ecc33dc9..e67dba781e 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx @@ -10,6 +10,7 @@ import { Providers, providerLogoMap, getPlaceholder } from "../provider_info_hel import type { Team } from "../key_team_helpers/key_list"; import { CredentialItem } from "../networking"; import ConnectionErrorDisplay from "./ConnectionErrorDisplay"; +import { TEST_MODES } from "./add_model_modes"; interface AddModelTabProps { form: FormInstance; @@ -29,17 +30,6 @@ interface AddModelTabProps { const { Title, Link } = Typography; -// Define the available test modes -const TEST_MODES = [ - { value: "chat", label: "Chat" }, - { value: "completion", label: "Completion" }, - { value: "embedding", label: "Embedding" }, - { value: "audio_speech", label: "Audio Speech" }, - { value: "audio_transcription", label: "Audio Transcription" }, - { value: "image_generation", label: "Image Generation" }, - { value: "rerank", label: "Rerank" } -]; - const AddModelTab: React.FC = ({ form, handleOk, @@ -139,9 +129,9 @@ const AddModelTab: React.FC = ({ {/* Select Mode */} Date: Fri, 14 Mar 2025 17:27:30 -0700 Subject: [PATCH 18/28] workign run test connection many times --- .../add_model/ConnectionErrorDisplay.tsx | 11 +++++- .../components/add_model/add_model_tab.tsx | 37 ++++++++++++------- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx b/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx index 7eb7e2a190..4a3fc30377 100644 --- a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx @@ -12,6 +12,7 @@ interface ConnectionErrorDisplayProps { testMode: string; modelName?: string; onClose?: () => void; + onTestComplete?: () => void; } const ConnectionErrorDisplay: React.FC = ({ @@ -19,7 +20,8 @@ const ConnectionErrorDisplay: React.FC = ({ accessToken, testMode, modelName = "this model", - onClose + onClose, + onTestComplete }) => { const [error, setError] = React.useState(null); const [rawRequest, setRawRequest] = React.useState(null); @@ -30,6 +32,12 @@ const ConnectionErrorDisplay: React.FC = ({ const testModelConnection = async () => { setIsLoading(true); + setShowDetails(false); + setError(null); + setRawRequest(null); + setRawResponse(null); + setIsSuccess(false); + try { const result = await prepareModelAddRequest(formValues, accessToken, null); if (!result) throw new Error("Failed to prepare model data"); @@ -55,6 +63,7 @@ const ConnectionErrorDisplay: React.FC = ({ setIsSuccess(false); } finally { setIsLoading(false); + if (onTestComplete) onTestComplete(); } }; diff --git a/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx index e67dba781e..1ae625d590 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx @@ -45,16 +45,21 @@ const AddModelTab: React.FC = ({ credentials, accessToken, }) => { - // Add state for test mode and connection error + // State for test mode and connection testing const [testMode, setTestMode] = useState("chat"); const [isResultModalVisible, setIsResultModalVisible] = useState(false); const [isTestingConnection, setIsTestingConnection] = useState(false); + // Using a unique ID to force the ConnectionErrorDisplay to remount and run a fresh test + const [connectionTestId, setConnectionTestId] = useState(""); - // Test connection directly when button is clicked + // Test connection when button is clicked const handleTestConnection = async () => { setIsTestingConnection(true); + // Generate a new test ID (using timestamp for uniqueness) + // This forces React to create a new instance of ConnectionErrorDisplay + setConnectionTestId(`test-${Date.now()}`); + // Show the modal with the fresh test setIsResultModalVisible(true); - // The actual testing is handled in ConnectionErrorDisplay component }; return ( @@ -241,16 +246,22 @@ const AddModelTab: React.FC = ({ ]} width={700} > - { - setIsResultModalVisible(false); - setIsTestingConnection(false); - }} - /> + {/* Only render the ConnectionErrorDisplay when modal is visible and we have a test ID */} + {isResultModalVisible && ( + { + setIsResultModalVisible(false); + setIsTestingConnection(false); + }} + onTestComplete={() => setIsTestingConnection(false)} + /> + )} ); From 5aec90c513d15c50d628616868bb711c8c25a19c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 17:29:53 -0700 Subject: [PATCH 19/28] fix dup close --- .../src/components/add_model/ConnectionErrorDisplay.tsx | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx b/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx index 4a3fc30377..7a97dede7e 100644 --- a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx @@ -232,12 +232,6 @@ ${formattedBody} > View Documentation - - {onClose && ( - - )}
); From 6787d0dabe3d77b3d5852438d7c4ff3c527cece8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 18:33:49 -0700 Subject: [PATCH 20/28] test_model_connection --- litellm/main.py | 11 ++--- .../health_endpoints/_health_endpoints.py | 45 +++++++++++-------- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 4e53f99258..64049c31d1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5487,7 +5487,9 @@ async def ahealth_check( model_params["litellm_logging_obj"] = litellm_logging_obj mode_handlers = { - "chat": lambda: litellm.acompletion(**model_params), + "chat": lambda: litellm.acompletion( + **model_params, + ), "completion": lambda: litellm.atext_completion( **_filter_model_params(model_params), prompt=prompt or "test", @@ -5544,12 +5546,7 @@ async def ahealth_check( "error": f"error:{str(e)}. Missing `mode`. Set the `mode` for the model - https://docs.litellm.ai/docs/proxy/health#embedding-models \nstacktrace: {stack_trace}" } - error_to_return = ( - str(e) - + "\nHave you set 'mode' - https://docs.litellm.ai/docs/proxy/health#embedding-models" - + "\nstack trace: " - + stack_trace - ) + error_to_return = str(e) + "\nstack trace: " + stack_trace raw_request_typed_dict = litellm_logging_obj.model_call_details.get( "raw_request_typed_dict" diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index d269bcf84a..378c94aacb 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -3,13 +3,14 @@ import copy import os import traceback from datetime import datetime, timedelta -from typing import Literal, Optional, Union +from typing import Any, Dict, Literal, Optional, Union import fastapi from fastapi import APIRouter, Depends, HTTPException, Request, Response, status import litellm from litellm._logging import verbose_proxy_logger +from litellm.constants import HEALTH_CHECK_TIMEOUT_SECONDS from litellm.proxy._types import ( AlertType, CallInfo, @@ -21,6 +22,7 @@ from litellm.proxy._types import ( from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.health_check import ( _clean_endpoint_data, + _update_litellm_params_for_health_check, perform_health_check, run_with_timeout, ) @@ -626,9 +628,9 @@ async def test_model_connection( "realtime", ] ] = fastapi.Body("chat", description="The mode to test the model with"), - prompt: Optional[str] = fastapi.Body(None, description="Test prompt for the model"), - timeout: Optional[int] = fastapi.Body( - 30, description="Timeout in seconds for the health check" + litellm_params: Dict = fastapi.Body( + None, + description="Parameters for litellm.completion, litellm.embedding for the health check", ), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): @@ -644,10 +646,14 @@ async def test_model_connection( -H 'Authorization: Bearer sk-1234' \\ -H 'Content-Type: application/json' \\ -d '{ - "model": "openai/gpt-3.5-turbo", - "mode": "chat", - "prompt": "Hello, world!", - "timeout": 30 + "litellm_params": { + "model": "gpt-4", + "custom_llm_provider": "azure_ai", + "litellm_credential_name": null, + "api_key": "6xxxxxxx", + "api_base": "https://litellm8397336933.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-10-21", + }, + "mode": "chat" }' ``` @@ -655,23 +661,26 @@ async def test_model_connection( dict: A dictionary containing the health check result with either success information or error details. """ try: - # Create basic params for the model - model_params = await request.json() - model_params.pop("mode") - - # Run the health check with timeout + # Include health_check_params if provided + litellm_params = _update_litellm_params_for_health_check( + model_info={}, + litellm_params=litellm_params, + ) + mode = mode or litellm_params.pop("mode", None) result = await run_with_timeout( litellm.ahealth_check( - model_params, + model_params=litellm_params, mode=mode, - prompt=prompt, - input=[prompt] if prompt else ["test from litellm"], + prompt="test from litellm", + input=["test from litellm"], ), - timeout, + HEALTH_CHECK_TIMEOUT_SECONDS, ) # Clean the result for display - cleaned_result = _clean_endpoint_data({**model_params, **result}, details=True) + cleaned_result = _clean_endpoint_data( + {**litellm_params, **result}, details=True + ) return { "status": "error" if "error" in result else "success", From fa25c4ba7ace84c665caa528e06017f73dd15ba0 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 18:34:22 -0700 Subject: [PATCH 21/28] testConnectionRequest --- ui/litellm-dashboard/src/components/networking.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index d4de7aad82..81b9bb93e0 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -2290,7 +2290,11 @@ export const testConnectionRequest = async ( 'Content-Type': 'application/json', [globalLitellmHeaderName]: `Bearer ${accessToken}` }, - body: JSON.stringify(requestBody) + body: JSON.stringify( + { + litellm_params: requestBody + } + ) }); // Check for non-JSON responses first From 66dc49ab929012375fe92e9abef66352770cfb11 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 18:53:14 -0700 Subject: [PATCH 22/28] fix mode --- .../components/add_model/ConnectionErrorDisplay.tsx | 5 ++++- .../components/add_model/handle_add_model_submit.tsx | 11 ++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx b/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx index 7a97dede7e..f52c3b7df0 100644 --- a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx @@ -42,9 +42,12 @@ const ConnectionErrorDisplay: React.FC = ({ const result = await prepareModelAddRequest(formValues, accessToken, null); if (!result) throw new Error("Failed to prepare model data"); - const { litellmParamsObj } = result; + console.log("result from prepareModelAddRequest:", result); + + const { litellmParamsObj, modelInfoObj, modelName: returnedModelName } = result; const requestBody = { ...litellmParamsObj, mode: testMode }; + const response = await testConnectionRequest(accessToken, requestBody); if (response.status === "success") { message.success("Connection test successful!"); diff --git a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx index bbb226cc11..54f79bd52d 100644 --- a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx +++ b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx @@ -72,7 +72,6 @@ export const prepareModelAddRequest = async ( } else if (key == "model") { continue; } - // Check if key is "base_model" else if (key === "base_model") { // Add key-value pair to model_info dictionary @@ -82,7 +81,11 @@ export const prepareModelAddRequest = async ( modelInfoObj["team_id"] = value; } else if (key == "mode") { + console.log("placing mode in modelInfo") modelInfoObj["mode"] = value; + + // remove "mode" from litellmParams + delete litellmParamsObj["mode"]; } else if (key === "custom_model_name") { litellmParamsObj["model"] = value; @@ -147,13 +150,13 @@ export const prepareModelAddRequest = async ( }; export const handleAddModelSubmit = async ( + values: any, accessToken: string, form: any, callback?: () => void, ) => { try { - const formValues = form.getFieldsValue(); - const result = await prepareModelAddRequest(formValues, accessToken, form); + const result = await prepareModelAddRequest(values, accessToken, form); if (!result) { return; // Exit if preparation failed @@ -172,8 +175,6 @@ export const handleAddModelSubmit = async ( callback && callback(); form.resetFields(); - - message.success("Model added successfully"); } catch (error) { message.error("Failed to add model: " + error, 10); } From 68fd735e97b1e3a90e7abbcc65ff2d9a9624a7a6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 18:57:39 -0700 Subject: [PATCH 23/28] fix params to test connection --- .../src/components/add_model/ConnectionErrorDisplay.tsx | 5 ++--- ui/litellm-dashboard/src/components/networking.tsx | 8 +++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx b/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx index f52c3b7df0..a46a1b5cfa 100644 --- a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx @@ -45,10 +45,9 @@ const ConnectionErrorDisplay: React.FC = ({ console.log("result from prepareModelAddRequest:", result); const { litellmParamsObj, modelInfoObj, modelName: returnedModelName } = result; - const requestBody = { ...litellmParamsObj, mode: testMode }; - const response = await testConnectionRequest(accessToken, requestBody); + const response = await testConnectionRequest(accessToken, litellmParamsObj, modelInfoObj?.mode); if (response.status === "success") { message.success("Connection test successful!"); setError(null); @@ -56,7 +55,7 @@ const ConnectionErrorDisplay: React.FC = ({ } else { const errorMessage = response.result?.error || response.message || "Unknown error"; setError(errorMessage); - setRawRequest(requestBody); + setRawRequest(litellmParamsObj); setRawResponse(response.result?.raw_request_typed_dict); setIsSuccess(false); } diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 81b9bb93e0..bb06608b11 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -2276,10 +2276,11 @@ export const keyInfoCall = async (accessToken: String, keys: String[]) => { export const testConnectionRequest = async ( accessToken: string, - requestBody: Record + litellm_params: Record, + mode: string, ) => { try { - console.log("Sending model connection test request:", JSON.stringify(requestBody)); + console.log("Sending model connection test request:", JSON.stringify(litellm_params)); // Construct the URL based on environment const url = proxyBaseUrl ? `${proxyBaseUrl}/health/test_connection` : `/health/test_connection`; @@ -2292,7 +2293,8 @@ export const testConnectionRequest = async ( }, body: JSON.stringify( { - litellm_params: requestBody + litellm_params: litellm_params, + mode: mode, } ) }); From 89817692b6fe963c5c42218437dee8736c9c73da Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 19:59:04 -0700 Subject: [PATCH 24/28] explain litellm mode --- .../src/components/add_model/add_model_tab.tsx | 12 +++++++++++- .../add_model/conditional_public_model_name.tsx | 8 -------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx index 1ae625d590..0994cddc2d 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx @@ -11,6 +11,8 @@ import type { Team } from "../key_team_helpers/key_list"; import { CredentialItem } from "../networking"; import ConnectionErrorDisplay from "./ConnectionErrorDisplay"; import { TEST_MODES } from "./add_model_modes"; +import { Row, Col } from "antd"; +import { Text, TextInput } from "@tremor/react"; interface AddModelTabProps { form: FormInstance; @@ -136,7 +138,7 @@ const AddModelTab: React.FC = ({ = ({ options={TEST_MODES} /> + + + + + Optional - LiteLLM endpoint to use when health checking this model Learn more + + + {/* Credentials */}
diff --git a/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.tsx b/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.tsx index 8e0f6a288b..75ca5ae328 100644 --- a/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.tsx +++ b/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.tsx @@ -98,14 +98,6 @@ const ConditionalPublicModelName: React.FC = () => { size="small" /> - - - - - Model name your users will pass in. - - - ); }; From 880cdc5d84ab560a70c7481dc0f9556ffe646c9c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 21:02:58 -0700 Subject: [PATCH 25/28] fix test connection --- .../add_model/ConnectionErrorDisplay.tsx | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx b/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx index a46a1b5cfa..00c9da9751 100644 --- a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx @@ -38,15 +38,25 @@ const ConnectionErrorDisplay: React.FC = ({ setRawResponse(null); setIsSuccess(false); + // Add a small delay to ensure form values are fully populated + await new Promise(resolve => setTimeout(resolve, 100)); + try { + console.log("Testing connection with form values:", formValues); const result = await prepareModelAddRequest(formValues, accessToken, null); - if (!result) throw new Error("Failed to prepare model data"); + + if (!result) { + console.log("No result from prepareModelAddRequest"); + setError("Failed to prepare model data. Please check your form inputs."); + setIsSuccess(false); + setIsLoading(false); + return; + } - console.log("result from prepareModelAddRequest:", result); + console.log("Result from prepareModelAddRequest:", result); const { litellmParamsObj, modelInfoObj, modelName: returnedModelName } = result; - const response = await testConnectionRequest(accessToken, litellmParamsObj, modelInfoObj?.mode); if (response.status === "success") { message.success("Connection test successful!"); @@ -70,8 +80,14 @@ const ConnectionErrorDisplay: React.FC = ({ }; React.useEffect(() => { - testModelConnection(); - }, []); + // Run the test once when component mounts + // Add a small timeout to ensure form values are ready + const timer = setTimeout(() => { + testModelConnection(); + }, 200); + + return () => clearTimeout(timer); + }, []); // Empty dependency array means this runs once on mount const getCleanErrorMessage = (errorMsg: string) => { if (!errorMsg) return "Unknown error"; From cbf0fa44b4a8093688a0c3972e82f29f48cdbe01 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 21:05:51 -0700 Subject: [PATCH 26/28] undo changes to route llm request --- litellm/proxy/route_llm_request.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index d5c2e2c087..ac9332b219 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -94,7 +94,9 @@ async def route_request( ) elif data["model"] not in router_model_names: - if ( + if llm_router.router_general_settings.pass_through_all_models: + return getattr(litellm, f"{route_type}")(**data) + elif ( llm_router.default_deployment is not None or len(llm_router.pattern_router.patterns) > 0 ): @@ -102,8 +104,6 @@ async def route_request( elif route_type == "amoderation": # moderation endpoint does not require `model` parameter return getattr(llm_router, f"{route_type}")(**data) - else: - return getattr(litellm, f"{route_type}")(**data) elif user_model is not None: return getattr(litellm, f"{route_type}")(**data) From d7e10fee79650c58b43ff52f9553822053cdc7bf Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 21:06:28 -0700 Subject: [PATCH 27/28] fix code quality --- litellm/proxy/health_endpoints/_health_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 378c94aacb..34e7d34bbf 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -3,7 +3,7 @@ import copy import os import traceback from datetime import datetime, timedelta -from typing import Any, Dict, Literal, Optional, Union +from typing import Dict, Literal, Optional, Union import fastapi from fastapi import APIRouter, Depends, HTTPException, Request, Response, status From ae75ce296644746c8b0b7ff40814948f14c7efc5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 14 Mar 2025 21:07:41 -0700 Subject: [PATCH 28/28] reanme toModelConnectionTest --- .../src/components/add_model/add_model_tab.tsx | 2 +- .../src/components/add_model/handle_add_model_submit.tsx | 2 +- ...ConnectionErrorDisplay.tsx => model_connection_test.tsx} | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) rename ui/litellm-dashboard/src/components/add_model/{ConnectionErrorDisplay.tsx => model_connection_test.tsx} (98%) diff --git a/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx index 0994cddc2d..e2115e8467 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_model_tab.tsx @@ -9,7 +9,7 @@ import AdvancedSettings from "./advanced_settings"; import { Providers, providerLogoMap, getPlaceholder } from "../provider_info_helpers"; import type { Team } from "../key_team_helpers/key_list"; import { CredentialItem } from "../networking"; -import ConnectionErrorDisplay from "./ConnectionErrorDisplay"; +import ConnectionErrorDisplay from "./model_connection_test"; import { TEST_MODES } from "./add_model_modes"; import { Row, Col } from "antd"; import { Text, TextInput } from "@tremor/react"; diff --git a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx index 54f79bd52d..d54198854c 100644 --- a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx +++ b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx @@ -2,7 +2,7 @@ import { message } from "antd"; import { provider_map, Providers } from "../provider_info_helpers"; import { modelCreateCall, Model, testConnectionRequest } from "../networking"; import React, { useState } from 'react'; -import ConnectionErrorDisplay from './ConnectionErrorDisplay'; +import ConnectionErrorDisplay from './model_connection_test'; export const prepareModelAddRequest = async ( formValues: Record, diff --git a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx b/ui/litellm-dashboard/src/components/add_model/model_connection_test.tsx similarity index 98% rename from ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx rename to ui/litellm-dashboard/src/components/add_model/model_connection_test.tsx index 00c9da9751..6c96fe318a 100644 --- a/ui/litellm-dashboard/src/components/add_model/ConnectionErrorDisplay.tsx +++ b/ui/litellm-dashboard/src/components/add_model/model_connection_test.tsx @@ -6,7 +6,7 @@ import { prepareModelAddRequest } from "./handle_add_model_submit"; const { Text } = Typography; -interface ConnectionErrorDisplayProps { +interface ModelConnectionTestProps { formValues: Record; accessToken: string; testMode: string; @@ -15,7 +15,7 @@ interface ConnectionErrorDisplayProps { onTestComplete?: () => void; } -const ConnectionErrorDisplay: React.FC = ({ +const ModelConnectionTest: React.FC = ({ formValues, accessToken, testMode, @@ -255,4 +255,4 @@ ${formattedBody} ); }; -export default ConnectionErrorDisplay; \ No newline at end of file +export default ModelConnectionTest; \ No newline at end of file