Merge pull request #9272 from BerriAI/litellm_add_test_connection_button
[Feat] UI - Add Test Connection
This commit is contained in:
Ishaan Jaff
GitHub
commit
bda5fe0fcf
@ -74,6 +74,7 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.realtime_api.main import _realtime_health_check
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import RawRequestTypedDict
|
||||
from litellm.utils import (
|
||||
CustomStreamWrapper,
|
||||
ProviderConfigManager,
|
||||
@ -2986,9 +2987,11 @@ def completion( # type: ignore # noqa: PLR0915
|
||||
)
|
||||
return response
|
||||
response = model_response
|
||||
elif custom_llm_provider == "snowflake" or model in litellm.snowflake_models:
|
||||
elif custom_llm_provider == "snowflake" or model in litellm.snowflake_models:
|
||||
try:
|
||||
client = HTTPHandler(timeout=timeout) if stream is False else None # Keep this here, otherwise, the httpx.client closes and streaming is impossible
|
||||
client = (
|
||||
HTTPHandler(timeout=timeout) if stream is False else None
|
||||
) # Keep this here, otherwise, the httpx.client closes and streaming is impossible
|
||||
response = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
@ -3001,12 +3004,11 @@ def completion( # type: ignore # noqa: PLR0915
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
timeout=timeout, # type: ignore
|
||||
client= client,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=encoding,
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
## LOGGING - log the original exception returned
|
||||
@ -3017,7 +3019,7 @@ def completion( # type: ignore # noqa: PLR0915
|
||||
additional_args={"headers": headers},
|
||||
)
|
||||
raise e
|
||||
|
||||
|
||||
elif custom_llm_provider == "custom":
|
||||
url = litellm.api_base or api_base or ""
|
||||
if url is None or url == "":
|
||||
@ -5449,6 +5451,17 @@ async def ahealth_check(
|
||||
"x-ms-region": str,
|
||||
}
|
||||
"""
|
||||
# Map modes to their corresponding health check calls
|
||||
litellm_logging_obj = Logging(
|
||||
model="",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="acompletion",
|
||||
litellm_call_id="1234",
|
||||
start_time=datetime.datetime.now(),
|
||||
function_id="1234",
|
||||
log_raw_request_response=True,
|
||||
)
|
||||
try:
|
||||
model: Optional[str] = model_params.get("model", None)
|
||||
if model is None:
|
||||
@ -5471,9 +5484,12 @@ async def ahealth_check(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model_params=model_params,
|
||||
)
|
||||
# Map modes to their corresponding health check calls
|
||||
model_params["litellm_logging_obj"] = litellm_logging_obj
|
||||
|
||||
mode_handlers = {
|
||||
"chat": lambda: litellm.acompletion(**model_params),
|
||||
"chat": lambda: litellm.acompletion(
|
||||
**model_params,
|
||||
),
|
||||
"completion": lambda: litellm.atext_completion(
|
||||
**_filter_model_params(model_params),
|
||||
prompt=prompt or "test",
|
||||
@ -5530,13 +5546,16 @@ async def ahealth_check(
|
||||
"error": f"error:{str(e)}. Missing `mode`. Set the `mode` for the model - https://docs.litellm.ai/docs/proxy/health#embedding-models \nstacktrace: {stack_trace}"
|
||||
}
|
||||
|
||||
error_to_return = (
|
||||
str(e)
|
||||
+ "\nHave you set 'mode' - https://docs.litellm.ai/docs/proxy/health#embedding-models"
|
||||
+ "\nstack trace: "
|
||||
+ stack_trace
|
||||
error_to_return = str(e) + "\nstack trace: " + stack_trace
|
||||
|
||||
raw_request_typed_dict = litellm_logging_obj.model_call_details.get(
|
||||
"raw_request_typed_dict"
|
||||
)
|
||||
return {"error": error_to_return}
|
||||
|
||||
return {
|
||||
"error": error_to_return,
|
||||
"raw_request_typed_dict": raw_request_typed_dict,
|
||||
}
|
||||
|
||||
|
||||
####### HELPER FUNCTIONS ################
|
||||
|
||||
@ -36,6 +36,7 @@ def _clean_endpoint_data(endpoint_data: dict, details: Optional[bool] = True):
|
||||
"""
|
||||
Clean the endpoint data for display to users.
|
||||
"""
|
||||
endpoint_data.pop("litellm_logging_obj", None)
|
||||
return (
|
||||
{k: v for k, v in endpoint_data.items() if k not in ILLEGAL_DISPLAY_PARAMS}
|
||||
if details is not False
|
||||
|
||||
@ -3,13 +3,14 @@ import copy
|
||||
import os
|
||||
import traceback
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Literal, Optional, Union
|
||||
from typing import Dict, Literal, Optional, Union
|
||||
|
||||
import fastapi
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import HEALTH_CHECK_TIMEOUT_SECONDS
|
||||
from litellm.proxy._types import (
|
||||
AlertType,
|
||||
CallInfo,
|
||||
@ -19,7 +20,12 @@ from litellm.proxy._types import (
|
||||
WebhookEvent,
|
||||
)
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.health_check import perform_health_check
|
||||
from litellm.proxy.health_check import (
|
||||
_clean_endpoint_data,
|
||||
_update_litellm_params_for_health_check,
|
||||
perform_health_check,
|
||||
run_with_timeout,
|
||||
)
|
||||
|
||||
#### Health ENDPOINTS ####
|
||||
|
||||
@ -600,3 +606,93 @@ async def health_liveliness_options():
|
||||
"Access-Control-Allow-Headers": "*",
|
||||
}
|
||||
return Response(headers=response_headers, status_code=200)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/health/test_connection",
|
||||
tags=["health"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
)
|
||||
async def test_model_connection(
|
||||
request: Request,
|
||||
mode: Optional[
|
||||
Literal[
|
||||
"chat",
|
||||
"completion",
|
||||
"embedding",
|
||||
"audio_speech",
|
||||
"audio_transcription",
|
||||
"image_generation",
|
||||
"batch",
|
||||
"rerank",
|
||||
"realtime",
|
||||
]
|
||||
] = fastapi.Body("chat", description="The mode to test the model with"),
|
||||
litellm_params: Dict = fastapi.Body(
|
||||
None,
|
||||
description="Parameters for litellm.completion, litellm.embedding for the health check",
|
||||
),
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
):
|
||||
"""
|
||||
Test a direct connection to a specific model.
|
||||
|
||||
This endpoint allows you to verify if your proxy can successfully connect to a specific model.
|
||||
It's useful for troubleshooting model connectivity issues without going through the full proxy routing.
|
||||
|
||||
Example:
|
||||
```bash
|
||||
curl -X POST 'http://localhost:4000/health/test_connection' \\
|
||||
-H 'Authorization: Bearer sk-1234' \\
|
||||
-H 'Content-Type: application/json' \\
|
||||
-d '{
|
||||
"litellm_params": {
|
||||
"model": "gpt-4",
|
||||
"custom_llm_provider": "azure_ai",
|
||||
"litellm_credential_name": null,
|
||||
"api_key": "6xxxxxxx",
|
||||
"api_base": "https://litellm8397336933.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-10-21",
|
||||
},
|
||||
"mode": "chat"
|
||||
}'
|
||||
```
|
||||
|
||||
Returns:
|
||||
dict: A dictionary containing the health check result with either success information or error details.
|
||||
"""
|
||||
try:
|
||||
# Include health_check_params if provided
|
||||
litellm_params = _update_litellm_params_for_health_check(
|
||||
model_info={},
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
mode = mode or litellm_params.pop("mode", None)
|
||||
result = await run_with_timeout(
|
||||
litellm.ahealth_check(
|
||||
model_params=litellm_params,
|
||||
mode=mode,
|
||||
prompt="test from litellm",
|
||||
input=["test from litellm"],
|
||||
),
|
||||
HEALTH_CHECK_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
# Clean the result for display
|
||||
cleaned_result = _clean_endpoint_data(
|
||||
{**litellm_params, **result}, details=True
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "error" if "error" in result else "success",
|
||||
"result": cleaned_result,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(
|
||||
f"litellm.proxy.health_endpoints.test_model_connection(): Exception occurred - {str(e)}"
|
||||
)
|
||||
verbose_proxy_logger.debug(traceback.format_exc())
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail={"error": f"Failed to test connection: {str(e)}"},
|
||||
)
|
||||
|
||||
@ -0,0 +1,11 @@
|
||||
// Define the available test modes
|
||||
export const TEST_MODES = [
|
||||
{ value: "chat", label: "Chat - /chat/completions" },
|
||||
{ value: "completion", label: "Completion - /completions" },
|
||||
{ value: "embedding", label: "Embedding - /embeddings" },
|
||||
{ value: "audio_speech", label: "Audio Speech - /audio/speech" },
|
||||
{ value: "audio_transcription", label: "Audio Transcription - /audio/transcriptions" },
|
||||
{ value: "image_generation", label: "Image Generation - /images/generations" },
|
||||
{ value: "rerank", label: "Rerank - /rerank" },
|
||||
{ value: "realtime", label: "Realtime - /realtime"}
|
||||
];
|
||||
@ -1,5 +1,5 @@
|
||||
import React from "react";
|
||||
import { Card, Form, Button, Tooltip, Typography, Select as AntdSelect } from "antd";
|
||||
import React, { useState } from "react";
|
||||
import { Card, Form, Button, Tooltip, Typography, Select as AntdSelect, Modal } from "antd";
|
||||
import type { FormInstance } from "antd";
|
||||
import type { UploadProps } from "antd/es/upload";
|
||||
import LiteLLMModelNameField from "./litellm_model_name";
|
||||
@ -9,6 +9,11 @@ import AdvancedSettings from "./advanced_settings";
|
||||
import { Providers, providerLogoMap, getPlaceholder } from "../provider_info_helpers";
|
||||
import type { Team } from "../key_team_helpers/key_list";
|
||||
import { CredentialItem } from "../networking";
|
||||
import ConnectionErrorDisplay from "./model_connection_test";
|
||||
import { TEST_MODES } from "./add_model_modes";
|
||||
import { Row, Col } from "antd";
|
||||
import { Text, TextInput } from "@tremor/react";
|
||||
|
||||
interface AddModelTabProps {
|
||||
form: FormInstance;
|
||||
handleOk: () => void;
|
||||
@ -22,6 +27,7 @@ interface AddModelTabProps {
|
||||
setShowAdvancedSettings: (show: boolean) => void;
|
||||
teams: Team[] | null;
|
||||
credentials: CredentialItem[];
|
||||
accessToken: string;
|
||||
}
|
||||
|
||||
const { Title, Link } = Typography;
|
||||
@ -39,158 +45,234 @@ const AddModelTab: React.FC<AddModelTabProps> = ({
|
||||
setShowAdvancedSettings,
|
||||
teams,
|
||||
credentials,
|
||||
accessToken,
|
||||
}) => {
|
||||
// State for test mode and connection testing
|
||||
const [testMode, setTestMode] = useState<string>("chat");
|
||||
const [isResultModalVisible, setIsResultModalVisible] = useState<boolean>(false);
|
||||
const [isTestingConnection, setIsTestingConnection] = useState<boolean>(false);
|
||||
// Using a unique ID to force the ConnectionErrorDisplay to remount and run a fresh test
|
||||
const [connectionTestId, setConnectionTestId] = useState<string>("");
|
||||
|
||||
// Test connection when button is clicked
|
||||
const handleTestConnection = async () => {
|
||||
setIsTestingConnection(true);
|
||||
// Generate a new test ID (using timestamp for uniqueness)
|
||||
// This forces React to create a new instance of ConnectionErrorDisplay
|
||||
setConnectionTestId(`test-${Date.now()}`);
|
||||
// Show the modal with the fresh test
|
||||
setIsResultModalVisible(true);
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<Title level={2}>Add new model</Title>
|
||||
<Card>
|
||||
<Form
|
||||
form={form}
|
||||
onFinish={handleOk}
|
||||
labelCol={{ span: 10 }}
|
||||
wrapperCol={{ span: 16 }}
|
||||
labelAlign="left"
|
||||
<Form
|
||||
form={form}
|
||||
onFinish={handleOk}
|
||||
labelCol={{ span: 10 }}
|
||||
wrapperCol={{ span: 16 }}
|
||||
labelAlign="left"
|
||||
>
|
||||
<>
|
||||
{/* Provider Selection */}
|
||||
<Form.Item
|
||||
rules={[{ required: true, message: "Required" }]}
|
||||
label="Provider:"
|
||||
name="custom_llm_provider"
|
||||
tooltip="E.g. OpenAI, Azure OpenAI, Anthropic, Bedrock, etc."
|
||||
labelCol={{ span: 10 }}
|
||||
labelAlign="left"
|
||||
>
|
||||
<AntdSelect
|
||||
showSearch={true}
|
||||
value={selectedProvider}
|
||||
onChange={(value) => {
|
||||
setSelectedProvider(value);
|
||||
setProviderModelsFn(value);
|
||||
form.setFieldsValue({
|
||||
model: [],
|
||||
model_name: undefined
|
||||
});
|
||||
}}
|
||||
>
|
||||
<>
|
||||
{/* Provider Selection */}
|
||||
<Form.Item
|
||||
rules={[{ required: true, message: "Required" }]}
|
||||
label="Provider:"
|
||||
name="custom_llm_provider"
|
||||
tooltip="E.g. OpenAI, Azure OpenAI, Anthropic, Bedrock, etc."
|
||||
labelCol={{ span: 10 }}
|
||||
labelAlign="left"
|
||||
{Object.entries(Providers).map(([providerEnum, providerDisplayName]) => (
|
||||
<AntdSelect.Option
|
||||
key={providerEnum}
|
||||
value={providerEnum}
|
||||
>
|
||||
<AntdSelect
|
||||
showSearch={true}
|
||||
value={selectedProvider}
|
||||
onChange={(value) => {
|
||||
setSelectedProvider(value);
|
||||
setProviderModelsFn(value);
|
||||
form.setFieldsValue({
|
||||
model: [],
|
||||
model_name: undefined
|
||||
});
|
||||
}}
|
||||
>
|
||||
{Object.entries(Providers).map(([providerEnum, providerDisplayName]) => (
|
||||
<AntdSelect.Option
|
||||
key={providerEnum}
|
||||
value={providerEnum}
|
||||
>
|
||||
<div className="flex items-center space-x-2">
|
||||
<img
|
||||
src={providerLogoMap[providerDisplayName]}
|
||||
alt={`${providerEnum} logo`}
|
||||
className="w-5 h-5"
|
||||
onError={(e) => {
|
||||
// Create a div with provider initial as fallback
|
||||
const target = e.target as HTMLImageElement;
|
||||
const parent = target.parentElement;
|
||||
if (parent) {
|
||||
const fallbackDiv = document.createElement('div');
|
||||
fallbackDiv.className = 'w-5 h-5 rounded-full bg-gray-200 flex items-center justify-center text-xs';
|
||||
fallbackDiv.textContent = providerDisplayName.charAt(0);
|
||||
parent.replaceChild(fallbackDiv, target);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<span>{providerDisplayName}</span>
|
||||
</div>
|
||||
</AntdSelect.Option>
|
||||
))}
|
||||
</AntdSelect>
|
||||
</Form.Item>
|
||||
<LiteLLMModelNameField
|
||||
<div className="flex items-center space-x-2">
|
||||
<img
|
||||
src={providerLogoMap[providerDisplayName]}
|
||||
alt={`${providerEnum} logo`}
|
||||
className="w-5 h-5"
|
||||
onError={(e) => {
|
||||
// Create a div with provider initial as fallback
|
||||
const target = e.target as HTMLImageElement;
|
||||
const parent = target.parentElement;
|
||||
if (parent) {
|
||||
const fallbackDiv = document.createElement('div');
|
||||
fallbackDiv.className = 'w-5 h-5 rounded-full bg-gray-200 flex items-center justify-center text-xs';
|
||||
fallbackDiv.textContent = providerDisplayName.charAt(0);
|
||||
parent.replaceChild(fallbackDiv, target);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<span>{providerDisplayName}</span>
|
||||
</div>
|
||||
</AntdSelect.Option>
|
||||
))}
|
||||
</AntdSelect>
|
||||
</Form.Item>
|
||||
<LiteLLMModelNameField
|
||||
selectedProvider={selectedProvider}
|
||||
providerModels={providerModels}
|
||||
getPlaceholder={getPlaceholder}
|
||||
/>
|
||||
|
||||
{/* Conditionally Render "Public Model Name" */}
|
||||
<ConditionalPublicModelName />
|
||||
|
||||
{/* Select Mode */}
|
||||
<Form.Item
|
||||
label="Mode"
|
||||
name="mode"
|
||||
className="mb-1"
|
||||
>
|
||||
<AntdSelect
|
||||
style={{ width: '100%' }}
|
||||
value={testMode}
|
||||
onChange={(value) => setTestMode(value)}
|
||||
options={TEST_MODES}
|
||||
/>
|
||||
</Form.Item>
|
||||
<Row>
|
||||
<Col span={10}></Col>
|
||||
<Col span={10}>
|
||||
<Text className="mb-5 mt-1">
|
||||
<strong>Optional</strong> - LiteLLM endpoint to use when health checking this model <Link href="https://docs.litellm.ai/docs/proxy/health#health" target="_blank">Learn more</Link>
|
||||
</Text>
|
||||
</Col>
|
||||
</Row>
|
||||
|
||||
{/* Credentials */}
|
||||
<div className="mb-4">
|
||||
<Typography.Text className="text-sm text-gray-500 mb-2">
|
||||
Either select existing credentials OR enter new provider credentials below
|
||||
</Typography.Text>
|
||||
</div>
|
||||
|
||||
<Form.Item
|
||||
label="Existing Credentials"
|
||||
name="litellm_credential_name"
|
||||
>
|
||||
<AntdSelect
|
||||
showSearch
|
||||
placeholder="Select or search for existing credentials"
|
||||
optionFilterProp="children"
|
||||
filterOption={(input, option) =>
|
||||
(option?.label ?? '').toLowerCase().includes(input.toLowerCase())
|
||||
}
|
||||
options={[
|
||||
{ value: null, label: 'None' },
|
||||
...credentials.map((credential) => ({
|
||||
value: credential.credential_name,
|
||||
label: credential.credential_name
|
||||
}))
|
||||
]}
|
||||
allowClear
|
||||
/>
|
||||
</Form.Item>
|
||||
|
||||
<div className="flex items-center my-4">
|
||||
<div className="flex-grow border-t border-gray-200"></div>
|
||||
<span className="px-4 text-gray-500 text-sm">OR</span>
|
||||
<div className="flex-grow border-t border-gray-200"></div>
|
||||
</div>
|
||||
|
||||
<Form.Item
|
||||
noStyle
|
||||
shouldUpdate={(prevValues, currentValues) =>
|
||||
prevValues.litellm_credential_name !== currentValues.litellm_credential_name ||
|
||||
prevValues.provider !== currentValues.provider
|
||||
}
|
||||
>
|
||||
{({ getFieldValue }) => {
|
||||
const credentialName = getFieldValue('litellm_credential_name');
|
||||
console.log("🔑 Credential Name Changed:", credentialName);
|
||||
// Only show provider specific fields if no credentials selected
|
||||
if (!credentialName) {
|
||||
return (
|
||||
<ProviderSpecificFields
|
||||
selectedProvider={selectedProvider}
|
||||
providerModels={providerModels}
|
||||
getPlaceholder={getPlaceholder}
|
||||
uploadProps={uploadProps}
|
||||
/>
|
||||
|
||||
{/* Conditionally Render "Public Model Name" */}
|
||||
<ConditionalPublicModelName />
|
||||
|
||||
{/* Credentials */}
|
||||
<div className="mb-4">
|
||||
<Typography.Text className="text-sm text-gray-500 mb-2">
|
||||
Either select existing credentials OR enter new provider credentials below
|
||||
</Typography.Text>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<div className="text-gray-500 text-sm text-center">
|
||||
Using existing credentials - no additional provider fields needed
|
||||
</div>
|
||||
);
|
||||
}}
|
||||
</Form.Item>
|
||||
<AdvancedSettings
|
||||
showAdvancedSettings={showAdvancedSettings}
|
||||
setShowAdvancedSettings={setShowAdvancedSettings}
|
||||
teams={teams}
|
||||
/>
|
||||
|
||||
<Form.Item
|
||||
label="Existing Credentials"
|
||||
name="litellm_credential_name"
|
||||
>
|
||||
<AntdSelect
|
||||
showSearch
|
||||
placeholder="Select or search for existing credentials"
|
||||
optionFilterProp="children"
|
||||
filterOption={(input, option) =>
|
||||
(option?.label ?? '').toLowerCase().includes(input.toLowerCase())
|
||||
}
|
||||
options={[
|
||||
{ value: null, label: 'None' },
|
||||
...credentials.map((credential) => ({
|
||||
value: credential.credential_name,
|
||||
label: credential.credential_name
|
||||
}))
|
||||
]}
|
||||
allowClear
|
||||
/>
|
||||
</Form.Item>
|
||||
|
||||
<div className="flex items-center my-4">
|
||||
<div className="flex-grow border-t border-gray-200"></div>
|
||||
<span className="px-4 text-gray-500 text-sm">OR</span>
|
||||
<div className="flex-grow border-t border-gray-200"></div>
|
||||
</div>
|
||||
|
||||
<Form.Item
|
||||
noStyle
|
||||
shouldUpdate={(prevValues, currentValues) =>
|
||||
prevValues.litellm_credential_name !== currentValues.litellm_credential_name ||
|
||||
prevValues.provider !== currentValues.provider
|
||||
}
|
||||
>
|
||||
{({ getFieldValue }) => {
|
||||
const credentialName = getFieldValue('litellm_credential_name');
|
||||
console.log("🔑 Credential Name Changed:", credentialName);
|
||||
// Only show provider specific fields if no credentials selected
|
||||
if (!credentialName) {
|
||||
return (
|
||||
<ProviderSpecificFields
|
||||
selectedProvider={selectedProvider}
|
||||
uploadProps={uploadProps}
|
||||
/>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<div className="text-gray-500 text-sm text-center">
|
||||
Using existing credentials - no additional provider fields needed
|
||||
</div>
|
||||
);
|
||||
}}
|
||||
</Form.Item>
|
||||
<AdvancedSettings
|
||||
showAdvancedSettings={showAdvancedSettings}
|
||||
setShowAdvancedSettings={setShowAdvancedSettings}
|
||||
teams={teams}
|
||||
/>
|
||||
|
||||
|
||||
<div className="flex justify-between items-center mb-4">
|
||||
<Tooltip title="Get help on our github">
|
||||
<Typography.Link href="https://github.com/BerriAI/litellm/issues">
|
||||
Need Help?
|
||||
</Typography.Link>
|
||||
</Tooltip>
|
||||
<Button htmlType="submit">Add Model</Button>
|
||||
</div>
|
||||
</>
|
||||
</Form>
|
||||
</Card>
|
||||
|
||||
<div className="flex justify-between items-center mb-4">
|
||||
<Tooltip title="Get help on our github">
|
||||
<Typography.Link href="https://github.com/BerriAI/litellm/issues">
|
||||
Need Help?
|
||||
</Typography.Link>
|
||||
</Tooltip>
|
||||
<div className="space-x-2">
|
||||
<Button onClick={handleTestConnection} loading={isTestingConnection}>Test Connect</Button>
|
||||
<Button htmlType="submit">Add Model</Button>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
</Form>
|
||||
</Card>
|
||||
|
||||
{/* Test Connection Results Modal */}
|
||||
<Modal
|
||||
title="Connection Test Results"
|
||||
open={isResultModalVisible}
|
||||
onCancel={() => {
|
||||
setIsResultModalVisible(false);
|
||||
setIsTestingConnection(false);
|
||||
}}
|
||||
footer={[
|
||||
<Button key="close" onClick={() => {
|
||||
setIsResultModalVisible(false);
|
||||
setIsTestingConnection(false);
|
||||
}}>
|
||||
Close
|
||||
</Button>
|
||||
]}
|
||||
width={700}
|
||||
>
|
||||
{/* Only render the ConnectionErrorDisplay when modal is visible and we have a test ID */}
|
||||
{isResultModalVisible && (
|
||||
<ConnectionErrorDisplay
|
||||
// The key prop tells React to create a fresh component instance when it changes
|
||||
key={connectionTestId}
|
||||
formValues={form.getFieldsValue()}
|
||||
accessToken={accessToken}
|
||||
testMode={testMode}
|
||||
modelName={form.getFieldValue('model_name') || form.getFieldValue('model')}
|
||||
onClose={() => {
|
||||
setIsResultModalVisible(false);
|
||||
setIsTestingConnection(false);
|
||||
}}
|
||||
onTestComplete={() => setIsTestingConnection(false)}
|
||||
/>
|
||||
)}
|
||||
</Modal>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
@ -98,14 +98,6 @@ const ConditionalPublicModelName: React.FC = () => {
|
||||
size="small"
|
||||
/>
|
||||
</Form.Item>
|
||||
<Row>
|
||||
<Col span={10}></Col>
|
||||
<Col span={10}>
|
||||
<Text className="mb-2">
|
||||
Model name your users will pass in.
|
||||
</Text>
|
||||
</Col>
|
||||
</Row>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
@ -1,13 +1,13 @@
|
||||
import { message } from "antd";
|
||||
import { provider_map, Providers } from "../provider_info_helpers";
|
||||
import { modelCreateCall, Model } from "../networking";
|
||||
import { modelCreateCall, Model, testConnectionRequest } from "../networking";
|
||||
import React, { useState } from 'react';
|
||||
import ConnectionErrorDisplay from './model_connection_test';
|
||||
|
||||
|
||||
export const handleAddModelSubmit = async (
|
||||
export const prepareModelAddRequest = async (
|
||||
formValues: Record<string, any>,
|
||||
accessToken: string,
|
||||
form: any,
|
||||
callback?: ()=>void
|
||||
) => {
|
||||
try {
|
||||
console.log("handling submit for formValues:", formValues);
|
||||
@ -72,7 +72,6 @@ export const handleAddModelSubmit = async (
|
||||
} else if (key == "model") {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if key is "base_model"
|
||||
else if (key === "base_model") {
|
||||
// Add key-value pair to model_info dictionary
|
||||
@ -81,6 +80,13 @@ export const handleAddModelSubmit = async (
|
||||
else if (key === "team_id") {
|
||||
modelInfoObj["team_id"] = value;
|
||||
}
|
||||
else if (key == "mode") {
|
||||
console.log("placing mode in modelInfo")
|
||||
modelInfoObj["mode"] = value;
|
||||
|
||||
// remove "mode" from litellmParams
|
||||
delete litellmParamsObj["mode"];
|
||||
}
|
||||
else if (key === "custom_model_name") {
|
||||
litellmParamsObj["model"] = value;
|
||||
} else if (key == "litellm_extra_params") {
|
||||
@ -135,20 +141,43 @@ export const handleAddModelSubmit = async (
|
||||
litellmParamsObj[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
const new_model: Model = {
|
||||
model_name: modelName,
|
||||
litellm_params: litellmParamsObj,
|
||||
model_info: modelInfoObj,
|
||||
};
|
||||
|
||||
const response: any = await modelCreateCall(accessToken, new_model);
|
||||
console.log(`response for model create call: ${response["data"]}`);
|
||||
}
|
||||
|
||||
callback && callback()
|
||||
form.resetFields();
|
||||
return { litellmParamsObj, modelInfoObj, modelName };
|
||||
}
|
||||
} catch (error) {
|
||||
message.error("Failed to create model: " + error, 10);
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
export const handleAddModelSubmit = async (
|
||||
values: any,
|
||||
accessToken: string,
|
||||
form: any,
|
||||
callback?: () => void,
|
||||
) => {
|
||||
try {
|
||||
const result = await prepareModelAddRequest(values, accessToken, form);
|
||||
|
||||
if (!result) {
|
||||
return; // Exit if preparation failed
|
||||
}
|
||||
|
||||
const { litellmParamsObj, modelInfoObj, modelName } = result;
|
||||
|
||||
const new_model: Model = {
|
||||
model_name: modelName,
|
||||
litellm_params: litellmParamsObj,
|
||||
model_info: modelInfoObj,
|
||||
};
|
||||
|
||||
const response: any = await modelCreateCall(accessToken, new_model);
|
||||
console.log(`response for model create call: ${response["data"]}`);
|
||||
|
||||
callback && callback();
|
||||
form.resetFields();
|
||||
} catch (error) {
|
||||
message.error("Failed to add model: " + error, 10);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -0,0 +1,258 @@
|
||||
import React from 'react';
|
||||
import { Typography, Space, Button, Divider, message } from 'antd';
|
||||
import { WarningOutlined, InfoCircleOutlined, CopyOutlined } from '@ant-design/icons';
|
||||
import { testConnectionRequest } from "../networking";
|
||||
import { prepareModelAddRequest } from "./handle_add_model_submit";
|
||||
|
||||
const { Text } = Typography;
|
||||
|
||||
interface ModelConnectionTestProps {
|
||||
formValues: Record<string, any>;
|
||||
accessToken: string;
|
||||
testMode: string;
|
||||
modelName?: string;
|
||||
onClose?: () => void;
|
||||
onTestComplete?: () => void;
|
||||
}
|
||||
|
||||
const ModelConnectionTest: React.FC<ModelConnectionTestProps> = ({
|
||||
formValues,
|
||||
accessToken,
|
||||
testMode,
|
||||
modelName = "this model",
|
||||
onClose,
|
||||
onTestComplete
|
||||
}) => {
|
||||
const [error, setError] = React.useState<Error | string | null>(null);
|
||||
const [rawRequest, setRawRequest] = React.useState<any>(null);
|
||||
const [rawResponse, setRawResponse] = React.useState<any>(null);
|
||||
const [isLoading, setIsLoading] = React.useState<boolean>(true);
|
||||
const [isSuccess, setIsSuccess] = React.useState<boolean>(false);
|
||||
const [showDetails, setShowDetails] = React.useState<boolean>(false);
|
||||
|
||||
const testModelConnection = async () => {
|
||||
setIsLoading(true);
|
||||
setShowDetails(false);
|
||||
setError(null);
|
||||
setRawRequest(null);
|
||||
setRawResponse(null);
|
||||
setIsSuccess(false);
|
||||
|
||||
// Add a small delay to ensure form values are fully populated
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
try {
|
||||
console.log("Testing connection with form values:", formValues);
|
||||
const result = await prepareModelAddRequest(formValues, accessToken, null);
|
||||
|
||||
if (!result) {
|
||||
console.log("No result from prepareModelAddRequest");
|
||||
setError("Failed to prepare model data. Please check your form inputs.");
|
||||
setIsSuccess(false);
|
||||
setIsLoading(false);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log("Result from prepareModelAddRequest:", result);
|
||||
|
||||
const { litellmParamsObj, modelInfoObj, modelName: returnedModelName } = result;
|
||||
|
||||
const response = await testConnectionRequest(accessToken, litellmParamsObj, modelInfoObj?.mode);
|
||||
if (response.status === "success") {
|
||||
message.success("Connection test successful!");
|
||||
setError(null);
|
||||
setIsSuccess(true);
|
||||
} else {
|
||||
const errorMessage = response.result?.error || response.message || "Unknown error";
|
||||
setError(errorMessage);
|
||||
setRawRequest(litellmParamsObj);
|
||||
setRawResponse(response.result?.raw_request_typed_dict);
|
||||
setIsSuccess(false);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Test connection error:", error);
|
||||
setError(error instanceof Error ? error.message : String(error));
|
||||
setIsSuccess(false);
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
if (onTestComplete) onTestComplete();
|
||||
}
|
||||
};
|
||||
|
||||
React.useEffect(() => {
|
||||
// Run the test once when component mounts
|
||||
// Add a small timeout to ensure form values are ready
|
||||
const timer = setTimeout(() => {
|
||||
testModelConnection();
|
||||
}, 200);
|
||||
|
||||
return () => clearTimeout(timer);
|
||||
}, []); // Empty dependency array means this runs once on mount
|
||||
|
||||
const getCleanErrorMessage = (errorMsg: string) => {
|
||||
if (!errorMsg) return "Unknown error";
|
||||
|
||||
const mainError = errorMsg.split('stack trace:')[0].trim();
|
||||
|
||||
const cleanedError = mainError.replace(/^litellm\.(.*?)Error: /, '');
|
||||
|
||||
return cleanedError;
|
||||
};
|
||||
|
||||
const errorMessage = typeof error === 'string'
|
||||
? getCleanErrorMessage(error)
|
||||
: error?.message ? getCleanErrorMessage(error.message) : "Unknown error";
|
||||
|
||||
const formatCurlCommand = (apiBase: string, requestBody: Record<string, any>, requestHeaders: Record<string, string>) => {
|
||||
const formattedBody = JSON.stringify(requestBody, null, 2)
|
||||
.split('\n')
|
||||
.map(line => ` ${line}`)
|
||||
.join('\n');
|
||||
|
||||
const headerString = Object.entries(requestHeaders)
|
||||
.map(([key, value]) => `-H '${key}: ${value}'`)
|
||||
.join(' \\\n ');
|
||||
|
||||
return `curl -X POST \\
|
||||
${apiBase} \\
|
||||
${headerString ? `${headerString} \\\n ` : ''}-H 'Content-Type: application/json' \\
|
||||
-d '{
|
||||
${formattedBody}
|
||||
}'`;
|
||||
};
|
||||
|
||||
const curlCommand = rawResponse ? formatCurlCommand(
|
||||
rawResponse.raw_request_api_base,
|
||||
rawResponse.raw_request_body,
|
||||
rawResponse.raw_request_headers || {}
|
||||
) : '';
|
||||
|
||||
return (
|
||||
<div style={{ padding: '24px', borderRadius: '8px', backgroundColor: '#fff' }}>
|
||||
{isLoading ? (
|
||||
<div style={{ textAlign: 'center', padding: '32px 20px' }}>
|
||||
<div className="loading-spinner" style={{ marginBottom: '16px' }}>
|
||||
{/* Simple CSS spinner */}
|
||||
<div style={{
|
||||
border: '3px solid #f3f3f3',
|
||||
borderTop: '3px solid #1890ff',
|
||||
borderRadius: '50%',
|
||||
width: '30px',
|
||||
height: '30px',
|
||||
animation: 'spin 1s linear infinite',
|
||||
margin: '0 auto'
|
||||
}} />
|
||||
</div>
|
||||
<Text style={{ fontSize: '16px' }}>Testing connection to {modelName}...</Text>
|
||||
<style jsx>{`
|
||||
@keyframes spin {
|
||||
0% { transform: rotate(0deg); }
|
||||
100% { transform: rotate(360deg); }
|
||||
}
|
||||
`}</style>
|
||||
</div>
|
||||
) : isSuccess ? (
|
||||
<div style={{ textAlign: 'center', padding: '32px 20px' }}>
|
||||
<div style={{ color: '#52c41a', fontSize: '32px', marginBottom: '16px' }}>
|
||||
<svg viewBox="64 64 896 896" focusable="false" data-icon="check-circle" width="1em" height="1em" fill="currentColor" aria-hidden="true">
|
||||
<path d="M512 64C264.6 64 64 264.6 64 512s200.6 448 448 448 448-200.6 448-448S759.4 64 512 64zm193.5 301.7l-210.6 292a31.8 31.8 0 01-51.7 0L318.5 484.9c-3.8-5.3 0-12.7 6.5-12.7h46.9c10.2 0 19.9 4.9 25.9 13.3l71.2 98.8 157.2-218c6-8.3 15.6-13.3 25.9-13.3H699c6.5 0 10.3 7.4 6.5 12.7z"></path>
|
||||
</svg>
|
||||
</div>
|
||||
<Text type="success" style={{ fontSize: '18px', fontWeight: 500 }}>
|
||||
Connection to {modelName} successful!
|
||||
</Text>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<div>
|
||||
<div style={{ display: 'flex', alignItems: 'center', marginBottom: '20px' }}>
|
||||
<WarningOutlined style={{ color: '#ff4d4f', fontSize: '24px', marginRight: '12px' }} />
|
||||
<Text type="danger" style={{ fontSize: '18px', fontWeight: 500 }}>Connection to {modelName} failed</Text>
|
||||
</div>
|
||||
|
||||
<div style={{
|
||||
backgroundColor: '#fff2f0',
|
||||
border: '1px solid #ffccc7',
|
||||
borderRadius: '8px',
|
||||
padding: '16px',
|
||||
marginBottom: '20px',
|
||||
boxShadow: '0 1px 2px rgba(0, 0, 0, 0.03)'
|
||||
}}>
|
||||
<Text strong style={{ display: 'block', marginBottom: '8px' }}>Error: </Text>
|
||||
<Text type="danger" style={{ fontSize: '14px', lineHeight: '1.5' }}>{errorMessage}</Text>
|
||||
|
||||
{error && (
|
||||
<div style={{ marginTop: '12px' }}>
|
||||
<Button
|
||||
type="link"
|
||||
onClick={() => setShowDetails(!showDetails)}
|
||||
style={{ paddingLeft: 0, height: 'auto' }}
|
||||
>
|
||||
{showDetails ? 'Hide Details' : 'Show Details'}
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{showDetails && (
|
||||
<div style={{ marginBottom: '20px' }}>
|
||||
<Text strong style={{ display: 'block', marginBottom: '8px', fontSize: '15px' }}>Troubleshooting Details</Text>
|
||||
<pre style={{
|
||||
backgroundColor: '#f5f5f5',
|
||||
padding: '16px',
|
||||
borderRadius: '8px',
|
||||
fontSize: '13px',
|
||||
maxHeight: '200px',
|
||||
overflow: 'auto',
|
||||
border: '1px solid #e8e8e8',
|
||||
lineHeight: '1.5'
|
||||
}}>
|
||||
{typeof error === 'string' ? error : JSON.stringify(error, null, 2)}
|
||||
</pre>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div>
|
||||
<Text strong style={{ display: 'block', marginBottom: '8px', fontSize: '15px' }}>API Request</Text>
|
||||
<pre style={{
|
||||
backgroundColor: '#f5f5f5',
|
||||
padding: '16px',
|
||||
borderRadius: '8px',
|
||||
fontSize: '13px',
|
||||
maxHeight: '250px',
|
||||
overflow: 'auto',
|
||||
border: '1px solid #e8e8e8',
|
||||
lineHeight: '1.5'
|
||||
}}>
|
||||
{curlCommand || "No request data available"}
|
||||
</pre>
|
||||
<Button
|
||||
style={{ marginTop: '8px' }}
|
||||
icon={<CopyOutlined />}
|
||||
onClick={() => {
|
||||
navigator.clipboard.writeText(curlCommand || '');
|
||||
message.success('Copied to clipboard');
|
||||
}}
|
||||
>
|
||||
Copy to Clipboard
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
<Divider style={{ margin: '24px 0 16px' }} />
|
||||
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
|
||||
<Button
|
||||
type="link"
|
||||
href="https://docs.litellm.ai/docs/providers"
|
||||
target="_blank"
|
||||
icon={<InfoCircleOutlined />}
|
||||
>
|
||||
View Documentation
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default ModelConnectionTest;
|
||||
@ -1140,6 +1140,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
|
||||
setShowAdvancedSettings={setShowAdvancedSettings}
|
||||
teams={teams}
|
||||
credentials={credentialsList}
|
||||
accessToken={accessToken}
|
||||
/>
|
||||
</TabPanel>
|
||||
<TabPanel>
|
||||
|
||||
@ -2313,6 +2313,63 @@ export const keyInfoCall = async (accessToken: String, keys: String[]) => {
|
||||
};
|
||||
|
||||
|
||||
export const testConnectionRequest = async (
|
||||
accessToken: string,
|
||||
litellm_params: Record<string, any>,
|
||||
mode: string,
|
||||
) => {
|
||||
try {
|
||||
console.log("Sending model connection test request:", JSON.stringify(litellm_params));
|
||||
|
||||
// Construct the URL based on environment
|
||||
const url = proxyBaseUrl ? `${proxyBaseUrl}/health/test_connection` : `/health/test_connection`;
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
[globalLitellmHeaderName]: `Bearer ${accessToken}`
|
||||
},
|
||||
body: JSON.stringify(
|
||||
{
|
||||
litellm_params: litellm_params,
|
||||
mode: mode,
|
||||
}
|
||||
)
|
||||
});
|
||||
|
||||
// Check for non-JSON responses first
|
||||
const contentType = response.headers.get('content-type');
|
||||
if (!contentType || !contentType.includes('application/json')) {
|
||||
const text = await response.text();
|
||||
console.error("Received non-JSON response:", text);
|
||||
throw new Error(`Received non-JSON response (${response.status}: ${response.statusText}). Check network tab for details.`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (!response.ok || data.status === "error") {
|
||||
// Return the error response instead of throwing an error
|
||||
// This allows the caller to handle the error format properly
|
||||
if (data.status === "error") {
|
||||
return data; // Return the full error response
|
||||
} else {
|
||||
return {
|
||||
status: "error",
|
||||
message: data.error?.message || `Connection test failed: ${response.status} ${response.statusText}`
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
} catch (error) {
|
||||
console.error("Model connection test error:", error);
|
||||
// For network errors or other exceptions, still throw
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
// ... existing code ...
|
||||
export const keyInfoV1Call = async (accessToken: string, key: string) => {
|
||||
try {
|
||||
let url = proxyBaseUrl ? `${proxyBaseUrl}/key/info` : `/key/info`;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user