diff --git a/docs/my-website/docs/proxy/request_headers.md b/docs/my-website/docs/proxy/request_headers.md index 7251262d3b..246d917d00 100644 --- a/docs/my-website/docs/proxy/request_headers.md +++ b/docs/my-website/docs/proxy/request_headers.md @@ -10,6 +10,8 @@ Special headers that are supported by LiteLLM. `x-litellm-tags`: Optional[str]: A comma separated list (e.g. `tag1,tag2,tag3`) of tags to use for [tag-based routing](./tag_routing) **OR** [spend-tracking](./enterprise.md#tracking-spend-for-custom-tags). +`x-litellm-num-retries`: Optional[int]: The number of retries for the request. + ## Anthropic Headers `anthropic-version` Optional[str]: The version of the Anthropic API to use. diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index b5c63026f6..3fbf887475 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2780,6 +2780,7 @@ class LitellmDataForBackendLLMCall(TypedDict, total=False): organization: str timeout: Optional[float] user: Optional[str] + num_retries: Optional[int] class JWTKeyItem(TypedDict, total=False): diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 256b51efd1..aaeb86e34c 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -272,6 +272,16 @@ class LiteLLMProxyRequestSetup: return float(timeout_header) return None + @staticmethod + def _get_num_retries_from_request(headers: dict) -> Optional[int]: + """ + Workaround for client request from Vercel's AI SDK. + """ + num_retries_header = headers.get("x-litellm-num-retries", None) + if num_retries_header is not None: + return int(num_retries_header) + return None + @staticmethod def _get_forwardable_headers( headers: Union[Headers, dict], @@ -407,6 +417,10 @@ class LiteLLMProxyRequestSetup: if timeout is not None: data["timeout"] = timeout + num_retries = LiteLLMProxyRequestSetup._get_num_retries_from_request(headers) + if num_retries is not None: + data["num_retries"] = num_retries + return data @staticmethod @@ -801,7 +815,10 @@ async def add_litellm_data_to_request( # noqa: PLR0915 data[k] = v # Add disabled callbacks from key metadata - if user_api_key_dict.metadata and "litellm_disabled_callbacks" in user_api_key_dict.metadata: + if ( + user_api_key_dict.metadata + and "litellm_disabled_callbacks" in user_api_key_dict.metadata + ): disabled_callbacks = user_api_key_dict.metadata["litellm_disabled_callbacks"] if disabled_callbacks and isinstance(disabled_callbacks, list): data["litellm_disabled_callbacks"] = disabled_callbacks diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 754169a819..bdad6cd648 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -80,6 +80,44 @@ def update_breakdown_metrics( ) ) + # Update model group breakdown + if record.model_group and record.model_group not in breakdown.model_groups: + breakdown.model_groups[record.model_group] = MetricWithMetadata( + metrics=SpendMetrics(), + metadata=model_metadata.get(record.model_group, {}), + ) + if record.model_group: + breakdown.model_groups[record.model_group].metrics = update_metrics( + breakdown.model_groups[record.model_group].metrics, record + ) + + # Update API key breakdown for this model + if ( + record.api_key + not in breakdown.model_groups[record.model_group].api_key_breakdown + ): + breakdown.model_groups[record.model_group].api_key_breakdown[ + record.api_key + ] = KeyMetricWithMetadata( + metrics=SpendMetrics(), + metadata=KeyMetadata( + key_alias=api_key_metadata.get(record.api_key, {}).get( + "key_alias", None + ), + team_id=api_key_metadata.get(record.api_key, {}).get( + "team_id", None + ), + ), + ) + breakdown.model_groups[record.model_group].api_key_breakdown[ + record.api_key + ].metrics = update_metrics( + breakdown.model_groups[record.model_group] + .api_key_breakdown[record.api_key] + .metrics, + record, + ) + if record.mcp_namespaced_tool_name: if record.mcp_namespaced_tool_name not in breakdown.mcp_servers: breakdown.mcp_servers[record.mcp_namespaced_tool_name] = MetricWithMetadata( @@ -295,22 +333,6 @@ async def get_daily_activity( take=page_size, ) - # # for 50% of the records, set the mcp_server_id to a random value - # mcp_server_dict = {"Zapier_Gmail_MCP", "Stripe_MCP"} - # import random - - # for idx, record in enumerate(daily_spend_data): - # record = LiteLLM_DailyUserSpend(**record.model_dump()) - # if random.random() < 0.5: - # record.mcp_server_id = random.choice(list(mcp_server_dict)) - # record.model = None - # record.model_group = None - # record.prompt_tokens = 0 - # record.completion_tokens = 0 - # record.cache_read_input_tokens = 0 - # record.cache_creation_input_tokens = 0 - # daily_spend_data[idx] = record - # Get all unique API keys from the spend data api_keys = set() for record in daily_spend_data: diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index fc87f63fdf..713ca56758 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -61,6 +61,9 @@ class BreakdownMetrics(BaseModel): models: Dict[str, MetricWithMetadata] = Field( default_factory=dict ) # model -> {metrics, metadata} + model_groups: Dict[str, MetricWithMetadata] = Field( + default_factory=dict + ) # model_group -> {metrics, metadata} providers: Dict[str, MetricWithMetadata] = Field( default_factory=dict ) # provider -> {metrics, metadata} diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index e68767204f..0ec3fd9393 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -243,15 +243,13 @@ async def test_add_litellm_data_to_request_disabled_callbacks(): api_key="test_api_key", user_id="test_user_id", org_id="test_org_id", - metadata={ - "litellm_disabled_callbacks": ["langfuse", "langsmith", "datadog"] - } + metadata={"litellm_disabled_callbacks": ["langfuse", "langsmith", "datadog"]}, ) # Setup request data data = { "model": "gpt-3.5-turbo", - "messages": [{"role": "user", "content": "Hello"}] + "messages": [{"role": "user", "content": "Hello"}], } # Setup proxy config @@ -262,7 +260,7 @@ async def test_add_litellm_data_to_request_disabled_callbacks(): data=data, request=request_mock, user_api_key_dict=user_api_key_dict, - proxy_config=proxy_config + proxy_config=proxy_config, ) # Verify that litellm_disabled_callbacks was added to the request data @@ -298,15 +296,13 @@ async def test_add_litellm_data_to_request_disabled_callbacks_empty(): api_key="test_api_key", user_id="test_user_id", org_id="test_org_id", - metadata={ - "litellm_disabled_callbacks": [] - } + metadata={"litellm_disabled_callbacks": []}, ) # Setup request data data = { "model": "gpt-3.5-turbo", - "messages": [{"role": "user", "content": "Hello"}] + "messages": [{"role": "user", "content": "Hello"}], } # Setup proxy config @@ -317,7 +313,7 @@ async def test_add_litellm_data_to_request_disabled_callbacks_empty(): data=data, request=request_mock, user_api_key_dict=user_api_key_dict, - proxy_config=proxy_config + proxy_config=proxy_config, ) # Verify that litellm_disabled_callbacks is not added when empty @@ -352,13 +348,13 @@ async def test_add_litellm_data_to_request_disabled_callbacks_not_present(): api_key="test_api_key", user_id="test_user_id", org_id="test_org_id", - metadata={} # No litellm_disabled_callbacks + metadata={}, # No litellm_disabled_callbacks ) # Setup request data data = { "model": "gpt-3.5-turbo", - "messages": [{"role": "user", "content": "Hello"}] + "messages": [{"role": "user", "content": "Hello"}], } # Setup proxy config @@ -369,7 +365,7 @@ async def test_add_litellm_data_to_request_disabled_callbacks_not_present(): data=data, request=request_mock, user_api_key_dict=user_api_key_dict, - proxy_config=proxy_config + proxy_config=proxy_config, ) # Verify that litellm_disabled_callbacks is not added when not present @@ -404,15 +400,13 @@ async def test_add_litellm_data_to_request_disabled_callbacks_invalid_type(): api_key="test_api_key", user_id="test_user_id", org_id="test_org_id", - metadata={ - "litellm_disabled_callbacks": "not_a_list" # Should be a list - } + metadata={"litellm_disabled_callbacks": "not_a_list"}, # Should be a list ) # Setup request data data = { "model": "gpt-3.5-turbo", - "messages": [{"role": "user", "content": "Hello"}] + "messages": [{"role": "user", "content": "Hello"}], } # Setup proxy config @@ -423,7 +417,7 @@ async def test_add_litellm_data_to_request_disabled_callbacks_invalid_type(): data=data, request=request_mock, user_api_key_dict=user_api_key_dict, - proxy_config=proxy_config + proxy_config=proxy_config, ) # Verify that litellm_disabled_callbacks is not added when invalid type @@ -460,16 +454,20 @@ async def test_add_litellm_data_to_request_disabled_callbacks_with_logging_setti org_id="test_org_id", metadata={ "logging": [ - {"callback_name": "langfuse", "callback_type": "success", "callback_vars": {}} + { + "callback_name": "langfuse", + "callback_type": "success", + "callback_vars": {}, + } ], - "litellm_disabled_callbacks": ["langsmith", "datadog"] - } + "litellm_disabled_callbacks": ["langsmith", "datadog"], + }, ) # Setup request data data = { "model": "gpt-3.5-turbo", - "messages": [{"role": "user", "content": "Hello"}] + "messages": [{"role": "user", "content": "Hello"}], } # Setup proxy config @@ -480,7 +478,7 @@ async def test_add_litellm_data_to_request_disabled_callbacks_with_logging_setti data=data, request=request_mock, user_api_key_dict=user_api_key_dict, - proxy_config=proxy_config + proxy_config=proxy_config, ) # Verify that both logging settings and disabled callbacks are handled correctly @@ -500,12 +498,8 @@ def test_key_dynamic_logging_settings(): # Test with arize logging key_with_arize = UserAPIKeyAuth( api_key="test-key", - metadata={ - "logging": [ - {"callback_name": "arize", "callback_type": "success"} - ] - }, - team_metadata={} + metadata={"logging": [{"callback_name": "arize", "callback_type": "success"}]}, + team_metadata={}, ) result = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings(key_with_arize) assert result == [{"callback_name": "arize", "callback_type": "success"}] @@ -514,22 +508,22 @@ def test_key_dynamic_logging_settings(): key_with_langfuse = UserAPIKeyAuth( api_key="test-key", metadata={ - "logging": [ - {"callback_name": "langfuse", "callback_type": "success"} - ] + "logging": [{"callback_name": "langfuse", "callback_type": "success"}] }, - team_metadata={} + team_metadata={}, + ) + result = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings( + key_with_langfuse ) - result = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings(key_with_langfuse) assert result == [{"callback_name": "langfuse", "callback_type": "success"}] # Test with no logging metadata key_without_logging = UserAPIKeyAuth( - api_key="test-key", - metadata={}, - team_metadata={} + api_key="test-key", metadata={}, team_metadata={} + ) + result = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings( + key_without_logging ) - result = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings(key_without_logging) assert result is None @@ -542,12 +536,12 @@ def test_team_dynamic_logging_settings(): api_key="test-key", metadata={}, team_metadata={ - "logging": [ - {"callback_name": "arize", "callback_type": "failure"} - ] - } + "logging": [{"callback_name": "arize", "callback_type": "failure"}] + }, + ) + result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings( + key_with_team_arize ) - result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(key_with_team_arize) assert result == [{"callback_name": "arize", "callback_type": "failure"}] # Test with langfuse team logging @@ -555,21 +549,21 @@ def test_team_dynamic_logging_settings(): api_key="test-key", metadata={}, team_metadata={ - "logging": [ - {"callback_name": "langfuse", "callback_type": "success"} - ] - } + "logging": [{"callback_name": "langfuse", "callback_type": "success"}] + }, + ) + result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings( + key_with_team_langfuse ) - result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(key_with_team_langfuse) assert result == [{"callback_name": "langfuse", "callback_type": "success"}] # Test with no team logging metadata key_without_team_logging = UserAPIKeyAuth( - api_key="test-key", - metadata={}, - team_metadata={} + api_key="test-key", metadata={}, team_metadata={} + ) + result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings( + key_without_team_logging ) - result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(key_without_team_logging) assert result is None @@ -588,22 +582,21 @@ def test_get_dynamic_logging_metadata_with_arize_team_logging(): "callback_type": "success", "callback_vars": { "arize_api_key": "test_arize_api_key", - "arize_space_id": "test_arize_space_id" - } + "arize_space_id": "test_arize_space_id", + }, } ] - } + }, ) - + # Mock proxy_config (not used in this test path since we have team dynamic logging) mock_proxy_config = MagicMock() - + # Call the function result = _get_dynamic_logging_metadata( - user_api_key_dict=user_api_key_dict, - proxy_config=mock_proxy_config + user_api_key_dict=user_api_key_dict, proxy_config=mock_proxy_config ) - + # Verify the result assert result is not None assert isinstance(result, TeamCallbackMetadata) @@ -613,6 +606,68 @@ def test_get_dynamic_logging_metadata_with_arize_team_logging(): assert result.callback_vars["arize_space_id"] == "test_arize_space_id" + +def test_get_num_retries_from_request(): + """ + Test LiteLLMProxyRequestSetup._get_num_retries_from_request method + """ + # Test case 1: Header is present with valid integer string + headers_with_retries = {"x-litellm-num-retries": "3"} + result = LiteLLMProxyRequestSetup._get_num_retries_from_request( + headers_with_retries + ) + assert result == 3 + + # Test case 2: Header is not present + headers_without_retries = {"Content-Type": "application/json"} + result = LiteLLMProxyRequestSetup._get_num_retries_from_request( + headers_without_retries + ) + assert result is None + + # Test case 3: Empty headers dictionary + empty_headers = {} + result = LiteLLMProxyRequestSetup._get_num_retries_from_request(empty_headers) + assert result is None + + # Test case 4: Header present with zero value + headers_with_zero = {"x-litellm-num-retries": "0"} + result = LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_with_zero) + assert result == 0 + + # Test case 5: Header present with large number + headers_with_large_number = {"x-litellm-num-retries": "100"} + result = LiteLLMProxyRequestSetup._get_num_retries_from_request( + headers_with_large_number + ) + assert result == 100 + + # Test case 6: Multiple headers with num retries header + headers_multiple = { + "Content-Type": "application/json", + "x-litellm-num-retries": "5", + "Authorization": "Bearer token", + } + result = LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_multiple) + assert result == 5 + + # Test case 7: Header present with invalid value (should raise ValueError when int() is called) + headers_with_invalid = {"x-litellm-num-retries": "invalid"} + with pytest.raises(ValueError): + LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_with_invalid) + + # Test case 8: Header present with float string (should raise ValueError when int() is called) + headers_with_float = {"x-litellm-num-retries": "3.5"} + with pytest.raises(ValueError): + LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_with_float) + + # Test case 9: Header present with negative number + headers_with_negative = {"x-litellm-num-retries": "-1"} + result = LiteLLMProxyRequestSetup._get_num_retries_from_request( + headers_with_negative + ) + assert result == -1 + def test_add_user_api_key_auth_to_request_metadata(): """ Test that add_user_api_key_auth_to_request_metadata properly adds user API key authentication data to request metadata @@ -668,4 +723,4 @@ def test_add_user_api_key_auth_to_request_metadata(): # Verify original data is preserved assert result["model"] == "gpt-3.5-turbo" - assert result["messages"] == [{"role": "user", "content": "Hello"}] + assert result["messages"] == [{"role": "user", "content": "Hello"}] \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/new_usage.tsx b/ui/litellm-dashboard/src/components/new_usage.tsx index 7a7d437ab1..34b3523499 100644 --- a/ui/litellm-dashboard/src/components/new_usage.tsx +++ b/ui/litellm-dashboard/src/components/new_usage.tsx @@ -86,6 +86,7 @@ const NewUsagePage: React.FC = ({ }); const [allTags, setAllTags] = useState([]); + const [modelViewType, setModelViewType] = useState<'groups' | 'individual'>('groups'); const getAllTags = async () => { if (!accessToken) { @@ -160,6 +161,58 @@ const NewUsagePage: React.FC = ({ .slice(0, 5); }; + const getTopModelGroups = () => { + const modelGroupSpend: { [key: string]: MetricWithMetadata } = {}; + userSpendData.results.forEach((day) => { + Object.entries(day.breakdown.model_groups || {}).forEach(([modelGroup, metrics]) => { + if (!modelGroupSpend[modelGroup]) { + modelGroupSpend[modelGroup] = { + metrics: { + spend: 0, + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + api_requests: 0, + successful_requests: 0, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + metadata: {}, + api_key_breakdown: {} + }; + } + modelGroupSpend[modelGroup].metrics.spend += metrics.metrics.spend; + modelGroupSpend[modelGroup].metrics.prompt_tokens += + metrics.metrics.prompt_tokens; + modelGroupSpend[modelGroup].metrics.completion_tokens += + metrics.metrics.completion_tokens; + modelGroupSpend[modelGroup].metrics.total_tokens += metrics.metrics.total_tokens; + modelGroupSpend[modelGroup].metrics.api_requests += metrics.metrics.api_requests; + modelGroupSpend[modelGroup].metrics.successful_requests += + metrics.metrics.successful_requests || 0; + modelGroupSpend[modelGroup].metrics.failed_requests += + metrics.metrics.failed_requests || 0; + modelGroupSpend[modelGroup].metrics.cache_read_input_tokens += + metrics.metrics.cache_read_input_tokens || 0; + modelGroupSpend[modelGroup].metrics.cache_creation_input_tokens += + metrics.metrics.cache_creation_input_tokens || 0; + }); + }); + + return Object.entries(modelGroupSpend) + .map(([modelGroup, metrics]) => ({ + key: modelGroup, + spend: metrics.metrics.spend, + requests: metrics.metrics.api_requests, + successful_requests: metrics.metrics.successful_requests, + failed_requests: metrics.metrics.failed_requests, + tokens: metrics.metrics.total_tokens, + })) + .sort((a, b) => b.spend - a.spend) + .slice(0, 5); + }; + // Calculate provider spend from the breakdown data const getProviderSpend = () => { const providerSpend: { [key: string]: MetricWithMetadata } = {}; @@ -527,11 +580,35 @@ const NewUsagePage: React.FC = ({
- Top Models + + {modelViewType === 'groups' ? 'Top Public Model Names' : 'Top Litellm Models'} + +
+ + +