ai-workspace-services/litellm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #24088 from Point72/ephrimstanley/limits

Browse Source 
feat: add proxy-wide default api key tpm/rpm limits
This commit is contained in:
Sameer Kankute 2026-03-20 14:46:38 +05:30 committed by GitHub
commit 009cbfa799
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 500 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
litellm/proxy/auth/auth_utils.py
View File

@ -539,8 +539,44 @@ def bytes_to_mb(bytes_value: int):
# helpers used by parallel request limiter to handle model rpm/tpm limits for a given api key
def _get_deployment_default_limit(model_name: str, field: str) -> Optional[int]:
"""
Return the minimum value of `field` across all deployments for model_name,
or None if no deployment has the field set.
When multiple deployments share the same model name, taking the minimum is
the safest choice for load-balanced setups: it ensures no deployment is
over-consumed regardless of which one actually serves a given request.
"""
from litellm.proxy.proxy_server import llm_router
if llm_router is None:
return None
deployments = llm_router.get_model_list(model_name=model_name)
if not deployments:
return None
limits = []
for deployment in deployments:
raw = deployment.get("litellm_params", {}).get(field)
if raw is not None:
try:
limits.append(int(raw))
except (ValueError, TypeError):
pass
return min(limits) if limits else None
def _get_deployment_default_rpm_limit(model_name: str) -> Optional[int]:
return _get_deployment_default_limit(model_name, "default_api_key_rpm_limit")
def _get_deployment_default_tpm_limit(model_name: str) -> Optional[int]:
return _get_deployment_default_limit(model_name, "default_api_key_tpm_limit")
def get_key_model_rpm_limit(
user_api_key_dict: UserAPIKeyAuth,
model_name: Optional[str] = None,
) -> Optional[Dict[str, int]]:
"""
Get the model rpm limit for a given api key.
@ -549,6 +585,7 @@ def get_key_model_rpm_limit(
1. Key metadata (model_rpm_limit)
2. Key model_max_budget (rpm_limit per model)
3. Team metadata (model_rpm_limit)
4. Deployment default_api_key_rpm_limit (when model_name is provided)
"""
# 1. Check key metadata first (takes priority)
if user_api_key_dict.metadata:
@ -567,13 +604,22 @@ def get_key_model_rpm_limit(
# 3. Fallback to team metadata
if user_api_key_dict.team_metadata:
return user_api_key_dict.team_metadata.get("model_rpm_limit")
team_limit = user_api_key_dict.team_metadata.get("model_rpm_limit")
if team_limit is not None:
return team_limit
# 4. Fallback to deployment default_api_key_rpm_limit
if model_name is not None:
default_limit = _get_deployment_default_rpm_limit(model_name)
if default_limit is not None:
return {model_name: default_limit}
return None
def get_key_model_tpm_limit(
user_api_key_dict: UserAPIKeyAuth,
model_name: Optional[str] = None,
) -> Optional[Dict[str, int]]:
"""
Get the model tpm limit for a given api key.
@ -582,6 +628,7 @@ def get_key_model_tpm_limit(
1. Key metadata (model_tpm_limit)
2. Key model_max_budget (tpm_limit per model)
3. Team metadata (model_tpm_limit)
4. Deployment default_api_key_tpm_limit (when model_name is provided)
"""
# 1. Check key metadata first (takes priority)
if user_api_key_dict.metadata:
@ -600,7 +647,15 @@ def get_key_model_tpm_limit(
# 3. Fallback to team metadata
if user_api_key_dict.team_metadata:
return user_api_key_dict.team_metadata.get("model_tpm_limit")
team_limit = user_api_key_dict.team_metadata.get("model_tpm_limit")
if team_limit is not None:
return team_limit
# 4. Fallback to deployment default_api_key_tpm_limit
if model_name is not None:
default_limit = _get_deployment_default_tpm_limit(model_name)
if default_limit is not None:
return {model_name: default_limit}
return None

11
litellm/proxy/common_utils/openai_endpoint_utils.py
View File

@ -32,8 +32,17 @@ def remove_sensitive_info_from_deployment(
deployment_dict["litellm_params"].pop("aws_access_key_id", None)
deployment_dict["litellm_params"].pop("aws_secret_access_key", None)
# Rate-limit config fields must never be masked — they are integers, not credentials.
# The field names contain "key" which matches the masker's sensitive pattern, so we
# explicitly exclude them here rather than widening the global non_sensitive_overrides.
_rate_limit_config_keys = {
"default_api_key_tpm_limit",
"default_api_key_rpm_limit",
}
_excluded = (excluded_keys or set()) | _rate_limit_config_keys
deployment_dict["litellm_params"] = SENSITIVE_DATA_MASKER.mask_dict(
deployment_dict["litellm_params"], excluded_keys=excluded_keys
deployment_dict["litellm_params"], excluded_keys=_excluded
)
return deployment_dict

27
litellm/proxy/hooks/parallel_request_limiter.py
View File

@ -295,16 +295,17 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
)
# Check if request under RPM/TPM per model for a given API Key
if (
get_key_model_tpm_limit(user_api_key_dict) is not None
or get_key_model_rpm_limit(user_api_key_dict) is not None
):
_model = data.get("model", None)
_model = data.get("model", None)
_tpm_limit_for_key_model = get_key_model_tpm_limit(
user_api_key_dict, model_name=_model
)
_rpm_limit_for_key_model = get_key_model_rpm_limit(
user_api_key_dict, model_name=_model
)
if _tpm_limit_for_key_model is not None or _rpm_limit_for_key_model is not None:
request_count_api_key = (
f"{api_key}::{_model}::{precise_minute}::request_count"
)
_tpm_limit_for_key_model = get_key_model_tpm_limit(user_api_key_dict)
_rpm_limit_for_key_model = get_key_model_rpm_limit(user_api_key_dict)
tpm_limit_for_model = None
rpm_limit_for_model = None
@ -538,6 +539,16 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
# Update usage - model group + API Key
# ------------
model_group = get_model_group_from_litellm_kwargs(kwargs)
_success_tpm_limit = (
get_key_model_tpm_limit(user_api_key_dict, model_name=model_group)
if model_group is not None
else None
)
_success_rpm_limit = (
get_key_model_rpm_limit(user_api_key_dict, model_name=model_group)
if model_group is not None
else None
)
if (
user_api_key is not None
and model_group is not None
@ -545,6 +556,8 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
"model_rpm_limit" in user_api_key_metadata
or "model_tpm_limit" in user_api_key_metadata
or user_api_key_model_max_budget is not None
or _success_tpm_limit is not None
or _success_rpm_limit is not None
)
):
request_count_api_key = (

8
litellm/proxy/hooks/parallel_request_limiter_v3.py
View File

@ -687,8 +687,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
if not requested_model:
return
_tpm_limit_for_key_model = get_key_model_tpm_limit(user_api_key_dict)
_rpm_limit_for_key_model = get_key_model_rpm_limit(user_api_key_dict)
_tpm_limit_for_key_model = get_key_model_tpm_limit(
user_api_key_dict, model_name=requested_model
)
_rpm_limit_for_key_model = get_key_model_rpm_limit(
user_api_key_dict, model_name=requested_model
)
if _tpm_limit_for_key_model is None and _rpm_limit_for_key_model is None:
return

5
litellm/types/router.py
View File

@ -188,6 +188,11 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
max_file_size_mb: Optional[float] = None
# Proxy-wide default rate limits applied to any API key using this deployment
# when the key does not have a model-specific tpm/rpm limit configured.
default_api_key_tpm_limit: Optional[int] = None
default_api_key_rpm_limit: Optional[int] = None
# Deployment budgets
max_budget: Optional[float] = None
budget_duration: Optional[str] = None

236
tests/test_litellm/proxy/auth/test_auth_utils.py
View File

@ -2,7 +2,8 @@
Unit tests for auth_utils functions related to rate limiting and customer ID extraction.
"""
from unittest.mock import patch
from typing import Optional
from unittest.mock import MagicMock, patch
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.auth_utils import (
@ -70,6 +71,19 @@ class TestGetKeyModelRpmLimit:
assert result is None
def test_team_metadata_empty_rpm_dict_falls_through_to_deployment_default(self):
"""Explicitly empty team model_rpm_limit ({}) should be returned as-is, not fallen through."""
# An empty dict is a valid team limit map (no per-model limits configured).
# It should be returned directly rather than falling through to deployment defaults,
# so a team with an empty map is treated as unconstrained at the team level.
user_api_key_dict = UserAPIKeyAuth(
api_key="sk-123",
team_metadata={"model_rpm_limit": {}},
)
result = get_key_model_rpm_limit(user_api_key_dict)
assert result == {}
class TestGetKeyModelTpmLimit:
"""Tests for get_key_model_tpm_limit function."""
@ -136,6 +150,33 @@ class TestGetKeyModelTpmLimit:
assert result == {"gpt-4": 10000}
def test_team_metadata_empty_tpm_dict_falls_through_to_deployment_default(self):
"""Explicitly empty team model_tpm_limit ({}) should be returned as-is, not fallen through."""
# An empty dict is a valid team limit map (no per-model limits configured).
# It should be returned directly rather than falling through to deployment defaults,
# so a team with an empty map is treated as unconstrained at the team level.
user_api_key_dict = UserAPIKeyAuth(
api_key="sk-123",
team_metadata={"model_tpm_limit": {}},
)
result = get_key_model_tpm_limit(user_api_key_dict)
assert result == {}
def test_skips_deployments_with_malformed_limit_value(self):
"""Deployments with non-integer-parseable limit values are skipped without raising."""
user_api_key_dict = UserAPIKeyAuth(api_key="sk-123")
mock_router = MagicMock()
mock_router.get_model_list.return_value = [
{"model_name": "model1", "litellm_params": {"default_api_key_tpm_limit": "not-a-number"}},
_make_deployment_dict("model1", tpm=500),
]
with patch(_ROUTER_PATCH, mock_router):
result = get_key_model_tpm_limit(user_api_key_dict, model_name="model1")
# The malformed deployment is skipped; the valid one provides 500
assert result == {"model1": 500}
class TestGetCustomerIdFromStandardHeaders:
"""Tests for _get_customer_id_from_standard_headers helper function."""
@ -315,3 +356,196 @@ def test_get_end_user_id_falls_back_to_deprecated_user_header_name():
result = get_end_user_id_from_request_body(request_body={}, request_headers=headers)
assert result == "user-legacy"
def _make_deployment_dict(model_name: str, tpm: Optional[int] = None, rpm: Optional[int] = None) -> dict:
"""Helper to build a minimal deployment dict as returned by router.get_model_list."""
litellm_params: dict = {"model": model_name}
if tpm is not None:
litellm_params["default_api_key_tpm_limit"] = tpm
if rpm is not None:
litellm_params["default_api_key_rpm_limit"] = rpm
return {"model_name": model_name, "litellm_params": litellm_params}
_ROUTER_PATCH = "litellm.proxy.proxy_server.llm_router"
class TestDeploymentDefaultRpmLimit:
"""Tests for deployment default_api_key_rpm_limit fallback in get_key_model_rpm_limit."""
def test_returns_deployment_default_when_key_has_no_limits(self):
"""Case 2 from spec: key has no model-specific limits, falls back to deployment default."""
user_api_key_dict = UserAPIKeyAuth(api_key="sk-123")
mock_router = MagicMock()
mock_router.get_model_list.return_value = [
_make_deployment_dict("model1", rpm=200)
]
with patch(_ROUTER_PATCH, mock_router):
result = get_key_model_rpm_limit(user_api_key_dict, model_name="model1")
assert result == {"model1": 200}
def test_key_model_limit_takes_priority_over_deployment_default(self):
"""Case 1 from spec: key model-specific limit wins over deployment default."""
user_api_key_dict = UserAPIKeyAuth(
api_key="sk-123",
metadata={"model_rpm_limit": {"model1": 10}},
)
mock_router = MagicMock()
mock_router.get_model_list.return_value = [
_make_deployment_dict("model1", rpm=200)
]
with patch(_ROUTER_PATCH, mock_router):
result = get_key_model_rpm_limit(user_api_key_dict, model_name="model1")
assert result == {"model1": 10}
def test_returns_none_when_no_deployment_default_and_no_key_limits(self):
"""Returns None when neither the key nor the deployment has any rpm limit."""
user_api_key_dict = UserAPIKeyAuth(api_key="sk-123")
mock_router = MagicMock()
mock_router.get_model_list.return_value = [
_make_deployment_dict("model1") # no rpm default
]
with patch(_ROUTER_PATCH, mock_router):
result = get_key_model_rpm_limit(user_api_key_dict, model_name="model1")
assert result is None
def test_returns_none_without_model_name_even_when_deployment_has_default(self):
"""No model_name means deployment fallback is skipped."""
user_api_key_dict = UserAPIKeyAuth(api_key="sk-123")
mock_router = MagicMock()
mock_router.get_model_list.return_value = [
_make_deployment_dict("model1", rpm=200)
]
with patch(_ROUTER_PATCH, mock_router):
result = get_key_model_rpm_limit(user_api_key_dict)
assert result is None
def test_returns_none_when_llm_router_is_none(self):
"""No router means deployment fallback returns None gracefully."""
user_api_key_dict = UserAPIKeyAuth(api_key="sk-123")
with patch(_ROUTER_PATCH, None):
result = get_key_model_rpm_limit(user_api_key_dict, model_name="model1")
assert result is None
def test_returns_minimum_across_multiple_deployments(self):
"""When multiple deployments share a model name, the minimum rpm limit is used."""
user_api_key_dict = UserAPIKeyAuth(api_key="sk-123")
mock_router = MagicMock()
mock_router.get_model_list.return_value = [
_make_deployment_dict("model1", rpm=200),
_make_deployment_dict("model1", rpm=50),
_make_deployment_dict("model1", rpm=150),
]
with patch(_ROUTER_PATCH, mock_router):
result = get_key_model_rpm_limit(user_api_key_dict, model_name="model1")
assert result == {"model1": 50}
def test_ignores_deployments_without_default_when_others_have_it(self):
"""Deployments missing the field are skipped; min is taken over those that have it."""
user_api_key_dict = UserAPIKeyAuth(api_key="sk-123")
mock_router = MagicMock()
mock_router.get_model_list.return_value = [
_make_deployment_dict("model1"), # no rpm default
_make_deployment_dict("model1", rpm=75),
]
with patch(_ROUTER_PATCH, mock_router):
result = get_key_model_rpm_limit(user_api_key_dict, model_name="model1")
assert result == {"model1": 75}
def test_skips_deployments_with_malformed_limit_value(self):
"""Deployments with non-integer-parseable limit values are skipped without raising."""
user_api_key_dict = UserAPIKeyAuth(api_key="sk-123")
mock_router = MagicMock()
mock_router.get_model_list.return_value = [
{"model_name": "model1", "litellm_params": {"default_api_key_rpm_limit": "not-a-number"}},
_make_deployment_dict("model1", rpm=100),
]
with patch(_ROUTER_PATCH, mock_router):
result = get_key_model_rpm_limit(user_api_key_dict, model_name="model1")
# The malformed deployment is skipped; the valid one provides 100
assert result == {"model1": 100}
class TestDeploymentDefaultTpmLimit:
"""Tests for deployment default_api_key_tpm_limit fallback in get_key_model_tpm_limit."""
def test_returns_deployment_default_when_key_has_no_limits(self):
"""Case 2 from spec: key has no model-specific limits, falls back to deployment default."""
user_api_key_dict = UserAPIKeyAuth(api_key="sk-123")
mock_router = MagicMock()
mock_router.get_model_list.return_value = [
_make_deployment_dict("model1", tpm=100)
]
with patch(_ROUTER_PATCH, mock_router):
result = get_key_model_tpm_limit(user_api_key_dict, model_name="model1")
assert result == {"model1": 100}
def test_key_model_limit_takes_priority_over_deployment_default(self):
"""Case 1 from spec: key model-specific limit wins over deployment default."""
user_api_key_dict = UserAPIKeyAuth(
api_key="sk-123",
metadata={"model_tpm_limit": {"model1": 20}},
)
mock_router = MagicMock()
mock_router.get_model_list.return_value = [
_make_deployment_dict("model1", tpm=100)
]
with patch(_ROUTER_PATCH, mock_router):
result = get_key_model_tpm_limit(user_api_key_dict, model_name="model1")
assert result == {"model1": 20}
def test_returns_none_when_no_deployment_default_and_no_key_limits(self):
"""Returns None when neither the key nor the deployment has any tpm limit."""
user_api_key_dict = UserAPIKeyAuth(api_key="sk-123")
mock_router = MagicMock()
mock_router.get_model_list.return_value = [
_make_deployment_dict("model1") # no tpm default
]
with patch(_ROUTER_PATCH, mock_router):
result = get_key_model_tpm_limit(user_api_key_dict, model_name="model1")
assert result is None
def test_returns_none_without_model_name_even_when_deployment_has_default(self):
"""No model_name means deployment fallback is skipped."""
user_api_key_dict = UserAPIKeyAuth(api_key="sk-123")
mock_router = MagicMock()
mock_router.get_model_list.return_value = [
_make_deployment_dict("model1", tpm=100)
]
with patch(_ROUTER_PATCH, mock_router):
result = get_key_model_tpm_limit(user_api_key_dict)
assert result is None
def test_returns_none_when_llm_router_is_none(self):
"""No router means deployment fallback returns None gracefully."""
user_api_key_dict = UserAPIKeyAuth(api_key="sk-123")
with patch(_ROUTER_PATCH, None):
result = get_key_model_tpm_limit(user_api_key_dict, model_name="model1")
assert result is None
def test_returns_minimum_across_multiple_deployments(self):
"""When multiple deployments share a model name, the minimum tpm limit is used."""
user_api_key_dict = UserAPIKeyAuth(api_key="sk-123")
mock_router = MagicMock()
mock_router.get_model_list.return_value = [
_make_deployment_dict("model1", tpm=1000),
_make_deployment_dict("model1", tpm=300),
_make_deployment_dict("model1", tpm=700),
]
with patch(_ROUTER_PATCH, mock_router):
result = get_key_model_tpm_limit(user_api_key_dict, model_name="model1")
assert result == {"model1": 300}
def test_ignores_deployments_without_default_when_others_have_it(self):
"""Deployments missing the field are skipped; min is taken over those that have it."""
user_api_key_dict = UserAPIKeyAuth(api_key="sk-123")
mock_router = MagicMock()
mock_router.get_model_list.return_value = [
_make_deployment_dict("model1"), # no tpm default
_make_deployment_dict("model1", tpm=400),
]
with patch(_ROUTER_PATCH, mock_router):
result = get_key_model_tpm_limit(user_api_key_dict, model_name="model1")
assert result == {"model1": 400}

167
tests/test_litellm/proxy/test_model_info_default_limits.py Normal file
View File

@ -0,0 +1,167 @@
"""
Tests verifying that default_api_key_tpm_limit and default_api_key_rpm_limit set in
litellm_params are returned by the /model/info endpoint.
"""
from typing import Optional
from unittest.mock import MagicMock, patch
import pytest
from litellm.proxy.proxy_server import _get_proxy_model_info
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
def _make_deployment(
model_name: str,
default_tpm: Optional[int] = None,
default_rpm: Optional[int] = None,
) -> Deployment:
params: dict = {"model": f"openai/{model_name}"}
if default_tpm is not None:
params["default_api_key_tpm_limit"] = default_tpm
if default_rpm is not None:
params["default_api_key_rpm_limit"] = default_rpm
return Deployment(
model_name=model_name,
litellm_params=LiteLLM_Params(**params),
model_info=ModelInfo(),
)
class TestModelInfoDefaultLimitsInResponse:
"""
Verify _get_proxy_model_info (the helper used by the /model/info endpoint) returns
default_api_key_tpm_limit and default_api_key_rpm_limit from litellm_params.
"""
def test_default_tpm_and_rpm_present_in_model_info_response(self):
"""Both defaults should appear in the litellm_params section of the response."""
deployment = _make_deployment("model1", default_tpm=100, default_rpm=200)
model_dict = deployment.model_dump(exclude_none=True)
result = _get_proxy_model_info(model=model_dict)
litellm_params = result["litellm_params"]
assert litellm_params.get("default_api_key_tpm_limit") == 100
assert litellm_params.get("default_api_key_rpm_limit") == 200
def test_default_tpm_only_present_when_only_tpm_configured(self):
"""Only the configured default appears; the other stays absent."""
deployment = _make_deployment("model1", default_tpm=500)
model_dict = deployment.model_dump(exclude_none=True)
result = _get_proxy_model_info(model=model_dict)
litellm_params = result["litellm_params"]
assert litellm_params.get("default_api_key_tpm_limit") == 500
assert "default_api_key_rpm_limit" not in litellm_params
def test_default_rpm_only_present_when_only_rpm_configured(self):
"""Only the configured default appears; the other stays absent."""
deployment = _make_deployment("model1", default_rpm=300)
model_dict = deployment.model_dump(exclude_none=True)
result = _get_proxy_model_info(model=model_dict)
litellm_params = result["litellm_params"]
assert litellm_params.get("default_api_key_rpm_limit") == 300
assert "default_api_key_tpm_limit" not in litellm_params
def test_defaults_absent_when_not_configured(self):
"""Neither field appears when not set on the deployment."""
deployment = _make_deployment("model1")
model_dict = deployment.model_dump(exclude_none=True)
result = _get_proxy_model_info(model=model_dict)
litellm_params = result["litellm_params"]
assert "default_api_key_tpm_limit" not in litellm_params
assert "default_api_key_rpm_limit" not in litellm_params
def test_defaults_not_masked_or_stripped_by_sensitive_data_filter(self):
"""
default_api_key_tpm_limit / default_api_key_rpm_limit must not be
treated as sensitive and must survive remove_sensitive_info_from_deployment.
They contain "key" which normally triggers masking; the call site explicitly
excludes these two fields via excluded_keys rather than widening the global
non_sensitive_overrides.
"""
deployment = _make_deployment("model1", default_tpm=100, default_rpm=200)
model_dict = deployment.model_dump(exclude_none=True)
result = _get_proxy_model_info(model=model_dict)
# Values should be unchanged integers, not masked strings
assert result["litellm_params"]["default_api_key_tpm_limit"] == 100
assert result["litellm_params"]["default_api_key_rpm_limit"] == 200
class TestModelInfoEndpointWithRouter:
"""
Integration-style tests simulating the /model/info endpoint reading from the router.
"""
@pytest.mark.asyncio
async def test_model_info_endpoint_returns_defaults_for_specific_model_id(self):
"""
When litellm_model_id is provided, the endpoint should return the deployment's
default limits in litellm_params.
"""
from litellm.proxy.proxy_server import model_info_v1
from litellm.proxy._types import UserAPIKeyAuth
deployment = _make_deployment("model1", default_tpm=100, default_rpm=200)
mock_router = MagicMock()
mock_router.get_deployment.return_value = deployment
user_api_key_dict = UserAPIKeyAuth(api_key="sk-test")
with patch("litellm.proxy.proxy_server.llm_router", mock_router), \
patch("litellm.proxy.proxy_server.llm_model_list", []), \
patch("litellm.proxy.proxy_server.user_model", None):
response = await model_info_v1(
user_api_key_dict=user_api_key_dict,
litellm_model_id="some-model-id",
)
assert len(response["data"]) == 1
litellm_params = response["data"][0]["litellm_params"]
assert litellm_params.get("default_api_key_tpm_limit") == 100
assert litellm_params.get("default_api_key_rpm_limit") == 200
@pytest.mark.asyncio
async def test_model_info_endpoint_returns_defaults_in_full_model_list(self):
"""
Without litellm_model_id, the endpoint iterates all models. Each deployment's
default limits should appear in its litellm_params entry.
"""
from litellm.proxy.proxy_server import model_info_v1
from litellm.proxy._types import UserAPIKeyAuth
deployment = _make_deployment("model1", default_tpm=100, default_rpm=200)
deployment_dict = deployment.model_dump(exclude_none=True)
mock_router = MagicMock()
mock_router.get_model_names.return_value = ["model1"]
mock_router.get_model_access_groups.return_value = {}
mock_router.get_model_list.return_value = [deployment_dict]
user_api_key_dict = UserAPIKeyAuth(api_key="sk-test")
with patch("litellm.proxy.proxy_server.llm_router", mock_router), \
patch("litellm.proxy.proxy_server.llm_model_list", [deployment_dict]), \
patch("litellm.proxy.proxy_server.user_model", None), \
patch("litellm.proxy.proxy_server.get_key_models", return_value=["model1"]), \
patch("litellm.proxy.proxy_server.get_team_models", return_value=["model1"]), \
patch("litellm.proxy.proxy_server.get_complete_model_list", return_value=["model1"]):
response = await model_info_v1(
user_api_key_dict=user_api_key_dict,
litellm_model_id=None,
)
assert len(response["data"]) >= 1
litellm_params = response["data"][0]["litellm_params"]
assert litellm_params.get("default_api_key_tpm_limit") == 100
assert litellm_params.get("default_api_key_rpm_limit") == 200