ai-workspace-services/litellm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #5052 from BerriAI/litellm_stability_fixes

Browse Source 
fix: bump default allowed_fails + reduce default db pool limit
This commit is contained in:
Krish Dholakia 2024-08-05 20:54:23 -07:00 committed by GitHub
commit 2f304c92e4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 11 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
litellm/__init__.py
View File

@ -271,7 +271,7 @@ default_fallbacks: Optional[List] = None
fallbacks: Optional[List] = None
context_window_fallbacks: Optional[List] = None
content_policy_fallbacks: Optional[List] = None
allowed_fails: int = 0
allowed_fails: int = 3
num_retries_per_request: Optional[int] = (
None # for the request overall (incl. fallbacks + model retries)
)

12
litellm/proxy/proxy_cli.py
View File

@ -19,11 +19,17 @@ litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV"
if litellm_mode == "DEV":
load_dotenv()
import shutil
from enum import Enum
from importlib import resources
telemetry = None
class LiteLLMDatabaseConnectionPool(Enum):
database_connection_pool_limit = 10
database_connection_pool_timeout = 60
def append_query_params(url, params) -> str:
from litellm._logging import verbose_proxy_logger
@ -526,10 +532,12 @@ def run_server(
)
database_url = general_settings.get("database_url", None)
db_connection_pool_limit = general_settings.get(
"database_connection_pool_limit", 100
"database_connection_pool_limit",
LiteLLMDatabaseConnectionPool.database_connection_pool_limit.value,
)
db_connection_timeout = general_settings.get(
"database_connection_timeout", 60
"database_connection_timeout",
LiteLLMDatabaseConnectionPool.database_connection_pool_timeout.value,
)
if database_url and database_url.startswith("os.environ/"):
original_dir = os.getcwd()