e0129710c8
* fix(proxy): add prisma reconnect primitive and db watchdog * fix(proxy): start and stop prisma watchdog in lifecycle * fix(auth): retry key lookup once after prisma reconnect * test(proxy): add prisma self-heal watchdog coverage * test(auth): cover reconnect-once behavior for key lookup * refactor(auth): extract db reconnect helper and remove inline import * fix(proxy): apply reconnect cooldown after attempt and add auth timeout path * fix(auth): bound reconnect latency on key lookup path * test(auth): assert reconnect timeout argument in key lookup * test(proxy): verify reconnect cooldown timestamp set after attempt * fix(proxy): harden prisma reconnect cycle semantics * test(proxy): cover watchdog reconnect + timeout budget * fix(proxy): bound watchdog probe and reconnect paths * test(proxy): cover watchdog timeout and probe behavior * fix(proxy): narrow prisma db connection error classification * fix(proxy): add auth reconnect lock timeout budget * fix(auth): pass lock timeout for db reconnect retries * test(proxy): cover narrow prisma connection error detection * test(proxy): add reconnect lock-timeout behavior coverage * test(auth): assert reconnect lock timeout argument * fix(proxy): avoid lock leak race in reconnect lock timeout path * test(proxy): cover reconnect lock-timeout race cleanup
84 lines
2.9 KiB
Python
84 lines
2.9 KiB
Python
from typing import Union
|
|
|
|
from litellm.proxy._types import (
|
|
DB_CONNECTION_ERROR_TYPES,
|
|
ProxyErrorTypes,
|
|
ProxyException,
|
|
)
|
|
from litellm.secret_managers.main import str_to_bool
|
|
|
|
|
|
class PrismaDBExceptionHandler:
|
|
"""
|
|
Class to handle DB Exceptions or Connection Errors
|
|
"""
|
|
|
|
@staticmethod
|
|
def should_allow_request_on_db_unavailable() -> bool:
|
|
"""
|
|
Returns True if the request should be allowed to proceed despite the DB connection error
|
|
"""
|
|
from litellm.proxy.proxy_server import general_settings
|
|
|
|
_allow_requests_on_db_unavailable: Union[bool, str] = general_settings.get(
|
|
"allow_requests_on_db_unavailable", False
|
|
)
|
|
if isinstance(_allow_requests_on_db_unavailable, bool):
|
|
return _allow_requests_on_db_unavailable
|
|
if str_to_bool(_allow_requests_on_db_unavailable) is True:
|
|
return True
|
|
return False
|
|
|
|
@staticmethod
|
|
def is_database_connection_error(e: Exception) -> bool:
|
|
"""
|
|
Returns True if the exception is from a database outage / connection error
|
|
"""
|
|
import prisma
|
|
|
|
if isinstance(e, DB_CONNECTION_ERROR_TYPES):
|
|
return True
|
|
if isinstance(
|
|
e, (prisma.errors.ClientNotConnectedError, prisma.errors.HTTPClientClosedError)
|
|
):
|
|
return True
|
|
if isinstance(e, prisma.errors.PrismaError):
|
|
error_message = str(e).lower()
|
|
# Treat generic PrismaError as connection error only when its text
|
|
# clearly indicates transport/connectivity failure.
|
|
connection_keywords = (
|
|
"can't reach database server",
|
|
"cannot reach database server",
|
|
"can't connect",
|
|
"cannot connect",
|
|
"connection error",
|
|
"connection closed",
|
|
"timed out",
|
|
"timeout",
|
|
"connection refused",
|
|
"network is unreachable",
|
|
"no route to host",
|
|
"broken pipe",
|
|
)
|
|
if any(keyword in error_message for keyword in connection_keywords):
|
|
return True
|
|
if isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection:
|
|
return True
|
|
return False
|
|
|
|
@staticmethod
|
|
def handle_db_exception(e: Exception):
|
|
"""
|
|
Primary handler for `allow_requests_on_db_unavailable` flag. Decides whether to raise a DB Exception or not based on the flag.
|
|
|
|
- If exception is a DB Connection Error, and `allow_requests_on_db_unavailable` is True,
|
|
- Do not raise an exception, return None
|
|
- Else, raise the exception
|
|
"""
|
|
if (
|
|
PrismaDBExceptionHandler.is_database_connection_error(e)
|
|
and PrismaDBExceptionHandler.should_allow_request_on_db_unavailable()
|
|
):
|
|
return None
|
|
raise e
|