2024-01-06 17:29:10 +08:00
datasource client {
provider = "postgresql"
url = env("DATABASE_URL")
}
2024-03-29 03:25:28 +08:00
generator client {
provider = "prisma-client-py"
2026-02-01 07:07:28 +08:00
binaryTargets = ["native", "debian-openssl-1.1.x", "debian-openssl-3.0.x", "linux-musl", "linux-musl-openssl-3.0.x"]
2024-03-29 03:25:28 +08:00
}
2024-03-03 03:59:17 +08:00
// Budget / Rate Limits for an org
model LiteLLM_BudgetTable {
budget_id String @id @default(uuid())
max_budget Float?
2024-03-03 04:25:40 +08:00
soft_budget Float?
2024-03-03 03:59:17 +08:00
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
2024-03-03 06:38:42 +08:00
model_max_budget Json?
feat(teams): per-member model scope + team default_team_member_models (#24950)
* fix(bedrock): strip [1m]/[200k] context window suffixes before cost lookup
* test(bedrock): add test for [1m] context window suffix stripping in cost lookup
* schema: add allowed_models to BudgetTable, default_team_member_models to TeamTable
* migration: add allowed_models and default_team_member_models columns
* types: add allowed_models to TeamMemberAddRequest, TeamMemberUpdateRequest, UpdateTeamRequest
* utils: add allowed_models param to add_new_member, persist to budget table
* common_utils: add allowed_models to _upsert_budget_and_membership
* team endpoints: seed allowed_models on member_add, persist on member_update and team/update
* auth: enforce per-member allowed_models at request time
* networking: add allowed_models to Member type and teamMemberUpdateCall
* TeamMemberTab: add Model Scope column showing per-member allowed_models
* EditMembership: add Allowed Models multi-select field
* TeamInfo: add default_team_member_models field in Settings tab
* chore: sync schema.prisma copies from root
* fix(team_member_update): update existing budget in-place instead of creating new one
When a member already has a budget_id, patch only the fields the caller
provided rather than always creating a fresh budget record. The old
code ignored existing_budget_id entirely, so updating only allowed_models
silently dropped the stored max_budget / tpm_limit / rpm_limit values.
* fix(auth): pass llm_router to _check_team_member_model_access
Without the router, _can_object_call_model cannot resolve wildcard model
names (e.g. openai/*) or access-group names in allowed_models, causing
legitimate requests to be denied. Thread the existing llm_router from
_run_common_checks through to the new member-scope check.
* feat(ui): add Team Member Settings accordion to Create Team modal
Groups default_team_member_models, member budget/key duration, and
tpm/rpm defaults into a single collapsible section. The model picker
is filtered to only show the models selected for the team, and the
copy distinguishes it from the team-level Models field.
* feat(ui): consolidate Team Member Settings into accordion in edit team form
Moves default_team_member_models + per-member budget/key/tpm/rpm fields
into a collapsible "Team Member Settings" panel. Keeps the top-level
form focused on team-wide settings (team models, team budget, tpm/rpm).
* fix(ui): use tremor Accordion for Team Member Settings in edit team form
* fix(ui): move Team Member Settings accordion above budget fields in Create Team
* chore: fixes
---------
Co-authored-by: Ishaan Jaffer <ishaanjaffer0324@gmail.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Yuneng Jiang <yuneng@berri.ai>
2026-04-07 04:48:43 +08:00
budget_duration String?
2024-03-03 03:59:17 +08:00
budget_reset_at DateTime?
feat(teams): per-member model scope + team default_team_member_models (#24950)
* fix(bedrock): strip [1m]/[200k] context window suffixes before cost lookup
* test(bedrock): add test for [1m] context window suffix stripping in cost lookup
* schema: add allowed_models to BudgetTable, default_team_member_models to TeamTable
* migration: add allowed_models and default_team_member_models columns
* types: add allowed_models to TeamMemberAddRequest, TeamMemberUpdateRequest, UpdateTeamRequest
* utils: add allowed_models param to add_new_member, persist to budget table
* common_utils: add allowed_models to _upsert_budget_and_membership
* team endpoints: seed allowed_models on member_add, persist on member_update and team/update
* auth: enforce per-member allowed_models at request time
* networking: add allowed_models to Member type and teamMemberUpdateCall
* TeamMemberTab: add Model Scope column showing per-member allowed_models
* EditMembership: add Allowed Models multi-select field
* TeamInfo: add default_team_member_models field in Settings tab
* chore: sync schema.prisma copies from root
* fix(team_member_update): update existing budget in-place instead of creating new one
When a member already has a budget_id, patch only the fields the caller
provided rather than always creating a fresh budget record. The old
code ignored existing_budget_id entirely, so updating only allowed_models
silently dropped the stored max_budget / tpm_limit / rpm_limit values.
* fix(auth): pass llm_router to _check_team_member_model_access
Without the router, _can_object_call_model cannot resolve wildcard model
names (e.g. openai/*) or access-group names in allowed_models, causing
legitimate requests to be denied. Thread the existing llm_router from
_run_common_checks through to the new member-scope check.
* feat(ui): add Team Member Settings accordion to Create Team modal
Groups default_team_member_models, member budget/key duration, and
tpm/rpm defaults into a single collapsible section. The model picker
is filtered to only show the models selected for the team, and the
copy distinguishes it from the team-level Models field.
* feat(ui): consolidate Team Member Settings into accordion in edit team form
Moves default_team_member_models + per-member budget/key/tpm/rpm fields
into a collapsible "Team Member Settings" panel. Keeps the top-level
form focused on team-wide settings (team models, team budget, tpm/rpm).
* fix(ui): use tremor Accordion for Team Member Settings in edit team form
* fix(ui): move Team Member Settings accordion above budget fields in Create Team
* chore: fixes
---------
Co-authored-by: Ishaan Jaffer <ishaanjaffer0324@gmail.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Yuneng Jiang <yuneng@berri.ai>
2026-04-07 04:48:43 +08:00
allowed_models String[] @default([]) // per-member model scope; empty = inherit team models
2024-03-03 03:59:17 +08:00
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
2024-03-03 04:18:28 +08:00
organization LiteLLM_OrganizationTable[] // multiple orgs can have the same budget
2025-11-13 06:38:15 +08:00
projects LiteLLM_ProjectTable[] // multiple projects can have the same budget
2024-03-03 10:34:18 +08:00
keys LiteLLM_VerificationToken[] // multiple keys can have the same budget
2024-03-17 03:26:29 +08:00
end_users LiteLLM_EndUserTable[] // multiple end-users can have the same budget
2025-10-11 10:24:50 +08:00
tags LiteLLM_TagTable[] // multiple tags can have the same budget
2025-06-26 13:37:45 +08:00
team_membership LiteLLM_TeamMembership[] // budgets of Users within a Team
organization_membership LiteLLM_OrganizationMembership[] // budgets of Users within a Organization
2024-03-03 03:59:17 +08:00
}
2025-03-11 09:27:43 +08:00
// Models on proxy
model LiteLLM_CredentialsTable {
credential_id String @id @default(uuid())
credential_name String @unique
credential_values Json
2025-06-26 13:37:45 +08:00
credential_info Json?
2025-03-11 09:27:43 +08:00
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
}
2024-04-04 11:17:34 +08:00
// Models on proxy
model LiteLLM_ProxyModelTable {
model_id String @id @default(uuid())
2025-06-26 13:37:45 +08:00
model_name String
2024-04-04 11:17:34 +08:00
litellm_params Json
2025-06-26 13:37:45 +08:00
model_info Json?
2024-04-04 11:17:34 +08:00
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
2025-11-15 10:23:30 +08:00
updated_by String
}
// Agents on proxy
model LiteLLM_AgentsTable {
agent_id String @id @default(uuid())
agent_name String @unique
litellm_params Json?
agent_card_params Json
2026-03-07 15:39:08 +08:00
static_headers Json? @default("{}")
extra_headers String[] @default([])
2025-12-05 08:31:00 +08:00
agent_access_groups String[] @default([])
2026-02-26 03:44:30 +08:00
object_permission_id String?
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
2026-03-08 11:12:42 +08:00
spend Float @default(0.0)
tpm_limit Int?
rpm_limit Int?
session_tpm_limit Int?
session_rpm_limit Int?
2025-11-15 10:23:30 +08:00
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
2024-04-04 11:17:34 +08:00
updated_by String
}
2024-03-03 03:59:17 +08:00
model LiteLLM_OrganizationTable {
organization_id String @id @default(uuid())
2024-03-03 04:18:28 +08:00
organization_alias String
2024-03-03 03:59:17 +08:00
budget_id String
metadata Json @default("{}")
models String[]
spend Float @default(0.0)
model_spend Json @default("{}")
2025-05-08 12:09:51 +08:00
object_permission_id String?
2024-03-03 03:59:17 +08:00
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
2024-03-03 04:18:28 +08:00
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
2025-06-26 13:37:45 +08:00
teams LiteLLM_TeamTable[]
2024-04-09 11:45:11 +08:00
users LiteLLM_UserTable[]
2025-02-11 11:13:32 +08:00
keys LiteLLM_VerificationToken[]
2025-01-05 09:31:24 +08:00
members LiteLLM_OrganizationMembership[] @relation("OrganizationToMembership")
2025-05-08 12:09:51 +08:00
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
2024-03-03 03:59:17 +08:00
}
2024-03-07 10:55:40 +08:00
// Model info for teams, just has model aliases for now.
model LiteLLM_ModelTable {
id Int @id @default(autoincrement())
model_aliases Json? @map("aliases")
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
team LiteLLM_TeamTable?
}
2024-03-27 08:03:21 +08:00
2025-06-26 13:37:45 +08:00
// Assign prod keys to groups, not individuals
2024-02-15 09:20:41 +08:00
model LiteLLM_TeamTable {
2024-03-03 03:59:17 +08:00
team_id String @id @default(uuid())
2025-06-26 13:37:45 +08:00
team_alias String?
2024-03-03 03:59:17 +08:00
organization_id String?
2025-05-08 12:09:51 +08:00
object_permission_id String?
2024-02-15 09:20:41 +08:00
admins String[]
members String[]
2024-02-22 05:29:42 +08:00
members_with_roles Json @default("{}")
2024-02-15 09:20:41 +08:00
metadata Json @default("{}")
max_budget Float?
2026-02-06 06:43:48 +08:00
soft_budget Float?
2024-02-15 09:20:41 +08:00
spend Float @default(0.0)
models String[]
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
2025-06-26 13:37:45 +08:00
budget_duration String?
2024-02-15 09:20:41 +08:00
budget_reset_at DateTime?
2024-03-27 08:03:21 +08:00
blocked Boolean @default(false)
2024-02-15 09:20:41 +08:00
created_at DateTime @default(now()) @map("created_at")
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
2024-02-17 08:32:17 +08:00
model_spend Json @default("{}")
model_max_budget Json @default("{}")
2026-01-06 08:19:42 +08:00
router_settings Json? @default("{}")
2025-04-13 00:06:04 +08:00
team_member_permissions String[] @default([])
2026-02-13 04:30:22 +08:00
access_group_ids String[] @default([])
2026-01-24 05:16:58 +08:00
policies String[] @default([])
feat(teams): per-member model scope + team default_team_member_models (#24950)
* fix(bedrock): strip [1m]/[200k] context window suffixes before cost lookup
* test(bedrock): add test for [1m] context window suffix stripping in cost lookup
* schema: add allowed_models to BudgetTable, default_team_member_models to TeamTable
* migration: add allowed_models and default_team_member_models columns
* types: add allowed_models to TeamMemberAddRequest, TeamMemberUpdateRequest, UpdateTeamRequest
* utils: add allowed_models param to add_new_member, persist to budget table
* common_utils: add allowed_models to _upsert_budget_and_membership
* team endpoints: seed allowed_models on member_add, persist on member_update and team/update
* auth: enforce per-member allowed_models at request time
* networking: add allowed_models to Member type and teamMemberUpdateCall
* TeamMemberTab: add Model Scope column showing per-member allowed_models
* EditMembership: add Allowed Models multi-select field
* TeamInfo: add default_team_member_models field in Settings tab
* chore: sync schema.prisma copies from root
* fix(team_member_update): update existing budget in-place instead of creating new one
When a member already has a budget_id, patch only the fields the caller
provided rather than always creating a fresh budget record. The old
code ignored existing_budget_id entirely, so updating only allowed_models
silently dropped the stored max_budget / tpm_limit / rpm_limit values.
* fix(auth): pass llm_router to _check_team_member_model_access
Without the router, _can_object_call_model cannot resolve wildcard model
names (e.g. openai/*) or access-group names in allowed_models, causing
legitimate requests to be denied. Thread the existing llm_router from
_run_common_checks through to the new member-scope check.
* feat(ui): add Team Member Settings accordion to Create Team modal
Groups default_team_member_models, member budget/key duration, and
tpm/rpm defaults into a single collapsible section. The model picker
is filtered to only show the models selected for the team, and the
copy distinguishes it from the team-level Models field.
* feat(ui): consolidate Team Member Settings into accordion in edit team form
Moves default_team_member_models + per-member budget/key/tpm/rpm fields
into a collapsible "Team Member Settings" panel. Keeps the top-level
form focused on team-wide settings (team models, team budget, tpm/rpm).
* fix(ui): use tremor Accordion for Team Member Settings in edit team form
* fix(ui): move Team Member Settings accordion above budget fields in Create Team
* chore: fixes
---------
Co-authored-by: Ishaan Jaffer <ishaanjaffer0324@gmail.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Yuneng Jiang <yuneng@berri.ai>
2026-04-07 04:48:43 +08:00
default_team_member_models String[] @default([]) // default allowed_models for newly added team members; empty = no per-member restriction
2026-04-18 05:47:05 +08:00
budget_limits Json? // per-model budget limits for the team
2024-06-09 10:03:45 +08:00
model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases
2026-02-05 11:51:20 +08:00
allow_team_guardrail_config Boolean @default(false) // if true, team admin can configure guardrails for this team
2024-03-03 04:18:28 +08:00
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
2024-03-07 10:55:40 +08:00
litellm_model_table LiteLLM_ModelTable? @relation(fields: [model_id], references: [id])
2025-05-08 12:09:51 +08:00
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
2025-11-13 06:38:15 +08:00
projects LiteLLM_ProjectTable[]
2026-03-18 11:34:15 +08:00
@@index([organization_id])
@@index([team_alias])
@@index([created_at])
2025-11-13 06:38:15 +08:00
}
// Projects sit between teams and keys for use-case management
model LiteLLM_ProjectTable {
project_id String @id @default(uuid())
project_alias String?
2025-11-13 06:58:38 +08:00
description String?
2025-11-13 06:38:15 +08:00
team_id String?
budget_id String?
metadata Json @default("{}")
models String[]
spend Float @default(0.0)
model_spend Json @default("{}")
2025-11-13 06:58:38 +08:00
model_rpm_limit Json @default("{}")
model_tpm_limit Json @default("{}")
2025-11-13 06:38:15 +08:00
blocked Boolean @default(false)
object_permission_id String?
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
// Relations
litellm_team_table LiteLLM_TeamTable? @relation(fields: [team_id], references: [team_id])
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
keys LiteLLM_VerificationToken[]
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
2024-02-15 09:20:41 +08:00
}
2026-01-17 06:25:23 +08:00
// Audit table for deleted teams - preserves spend and team information for historical tracking
model LiteLLM_DeletedTeamTable {
id String @id @default(uuid())
team_id String // Original team_id
team_alias String?
organization_id String?
object_permission_id String?
admins String[]
members String[]
members_with_roles Json @default("{}")
metadata Json @default("{}")
max_budget Float?
2026-02-08 03:05:42 +08:00
soft_budget Float?
2026-01-17 06:25:23 +08:00
spend Float @default(0.0)
models String[]
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
budget_duration String?
budget_reset_at DateTime?
blocked Boolean @default(false)
model_spend Json @default("{}")
model_max_budget Json @default("{}")
router_settings Json? @default("{}")
team_member_permissions String[] @default([])
2026-02-13 04:30:22 +08:00
access_group_ids String[] @default([])
2026-01-24 05:16:58 +08:00
policies String[] @default([])
2026-01-17 06:25:23 +08:00
model_id Int? // id for LiteLLM_ModelTable -> stores team-level model aliases
2026-02-05 11:51:20 +08:00
allow_team_guardrail_config Boolean @default(false)
2026-02-15 01:49:11 +08:00
2026-01-17 06:25:23 +08:00
// Original timestamps from team creation/updates
created_at DateTime? @map("created_at")
updated_at DateTime? @map("updated_at")
// Deletion metadata
deleted_at DateTime @default(now()) @map("deleted_at")
deleted_by String? @map("deleted_by") // User who deleted the team
deleted_by_api_key String? @map("deleted_by_api_key") // API key hash that performed the deletion
litellm_changed_by String? @map("litellm_changed_by") // From litellm-changed-by header if provided
@@index([team_id])
@@index([deleted_at])
@@index([organization_id])
@@index([team_alias])
@@index([created_at])
}
2024-01-25 09:15:01 +08:00
// Track spend, rate limit, budget Users
2024-01-06 17:29:10 +08:00
model LiteLLM_UserTable {
2024-03-03 03:59:17 +08:00
user_id String @id
2025-06-26 13:37:45 +08:00
user_alias String?
2024-01-19 06:42:46 +08:00
team_id String?
2025-02-01 15:04:51 +08:00
sso_user_id String? @unique
2024-04-09 11:45:11 +08:00
organization_id String?
2025-05-08 12:09:51 +08:00
object_permission_id String?
2024-05-28 11:32:25 +08:00
password String?
2024-02-22 05:29:42 +08:00
teams String[] @default([])
2024-02-04 02:23:50 +08:00
user_role String?
2024-01-06 17:29:10 +08:00
max_budget Float?
spend Float @default(0.0)
user_email String?
2024-01-18 07:45:31 +08:00
models String[]
2024-09-24 08:49:36 +08:00
metadata Json @default("{}")
2024-01-19 09:03:18 +08:00
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
2025-06-26 13:37:45 +08:00
budget_duration String?
2024-01-25 06:27:13 +08:00
budget_reset_at DateTime?
2024-01-31 13:17:01 +08:00
allowed_cache_controls String[] @default([])
2026-01-24 05:16:58 +08:00
policies String[] @default([])
2024-02-17 08:32:17 +08:00
model_spend Json @default("{}")
model_max_budget Json @default("{}")
2024-12-08 11:08:37 +08:00
created_at DateTime? @default(now()) @map("created_at")
updated_at DateTime? @default(now()) @updatedAt @map("updated_at")
2024-10-09 17:48:18 +08:00
// relations
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
organization_memberships LiteLLM_OrganizationMembership[]
2024-05-28 11:32:25 +08:00
invitations_created LiteLLM_InvitationLink[] @relation("CreatedBy")
invitations_updated LiteLLM_InvitationLink[] @relation("UpdatedBy")
invitations_user LiteLLM_InvitationLink[] @relation("UserId")
2025-05-08 12:09:51 +08:00
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
}
model LiteLLM_ObjectPermissionTable {
object_permission_id String @id @default(uuid())
mcp_servers String[] @default([])
2025-07-11 12:59:25 +08:00
mcp_access_groups String[] @default([])
2025-10-07 09:49:32 +08:00
mcp_tool_permissions Json? // Tool-level permissions for MCP servers. Format: {"server_id": ["tool_name_1", "tool_name_2"]}
2025-05-29 07:58:53 +08:00
vector_stores String[] @default([])
2025-12-05 08:31:00 +08:00
agents String[] @default([])
agent_access_groups String[] @default([])
2026-03-11 12:03:20 +08:00
models String[] @default([])
2026-03-04 12:22:20 +08:00
blocked_tools String[] @default([]) // Tool names blocked for any key/team/user with this permission
Litellm ishaan march23 - MCP Toolsets + GCP Caching fix (#25146) (#25155)
* Litellm ishaan march23 - MCP Toolsets + GCP Caching fix (#25146)
* feat(mcp): MCP Toolsets — curated tool subsets from one or more MCP servers (#24335)
* feat(mcp): add LiteLLM_MCPToolsetTable and mcp_toolsets to ObjectPermissionTable
* feat(mcp): add prisma migration for MCPToolset table
* feat(mcp): add MCPToolset Python types
* feat(mcp): add toolset_db.py with CRUD helpers for MCPToolset
* feat(mcp): add toolset CRUD endpoints to mcp_management_endpoints
* fix(mcp): skip allow_all_keys servers when explicit mcp_servers permission is set (toolset scope fix)
* feat(mcp): add _apply_toolset_scope and toolset route handling in server.py
* fix(mcp): resolve toolset names in responses API before fetching tools
* feat(mcp): add mcp_toolsets field to LiteLLM_ObjectPermissionTable type
* feat(mcp): register LiteLLM_MCPToolsetTable in prisma client initialization
* feat(mcp): validate mcp_toolsets in key-vs-team permission check
* feat(mcp): register toolset routes in proxy_server.py
* feat(mcp): add MCPToolset and MCPToolsetTool TypeScript types
* feat(mcp): add fetchMCPToolsets, createMCPToolset, updateMCPToolset, deleteMCPToolset API functions
* feat(mcp): add useMCPToolsets React Query hook
* feat(mcp): add toolsets (purple) as third option type in MCPServerSelector
* feat(mcp): extract toolsets from combined MCP field in key form
* feat(mcp): extract toolsets from combined MCP field in team form
* feat(mcp): show toolsets section in MCPServerPermissions read view
* feat(mcp): pass mcp_toolsets through object_permissions_view
* feat(mcp): add MCPToolsetsTab component for creating and managing toolsets
* feat(mcp): add Toolsets tab to mcp_servers.tsx
* feat(mcp): pass mcpToolsets to playground chat and responses API calls
* feat(mcp): generate correct server_url for toolsets in playground API calls
* docs(mcp): add MCP Toolsets documentation
* docs(mcp): add mcp_toolsets to sidebar
* fix(mcp): replace x-mcp-toolset-id header with ContextVar to prevent client forgery
* fix(mcp): use ContextVar + StreamingResponse for toolset MCP routes (fixes SSE streaming)
* fix(mcp): cache toolset permission lookups to avoid per-request DB calls
* test(mcp): add tests for toolset scope enforcement, ContextVar isolation, and access control
* fix(mcp): cache toolset name lookups in MCPServerManager to avoid per-request DB calls
* fix(mcp): prevent body_iter deadlock + use cached toolset lookup in responses API
- _stream_mcp_asgi_response: add done callback to handler_task that puts
the EOF sentinel on body_queue when the task exits, preventing body_iter
from hanging forever if the handler raises after headers are sent.
- litellm_proxy_mcp_handler: replace raw get_mcp_toolset_by_name() DB call
with global_mcp_server_manager.get_toolset_by_name_cached() so toolset
resolution uses the 60s TTL cache added for this purpose instead of
hitting the DB on every responses-API request.
* fix(mcp): toolset access control, asyncio fix, and real unit tests
- server.py: _apply_toolset_scope now enforces that non-admin keys must
have the requested toolset_id in their mcp_toolsets grant list;
admin keys always bypass the check.
- mcp_management_endpoints.py: three access-control fixes:
* fetch_mcp_toolsets: non-admin keys with mcp_toolsets=None now
return [] instead of all toolsets (only admins get 'all' when
the field is absent)
* fetch_mcp_toolset: non-admin keys that haven't been granted the
requested toolset_id now get 403 instead of the full result
* add_mcp_toolset: duplicate toolset_name now returns 409 Conflict
instead of an opaque 500
- proxy_server.py: use asyncio.get_running_loop() instead of
get_event_loop() inside an already-running coroutine (Python 3.10+).
- test_mcp_toolset_scope.py: replace four hollow tests that only
asserted local variable properties with real tests that call the
production fetch_mcp_toolsets() and handle_streamable_http_mcp()
functions with mocked dependencies.
* fix(mcp): add mcp_toolsets to ObjectPermissionBase, fix multi-toolset overwrite, fix delete 404, allow standalone key toolsets
* fix(mcp): add auth check on toolset resolution in responses API; union mcp_servers in _merge_toolset_permissions
* fix(mcp): handle RecordNotFoundError in update_mcp_toolset; union direct servers with toolset servers
* fix(mcp): use _user_has_admin_view; deny None mcp_toolsets for non-admin; use direct RecordNotFoundError import; fix docstring
* fix(mcp): add @default(now()) to MCPToolsetTable.updated_at; fix test for non-admin toolset access
* fix: use UniqueViolationError import; guard _ensure_eof for error/cancel only
* fix(mcp): preserve mcp_access_groups in toolset scope, use shared Redis cache for toolset perms
- Remove mcp_access_groups=[] from _apply_toolset_scope (server.py) and the
responses API toolset path (litellm_proxy_mcp_handler.py). A key's access-group
grants remain valid even when the request is scoped to a single toolset; clearing
them silently revoked legitimate entitlements.
- Switch resolve_toolset_tool_permissions and get_toolset_by_name_cached to use
user_api_key_cache (Redis-backed DualCache in production) instead of per-instance
in-memory dicts. Cache entries are now shared across workers, eliminating the
per-worker stale-toolset-permission window flagged as a P1 by Greptile.
- Use union merge (set union of tool names per server) when applying toolset
permissions in the responses API path so direct-server tool restrictions are not
overwritten by toolset permissions.
* fix(mcp): return 404 when edit_mcp_toolset target does not exist
* fix(mcp): align mcp_toolsets default to None in LiteLLM_ObjectPermissionTable
* fix(mcp): admin toolset visibility, in-place tool name mutation, test helper coercion
* fix(mcp): treat None/[] team mcp_toolsets as no restriction in key validation
* fix(mcp): allow_all_keys backward compat, blocked_tools API write-path, efficient startup query
* fix(mcp): use _mcp_active_toolset_id ContextVar to detect toolset scope, avoiding DB-default false-positive
* fix(mcp): remove dead toolset cache stubs, log invalidation failures, align schema updated_at defaults
* fix(mcp): deserialise MCPToolset from Redis cache hit, replace fastapi import in test
* fix(mcp): evict name-cache on toolset mutation, 409 on rename conflict, warning-level list errors
* fix(redis): regenerate GCP IAM token per connection for async cluster (#24426)
* fix(redis): regenerate GCP IAM token per connection for async cluster clients
Async RedisCluster was generating the IAM token once at startup and
storing it as a static password. After the 1-hour GCP token TTL, any
new connection (including to newly-discovered cluster nodes) would fail
to authenticate.
Fix: introduce GCPIAMCredentialProvider that implements redis-py's
CredentialProvider protocol. It calls _generate_gcp_iam_access_token()
on every new connection, matching what the sync redis_connect_func
already does. async_redis.RedisCluster accepts a credential_provider
kwarg which is invoked per-connection.
* refactor(redis): move GCPIAMCredentialProvider to its own file
Extract GCPIAMCredentialProvider and _generate_gcp_iam_access_token
into litellm/_redis_credential_provider.py. _redis.py imports them
from there, keeping the public API unchanged.
* fix: address Greptile review issues
- GCPIAMCredentialProvider now inherits from redis.credentials.CredentialProvider
so redis-py's async path calls get_credentials_async() properly
- move _redis_credential_provider import to top of _redis.py (PEP 8)
- remove dead else-branch that silently no-oped (gcp_service_account from
redis_kwargs.get() was always None since it's popped by _get_redis_client_logic)
- remove mid-function 'from litellm import get_secret_str' inline import
- remove unused 'call' import from test_redis.py
* chore: retrigger CI/review
* chore: sync schema.prisma copies from root
* chore: sync schema.prisma copies from root
* fix(proxy_server): use bounded asyncio.Queue with maxsize to prevent unbounded growth
* fix(a2a/pydantic_ai): make api_base Optional to match base class signature
* fix(a2a/pydantic_ai): make api_base Optional in handler and guard against None
* fix(mcp): remove unused get_all_mcp_servers import
* fix(mcp): remove unused MCPToolset import
* refactor(mcp): extract toolset permission logic to reduce statement count below PLR0915 limit
* fix(tests): update reload_servers_from_database tests to mock prisma directly
---------
Co-authored-by: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
* fix(toolset_db): lazy-import prisma to avoid ImportError when prisma not installed
* fix(tests): update UI tests for toolset tab and updated empty state text
* fix(tests): add get_mcp_server_by_name to fake_manager stub
---------
Co-authored-by: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
2026-04-05 07:23:21 +08:00
mcp_toolsets String[] @default([]) // Toolset IDs granted to this key/team/user
2025-05-08 12:09:51 +08:00
teams LiteLLM_TeamTable[]
2025-11-13 06:38:15 +08:00
projects LiteLLM_ProjectTable[]
2025-05-08 12:09:51 +08:00
verification_tokens LiteLLM_VerificationToken[]
organizations LiteLLM_OrganizationTable[]
users LiteLLM_UserTable[]
2026-02-19 10:53:59 +08:00
end_users LiteLLM_EndUserTable[]
2026-02-26 03:44:30 +08:00
agents_table LiteLLM_AgentsTable[]
2025-05-08 12:09:51 +08:00
}
2025-06-26 13:37:45 +08:00
// Holds the MCP server configuration
2025-05-08 12:09:51 +08:00
model LiteLLM_MCPServerTable {
2025-07-12 11:25:26 +08:00
server_id String @id @default(uuid())
2025-07-26 08:57:52 +08:00
server_name String?
2025-07-12 11:25:26 +08:00
alias String?
description String?
2026-04-14 19:36:06 +08:00
instructions String?
2025-07-12 11:25:26 +08:00
url String?
2026-03-05 08:07:05 +08:00
spec_path String?
2025-07-12 11:25:26 +08:00
transport String @default("sse")
auth_type String?
2025-11-08 11:22:49 +08:00
credentials Json? @default("{}")
2025-07-12 11:25:26 +08:00
created_at DateTime? @default(now()) @map("created_at")
created_by String?
updated_at DateTime? @default(now()) @updatedAt @map("updated_at")
updated_by String?
mcp_info Json? @default("{}")
2025-07-11 12:59:25 +08:00
mcp_access_groups String[]
2025-10-04 01:16:29 +08:00
allowed_tools String[] @default([])
2026-03-07 15:39:08 +08:00
tool_name_to_display_name Json? @default("{}")
tool_name_to_description Json? @default("{}")
2025-10-04 10:07:50 +08:00
extra_headers String[] @default([])
2025-11-04 13:06:36 +08:00
static_headers Json? @default("{}")
2025-07-30 23:10:44 +08:00
// Health check status
status String? @default("unknown")
last_health_check DateTime?
health_check_error String?
2025-07-12 11:25:26 +08:00
// Stdio-specific fields
command String?
args String[] @default([])
env Json? @default("{}")
2026-01-02 13:07:58 +08:00
authorization_url String?
token_url String?
registration_url String?
2026-01-05 14:49:09 +08:00
allow_all_keys Boolean @default(false)
2026-02-28 12:06:07 +08:00
available_on_public_internet Boolean @default(true)
2026-03-07 15:39:08 +08:00
is_byok Boolean @default(false)
byok_description String[] @default([])
byok_api_key_help_url String?
2026-03-28 07:01:20 +08:00
source_url String?
// BYOM submission lifecycle
approval_status String? @default("active")
submitted_by String?
submitted_at DateTime?
reviewed_at DateTime?
review_notes String?
@@index([approval_status])
2026-03-07 15:39:08 +08:00
}
Litellm ishaan march23 - MCP Toolsets + GCP Caching fix (#25146) (#25155)
* Litellm ishaan march23 - MCP Toolsets + GCP Caching fix (#25146)
* feat(mcp): MCP Toolsets — curated tool subsets from one or more MCP servers (#24335)
* feat(mcp): add LiteLLM_MCPToolsetTable and mcp_toolsets to ObjectPermissionTable
* feat(mcp): add prisma migration for MCPToolset table
* feat(mcp): add MCPToolset Python types
* feat(mcp): add toolset_db.py with CRUD helpers for MCPToolset
* feat(mcp): add toolset CRUD endpoints to mcp_management_endpoints
* fix(mcp): skip allow_all_keys servers when explicit mcp_servers permission is set (toolset scope fix)
* feat(mcp): add _apply_toolset_scope and toolset route handling in server.py
* fix(mcp): resolve toolset names in responses API before fetching tools
* feat(mcp): add mcp_toolsets field to LiteLLM_ObjectPermissionTable type
* feat(mcp): register LiteLLM_MCPToolsetTable in prisma client initialization
* feat(mcp): validate mcp_toolsets in key-vs-team permission check
* feat(mcp): register toolset routes in proxy_server.py
* feat(mcp): add MCPToolset and MCPToolsetTool TypeScript types
* feat(mcp): add fetchMCPToolsets, createMCPToolset, updateMCPToolset, deleteMCPToolset API functions
* feat(mcp): add useMCPToolsets React Query hook
* feat(mcp): add toolsets (purple) as third option type in MCPServerSelector
* feat(mcp): extract toolsets from combined MCP field in key form
* feat(mcp): extract toolsets from combined MCP field in team form
* feat(mcp): show toolsets section in MCPServerPermissions read view
* feat(mcp): pass mcp_toolsets through object_permissions_view
* feat(mcp): add MCPToolsetsTab component for creating and managing toolsets
* feat(mcp): add Toolsets tab to mcp_servers.tsx
* feat(mcp): pass mcpToolsets to playground chat and responses API calls
* feat(mcp): generate correct server_url for toolsets in playground API calls
* docs(mcp): add MCP Toolsets documentation
* docs(mcp): add mcp_toolsets to sidebar
* fix(mcp): replace x-mcp-toolset-id header with ContextVar to prevent client forgery
* fix(mcp): use ContextVar + StreamingResponse for toolset MCP routes (fixes SSE streaming)
* fix(mcp): cache toolset permission lookups to avoid per-request DB calls
* test(mcp): add tests for toolset scope enforcement, ContextVar isolation, and access control
* fix(mcp): cache toolset name lookups in MCPServerManager to avoid per-request DB calls
* fix(mcp): prevent body_iter deadlock + use cached toolset lookup in responses API
- _stream_mcp_asgi_response: add done callback to handler_task that puts
the EOF sentinel on body_queue when the task exits, preventing body_iter
from hanging forever if the handler raises after headers are sent.
- litellm_proxy_mcp_handler: replace raw get_mcp_toolset_by_name() DB call
with global_mcp_server_manager.get_toolset_by_name_cached() so toolset
resolution uses the 60s TTL cache added for this purpose instead of
hitting the DB on every responses-API request.
* fix(mcp): toolset access control, asyncio fix, and real unit tests
- server.py: _apply_toolset_scope now enforces that non-admin keys must
have the requested toolset_id in their mcp_toolsets grant list;
admin keys always bypass the check.
- mcp_management_endpoints.py: three access-control fixes:
* fetch_mcp_toolsets: non-admin keys with mcp_toolsets=None now
return [] instead of all toolsets (only admins get 'all' when
the field is absent)
* fetch_mcp_toolset: non-admin keys that haven't been granted the
requested toolset_id now get 403 instead of the full result
* add_mcp_toolset: duplicate toolset_name now returns 409 Conflict
instead of an opaque 500
- proxy_server.py: use asyncio.get_running_loop() instead of
get_event_loop() inside an already-running coroutine (Python 3.10+).
- test_mcp_toolset_scope.py: replace four hollow tests that only
asserted local variable properties with real tests that call the
production fetch_mcp_toolsets() and handle_streamable_http_mcp()
functions with mocked dependencies.
* fix(mcp): add mcp_toolsets to ObjectPermissionBase, fix multi-toolset overwrite, fix delete 404, allow standalone key toolsets
* fix(mcp): add auth check on toolset resolution in responses API; union mcp_servers in _merge_toolset_permissions
* fix(mcp): handle RecordNotFoundError in update_mcp_toolset; union direct servers with toolset servers
* fix(mcp): use _user_has_admin_view; deny None mcp_toolsets for non-admin; use direct RecordNotFoundError import; fix docstring
* fix(mcp): add @default(now()) to MCPToolsetTable.updated_at; fix test for non-admin toolset access
* fix: use UniqueViolationError import; guard _ensure_eof for error/cancel only
* fix(mcp): preserve mcp_access_groups in toolset scope, use shared Redis cache for toolset perms
- Remove mcp_access_groups=[] from _apply_toolset_scope (server.py) and the
responses API toolset path (litellm_proxy_mcp_handler.py). A key's access-group
grants remain valid even when the request is scoped to a single toolset; clearing
them silently revoked legitimate entitlements.
- Switch resolve_toolset_tool_permissions and get_toolset_by_name_cached to use
user_api_key_cache (Redis-backed DualCache in production) instead of per-instance
in-memory dicts. Cache entries are now shared across workers, eliminating the
per-worker stale-toolset-permission window flagged as a P1 by Greptile.
- Use union merge (set union of tool names per server) when applying toolset
permissions in the responses API path so direct-server tool restrictions are not
overwritten by toolset permissions.
* fix(mcp): return 404 when edit_mcp_toolset target does not exist
* fix(mcp): align mcp_toolsets default to None in LiteLLM_ObjectPermissionTable
* fix(mcp): admin toolset visibility, in-place tool name mutation, test helper coercion
* fix(mcp): treat None/[] team mcp_toolsets as no restriction in key validation
* fix(mcp): allow_all_keys backward compat, blocked_tools API write-path, efficient startup query
* fix(mcp): use _mcp_active_toolset_id ContextVar to detect toolset scope, avoiding DB-default false-positive
* fix(mcp): remove dead toolset cache stubs, log invalidation failures, align schema updated_at defaults
* fix(mcp): deserialise MCPToolset from Redis cache hit, replace fastapi import in test
* fix(mcp): evict name-cache on toolset mutation, 409 on rename conflict, warning-level list errors
* fix(redis): regenerate GCP IAM token per connection for async cluster (#24426)
* fix(redis): regenerate GCP IAM token per connection for async cluster clients
Async RedisCluster was generating the IAM token once at startup and
storing it as a static password. After the 1-hour GCP token TTL, any
new connection (including to newly-discovered cluster nodes) would fail
to authenticate.
Fix: introduce GCPIAMCredentialProvider that implements redis-py's
CredentialProvider protocol. It calls _generate_gcp_iam_access_token()
on every new connection, matching what the sync redis_connect_func
already does. async_redis.RedisCluster accepts a credential_provider
kwarg which is invoked per-connection.
* refactor(redis): move GCPIAMCredentialProvider to its own file
Extract GCPIAMCredentialProvider and _generate_gcp_iam_access_token
into litellm/_redis_credential_provider.py. _redis.py imports them
from there, keeping the public API unchanged.
* fix: address Greptile review issues
- GCPIAMCredentialProvider now inherits from redis.credentials.CredentialProvider
so redis-py's async path calls get_credentials_async() properly
- move _redis_credential_provider import to top of _redis.py (PEP 8)
- remove dead else-branch that silently no-oped (gcp_service_account from
redis_kwargs.get() was always None since it's popped by _get_redis_client_logic)
- remove mid-function 'from litellm import get_secret_str' inline import
- remove unused 'call' import from test_redis.py
* chore: retrigger CI/review
* chore: sync schema.prisma copies from root
* chore: sync schema.prisma copies from root
* fix(proxy_server): use bounded asyncio.Queue with maxsize to prevent unbounded growth
* fix(a2a/pydantic_ai): make api_base Optional to match base class signature
* fix(a2a/pydantic_ai): make api_base Optional in handler and guard against None
* fix(mcp): remove unused get_all_mcp_servers import
* fix(mcp): remove unused MCPToolset import
* refactor(mcp): extract toolset permission logic to reduce statement count below PLR0915 limit
* fix(tests): update reload_servers_from_database tests to mock prisma directly
---------
Co-authored-by: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
* fix(toolset_db): lazy-import prisma to avoid ImportError when prisma not installed
* fix(tests): update UI tests for toolset tab and updated empty state text
* fix(tests): add get_mcp_server_by_name to fake_manager stub
---------
Co-authored-by: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
2026-04-05 07:23:21 +08:00
// Named collection of {server_id, tool_name} pairs that can be granted to keys/teams
model LiteLLM_MCPToolsetTable {
toolset_id String @id @default(uuid())
toolset_name String @unique
description String?
tools Json @default("[]") // [{server_id: string, tool_name: string}]
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
}
2026-03-07 15:39:08 +08:00
// Per-user BYOK credentials for MCP servers
model LiteLLM_MCPUserCredentials {
id String @id @default(uuid())
user_id String
server_id String
credential_b64 String
created_at DateTime @default(now()) @map("created_at")
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
@@unique([user_id, server_id])
2024-01-06 17:29:10 +08:00
}
2024-01-25 09:15:01 +08:00
// Generate Tokens for Proxy
2024-01-06 17:29:10 +08:00
model LiteLLM_VerificationToken {
2024-03-03 03:59:17 +08:00
token String @id
2024-01-27 12:53:03 +08:00
key_name String?
key_alias String?
2024-03-03 10:34:18 +08:00
soft_budget_cooldown Boolean @default(false) // key-level state on if budget alerts need to be cooled down
2024-01-06 17:29:10 +08:00
spend Float @default(0.0)
expires DateTime?
models String[]
aliases Json @default("{}")
config Json @default("{}")
2026-01-06 08:19:42 +08:00
router_settings Json? @default("{}")
2024-01-06 17:29:10 +08:00
user_id String?
2024-01-19 05:34:51 +08:00
team_id String?
2026-02-26 03:44:30 +08:00
agent_id String?
2025-11-13 06:38:15 +08:00
project_id String?
2024-02-16 13:29:34 +08:00
permissions Json @default("{}")
2024-01-06 17:29:10 +08:00
max_parallel_requests Int?
metadata Json @default("{}")
2024-09-21 10:40:40 +08:00
blocked Boolean?
2024-01-19 09:03:18 +08:00
tpm_limit BigInt?
rpm_limit BigInt?
2025-06-26 13:37:45 +08:00
max_budget Float?
budget_duration String?
2024-01-24 04:33:13 +08:00
budget_reset_at DateTime?
2024-01-31 13:17:01 +08:00
allowed_cache_controls String[] @default([])
2025-04-17 10:21:47 +08:00
allowed_routes String[] @default([])
2026-01-24 05:16:58 +08:00
policies String[] @default([])
2026-02-13 04:30:22 +08:00
access_group_ids String[] @default([])
2024-02-17 07:44:34 +08:00
model_spend Json @default("{}")
model_max_budget Json @default("{}")
2024-03-03 04:25:40 +08:00
budget_id String?
2025-05-27 13:03:18 +08:00
organization_id String?
2025-05-08 12:09:51 +08:00
object_permission_id String?
2024-10-25 11:19:29 +08:00
created_at DateTime? @default(now()) @map("created_at")
2025-02-28 10:12:58 +08:00
created_by String?
2024-10-25 11:19:29 +08:00
updated_at DateTime? @default(now()) @updatedAt @map("updated_at")
2025-02-28 10:12:58 +08:00
updated_by String?
2026-02-19 15:11:58 +08:00
last_active DateTime? // When this key was last used
2025-09-25 12:37:56 +08:00
rotation_count Int? @default(0) // Number of times key has been rotated
auto_rotate Boolean? @default(false) // Whether this key should be auto-rotated
rotation_interval String? // How often to rotate (e.g., "30d", "90d")
last_rotation_at DateTime? // When this key was last rotated
2025-09-27 07:24:40 +08:00
key_rotation_at DateTime? // When this key should next be rotated
2026-04-18 05:47:05 +08:00
budget_limits Json? // per-model budget limits for the key
2024-03-03 10:34:18 +08:00
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
2025-02-11 11:13:32 +08:00
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
2025-11-13 06:38:15 +08:00
litellm_project_table LiteLLM_ProjectTable? @relation(fields: [project_id], references: [project_id])
2025-05-08 12:09:51 +08:00
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
2026-03-07 15:39:08 +08:00
jwt_key_mappings LiteLLM_JWTKeyMapping[]
2026-02-01 07:07:47 +08:00
// SELECT COUNT(*) FROM (SELECT "public"."LiteLLM_VerificationToken"."token" FROM "public"."LiteLLM_VerificationToken" WHERE ("public"."LiteLLM_VerificationToken"."user_id" = $1 AND ("public"."LiteLLM_VerificationToken"."team_id" IS NULL OR "public"."LiteLLM_VerificationToken"."team_id" <> $2)) OFFSET $3 ) AS "sub"
// SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE "public"."LiteLLM_VerificationToken"."user_id" = $1 OFFSET $2
@@index([user_id, team_id])
// SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE "public"."LiteLLM_VerificationToken"."team_id" = $1 OFFSET $2
@@index([team_id])
// SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE (("public"."LiteLLM_VerificationToken"."expires" IS NULL OR "public"."LiteLLM_VerificationToken"."expires" > $1) AND "public"."LiteLLM_VerificationToken"."budget_reset_at" < $2) OFFSET $3
@@index([budget_reset_at, expires])
2024-01-06 17:29:10 +08:00
}
2026-03-07 15:39:08 +08:00
model LiteLLM_JWTKeyMapping {
id String @id @default(uuid())
jwt_claim_name String // e.g. "sub", "email"
jwt_claim_value String // The claim value to match
token String // Hashed virtual key (FK)
description String?
is_active Boolean @default(true)
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token])
@@unique([jwt_claim_name, jwt_claim_value])
@@index([jwt_claim_name, jwt_claim_value, is_active])
}
2026-02-03 12:20:10 +08:00
// Deprecated keys during grace period - allows old key to work until revoke_at
model LiteLLM_DeprecatedVerificationToken {
id String @id @default(uuid())
token String // Hashed old key
active_token_id String // Current token hash in LiteLLM_VerificationToken
revoke_at DateTime // When the old key stops working
created_at DateTime @default(now()) @map("created_at")
@@unique([token])
@@index([token, revoke_at])
@@index([revoke_at])
}
2026-01-17 06:25:23 +08:00
// Audit table for deleted keys - preserves spend and key information for historical tracking
model LiteLLM_DeletedVerificationToken {
id String @id @default(uuid())
token String // Original token (hashed)
key_name String?
key_alias String?
soft_budget_cooldown Boolean @default(false)
spend Float @default(0.0)
expires DateTime?
models String[]
aliases Json @default("{}")
config Json @default("{}")
user_id String?
team_id String?
2026-02-27 12:29:43 +08:00
agent_id String?
2026-02-20 02:50:23 +08:00
project_id String?
2026-01-17 06:25:23 +08:00
permissions Json @default("{}")
max_parallel_requests Int?
metadata Json @default("{}")
blocked Boolean?
tpm_limit BigInt?
rpm_limit BigInt?
max_budget Float?
budget_duration String?
budget_reset_at DateTime?
allowed_cache_controls String[] @default([])
allowed_routes String[] @default([])
2026-01-24 05:16:58 +08:00
policies String[] @default([])
2026-02-13 04:30:22 +08:00
access_group_ids String[] @default([])
2026-01-17 06:25:23 +08:00
model_spend Json @default("{}")
model_max_budget Json @default("{}")
router_settings Json? @default("{}")
budget_id String?
organization_id String?
object_permission_id String?
created_at DateTime? // Original creation timestamp
created_by String? // Original creator
updated_at DateTime? // Last update timestamp before deletion
updated_by String? // Last user who updated before deletion
2026-02-19 15:11:58 +08:00
last_active DateTime? // When this key was last used before deletion
2026-01-17 06:25:23 +08:00
rotation_count Int? @default(0)
auto_rotate Boolean? @default(false)
rotation_interval String?
last_rotation_at DateTime?
key_rotation_at DateTime?
2026-02-15 01:49:11 +08:00
2026-01-17 06:25:23 +08:00
// Deletion metadata
deleted_at DateTime @default(now()) @map("deleted_at")
deleted_by String? @map("deleted_by") // User who deleted the key
deleted_by_api_key String? @map("deleted_by_api_key") // API key hash that performed the deletion
litellm_changed_by String? @map("litellm_changed_by") // From litellm-changed-by header if provided
@@index([token])
@@index([deleted_at])
@@index([user_id])
@@index([team_id])
@@index([organization_id])
@@index([key_alias])
@@index([created_at])
}
2024-03-17 03:26:29 +08:00
model LiteLLM_EndUserTable {
2024-03-17 05:15:01 +08:00
user_id String @id
2024-03-17 03:26:29 +08:00
alias String? // admin-facing alias
spend Float @default(0.0)
2024-05-09 09:50:36 +08:00
allowed_model_region String? // require all user requests to use models in this specific region
default_model String? // use along with 'allowed_model_region'. if no available model in region, default to this model.
2024-03-17 03:26:29 +08:00
budget_id String?
2026-02-19 10:53:59 +08:00
object_permission_id String?
2024-03-17 03:26:29 +08:00
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
2026-02-19 10:53:59 +08:00
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
2024-03-17 03:26:29 +08:00
blocked Boolean @default(false)
}
2025-10-11 10:24:50 +08:00
// Track tags with budgets and spend
model LiteLLM_TagTable {
tag_name String @id
description String?
models String[]
model_info Json? // maps model_id to model_name
spend Float @default(0.0)
budget_id String?
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
created_at DateTime @default(now()) @map("created_at")
created_by String?
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
}
2024-01-25 09:15:01 +08:00
// store proxy config.yaml
2024-01-06 17:29:10 +08:00
model LiteLLM_Config {
param_name String @id
param_value Json?
2024-01-19 02:04:34 +08:00
}
2024-01-25 09:15:01 +08:00
// View spend, model, api_key per request
2024-01-19 02:04:34 +08:00
model LiteLLM_SpendLogs {
2024-03-03 03:59:17 +08:00
request_id String @id
2024-01-19 02:04:34 +08:00
call_type String
2024-07-24 07:33:04 +08:00
api_key String @default ("") // Hashed API Token. Not the actual Virtual Key. Equivalent to 'token' column in LiteLLM_VerificationToken
2024-01-19 05:16:25 +08:00
spend Float @default(0.0)
2024-01-27 05:23:51 +08:00
total_tokens Int @default(0)
prompt_tokens Int @default(0)
completion_tokens Int @default(0)
2024-01-19 02:04:34 +08:00
startTime DateTime // Assuming start_time is a DateTime field
endTime DateTime // Assuming end_time is a DateTime field
2026-02-25 13:04:53 +08:00
request_duration_ms Int?
2024-05-23 07:43:08 +08:00
completionStartTime DateTime? // Assuming completionStartTime is a DateTime field
2024-01-19 02:04:34 +08:00
model String @default("")
2024-05-23 08:29:44 +08:00
model_id String? @default("") // the model id stored in proxy model db
model_group String? @default("") // public model_name / model_group
2024-12-08 05:40:22 +08:00
custom_llm_provider String? @default("") // litellm used custom_llm_provider
2024-07-24 07:33:04 +08:00
api_base String? @default("")
user String? @default("")
2025-11-13 06:38:15 +08:00
metadata Json? @default("{}") // project_id stored here
2024-07-24 07:33:04 +08:00
cache_hit String? @default("")
cache_key String? @default("")
request_tags Json? @default("[]")
2025-06-26 13:37:45 +08:00
team_id String?
2025-11-23 04:54:49 +08:00
organization_id String?
2024-03-01 11:21:57 +08:00
end_user String?
2024-07-09 01:16:58 +08:00
requester_ip_address String?
2025-01-18 10:53:45 +08:00
messages Json? @default("{}")
response Json? @default("{}")
2025-04-26 14:24:24 +08:00
session_id String?
2025-05-09 02:29:25 +08:00
status String?
2025-07-09 13:08:16 +08:00
mcp_namespaced_tool_name String?
2025-12-11 08:09:56 +08:00
agent_id String?
2025-04-26 14:24:24 +08:00
proxy_server_request Json? @default("{}")
2024-09-13 04:39:50 +08:00
@@index([startTime])
[Release Fix] (#22411)
* fix(lint): suppress PLR0915 for 3 complex methods that exceed 50-statement limit
- streaming_iterator.py: _process_event (84 statements)
- transformation.py: translate_messages_to_responses_input (51 statements)
- transformation.py: transform_realtime_response (54 statements)
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix(mypy): resolve type errors in public_endpoints, user_api_key_auth, common_utils, transformation
- public_endpoints.py: fix _cached_endpoints type annotation
- user_api_key_auth.py: accept Optional[str] for end_user_id parameter
- common_utils.py: add NewProjectRequest/UpdateProjectRequest to Union type
- transformation.py: add ChatCompletionRedactedThinkingBlock and list[Any] to content type
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix(proxy-extras): bump version to 0.4.50 and sync schema
- Bump litellm-proxy-extras from 0.4.49 to 0.4.50
- Sync schema.prisma with main proxy schema
- Includes new LiteLLM_ClaudeCodePluginTable model
- Includes new @@index([startTime, request_id]) on SpendLogs
- Update version references in requirements.txt and pyproject.toml
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix(router): use string id in test_add_deployment and add defensive str() in register_model
- Change test to use string '100' instead of int 100 for model_info.id
- Add str() conversion in register_model to prevent AttributeError on non-string keys
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix(security): update minimatch to 10.2.4 to fix CVE-2026-27903 and CVE-2026-27904
- Run npm audit fix in docs/my-website
- Updates minimatch from 10.2.1 to 10.2.4 (fixes HIGH severity ReDoS vulnerabilities)
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix(test): update realtime guardrail test assertions to match actual guardrail behavior
- test_text_message_blocked_by_guardrail_no_ai_response: allow guardrail's own block
message text in response.done (previously expected empty content)
- test_voice_transcript_blocked_by_guardrail: allow guardrail to send response.cancel
+ block message + response.create flow (previously expected no response.create)
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix: revert proxy-extras version in requirements.txt and pyproject.toml
The litellm-proxy-extras 0.4.50 is not published to PyPI yet, so consumer
references must stay at 0.4.49. Only the source package pyproject.toml
should be bumped to 0.4.50 for the publish_proxy_extras CI job.
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix: make transcript delta check optional in voice guardrail test
The guardrail sends an error event (guardrail_violation) when blocking
voice transcripts; it does not always produce transcript deltas. Remove
the assertion requiring response.audio_transcript.delta since the error
event is the primary signal that blocked content was handled.
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* Add missing env keys to documentation: LITELLM_MAX_STREAMING_DURATION_SECONDS and LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES
These two environment variables were used in code but not documented in the
environment variables reference section of config_settings.md, causing the
test_env_keys.py CI test to fail.
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* Fix 13 mypy type errors across 6 files
- in_flight_requests_middleware.py: Fix type: ignore error codes from
[union-attr] to [attr-defined], add [arg-type] for Gauge **kwargs
- transformation.py: Add [assignment] ignore for output_format reassignment,
add fallback empty string for tool use id to fix arg-type
- responses/main.py: Remove redundant type annotation on second
secret_fields assignment to fix no-redef
- streaming_iterator.py: Add [assignment] ignores for intermediate
cache token assignments
- handler.py: Add [typeddict-item] ignore for AnthropicMessagesRequest
construction from dict
- public_endpoints.py: Add [arg-type] ignore for _load_endpoints()
return type mismatch with SupportedEndpoint model
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix: add auth overrides to spend tracking tests, fix realtime guardrail assertion, update UI minimatch
- Add app.dependency_overrides for user_api_key_auth in 4 spend tracking tests
that were returning 401 Unauthorized (error_code, error_message,
error_code_and_key_alias, key_hash)
- Fix realtime guardrail test to check ANY error event for guardrail_violation
instead of just the first (OpenAI may send its own errors first)
- Update ui/litellm-dashboard/package-lock.json to fix minimatch vulnerability
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* Fix failing MCP e2e and create_mcp_server UI tests
Test 1 (test_independent_clients_no_shared_session):
- Add allow_all_keys: true to MCP servers in test config. With master_key
and no DB, get_allowed_mcp_servers returned empty, causing 0 tools and
403 on tool calls. allow_all_keys bypasses per-key restrictions.
- Add asyncio.sleep(0.5) between client connections to allow MCP SDK
TaskGroup cleanup and avoid ExceptionGroup on connection close (MCP #915).
Test 2 (create_mcp_server 'auth value is provided'):
- Use userEvent.setup({ delay: null }) for instant keystrokes to avoid
timeout from default typing delay on CI.
- Increase per-test timeout to 15000ms for CI environments.
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix: stabilize proxy unit tests for parallel execution
- test_response_polling_handler: add xdist_group to prevent heavy import OOM
- test_db_schema_migration: use temp dir for worker isolation, sync schema.prisma index
- test_custom_tokenizer_bug: use lighter tokenizer to prevent OOM in parallel
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix: add auth overrides to more spend tracking and model info tests
- Fix test_ui_view_spend_logs_pagination missing auth override (401)
- Fix test_view_spend_tags missing auth override (401)
- Fix test_view_spend_tags_no_database missing auth override (401)
- Fix test_empty_model_list.py to use app.dependency_overrides instead of patch()
for FastAPI dependency injection auth
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix(test): use patch.object for aiohttp transport test to work in parallel execution
The @patch decorator was not intercepting the static method call in parallel
xdist workers. Using patch.object on the directly-imported class is more reliable.
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix(security): update minimatch from 10.2.1 to 10.2.4 in Dockerfile
The Docker image was explicitly pinning minimatch@10.2.1 which has HIGH
severity ReDoS vulnerabilities (GHSA-7r86-cg39-jmmj, GHSA-23c5-xmqv-rm74).
Update to 10.2.4 which includes fixes for both CVEs.
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix(ui): prevent MCP and TeamInfo test timeouts on CI
- Add userEvent.setup({ delay: null }) to all tests using userEvent in both files
- Add timeout: 15000 to tests with significant user interaction (typing, multiple clicks)
- Fixes: create_mcp_server Bearer Token test, TeamInfo cancel button test
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix: stabilize parallel test execution and aiohttp transport test
- test_aiohttp_handler: rewrite transport test to not rely on static method mock
(consistently fails in parallel xdist workers)
- test_proxy_cli: add xdist_group to prevent timeout during heavy imports
- test_swagger_chat_completions: add xdist_group to prevent timeout
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix(security): add serialize-javascript override to fix GHSA-5c6j-r48x-rmvq
Add npm override for serialize-javascript>=7.0.3 in docs/my-website
to fix HIGH severity RCE vulnerability via RegExp.flags.
Also bump minimatch override to >=10.2.4.
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* Fix flaky tests: remove broken Vertex model, add retries for Anthropic
- Remove vertex_ai/meta/llama-4-scout-17b-16e-instruct-maas from
test_partner_models_httpx_streaming - consistently returns 400 BadRequest
- Add @pytest.mark.flaky(retries=6, delay=10) to test_function_call_parsing
for transient Anthropic API overload errors
- Add @pytest.mark.flaky(retries=6, delay=10) to test_openai_stream_options_call
for transient Anthropic InternalServerError
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix(ci): add xdist_group(proxy_heavy) to prevent OOM in parallel proxy tests
- Add pytestmark = pytest.mark.xdist_group('proxy_heavy') to test_proxy_utils.py
- Change test_db_schema_migration.py from schema_migration to proxy_heavy group
- Add @pytest.mark.xdist_group('proxy_heavy') to test_proxy_server.py::test_health
Groups heavy proxy tests to run on same worker, avoiding worker OOM crashes.
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* Fix vertex AI qwen global endpoint test to mock vertexai module import
The test_vertex_ai_qwen_global_endpoint_url test was failing because the
VertexAIPartnerModels.completion() method tries to 'import vertexai' before
any of the mocked code runs. In environments without google-cloud-aiplatform
installed, this import fails with a VertexAIError(status_code=400).
Fix by:
- Adding patch.dict('sys.modules', {'vertexai': MagicMock()}) to mock the
vertexai module import
- Adding vertex_ai_location parameter to the acompletion call for completeness
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix(ci): add xdist_group to health endpoint and watsonx tests for parallel stability
- test_health_liveliness_endpoint: add xdist_group('proxy_health') to prevent timeout
- test_watsonx_gpt_oss tests: add xdist_group('watsonx_heavy') to prevent mock interference
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix(test): pre-populate WatsonX IAM token cache to prevent parallel test interference
The watsonx prompt transformation test was failing in parallel execution because
litellm.module_level_client.post mock was being interfered with by other tests.
Pre-populating the IAM token cache avoids the HTTP call entirely.
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix(test): add spend data polling with retries for e2e pass-through tests
- test_vertex_with_spend.test.js: Replace 15s fixed wait with polling loop
(up to 6 attempts, 10s apart) for spend data to appear in DB
- Increase test timeout from 25s to 90s to accommodate polling
- base_anthropic_messages_tool_search_test.py: Add flaky(retries=3) for
streaming test that depends on live Anthropic API
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix(ci): reduce parallel workers from 8 to 4 for proxy tests to prevent OOM
- litellm_proxy_unit_testing_part2: -n 8 -> -n 4
- litellm_mapped_tests_proxy_part2: -n 8 -> -n 4, timeout 60 -> 120
- Worker crashes consistently caused by too many parallel proxy tests
each loading the full FastAPI app and heavy dependency tree
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix(db): add migration for SpendLogs composite index (startTime, request_id)
The @@index([startTime, request_id]) was added to schema.prisma but had no
corresponding migration. This caused test_aaaasschema_migration_check to fail
because prisma migrate diff detected the missing index.
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix(db): add migration for MCP available_on_public_internet default change to true
The schema.prisma changed the default for available_on_public_internet from
false to true, but no migration was created. This caused the schema migration
test to detect drift.
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix(test): increase server wait time and add retry to flaky external API tests
- test_basic_python_version.py: increase server startup wait from 60s to 90s
for slower CI environments (fixes installing_litellm_on_python_3_13)
- test_a2a_agent.py: add flaky(retries=3, delay=5) for non-streaming test
that depends on live A2A agent endpoint
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix(test): add flaky retries to all intermittent external API tests for 0-fail CI
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
* fix(test): add auth overrides to file endpoint tests that return 500
The test_target_storage tests were getting 500 because the FastAPI auth
dependency wasn't overridden. Added app.dependency_overrides for proper
auth bypass in test environment.
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
---------
Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
2026-03-01 01:46:35 +08:00
@@index([startTime, request_id])
2024-09-13 04:39:50 +08:00
@@index([end_user])
2025-05-11 05:28:20 +08:00
@@index([session_id])
2024-02-20 08:35:20 +08:00
}
2024-05-01 02:42:17 +08:00
// View spend, model, api_key per request
model LiteLLM_ErrorLogs {
request_id String @id @default(uuid())
2024-05-01 04:34:14 +08:00
startTime DateTime // Assuming start_time is a DateTime field
endTime DateTime // Assuming end_time is a DateTime field
2025-06-26 13:37:45 +08:00
api_base String @default("")
2024-05-01 04:11:09 +08:00
model_group String @default("") // public model_name / model_group
2024-05-01 08:31:40 +08:00
litellm_model_name String @default("") // model passed to litellm
2024-05-01 02:42:17 +08:00
model_id String @default("") // ID of model in ProxyModelTable
request_kwargs Json @default("{}")
2024-05-01 03:31:19 +08:00
exception_type String @default("")
exception_string String @default("")
status_code String @default("")
2024-05-01 02:42:17 +08:00
}
2024-02-20 08:35:20 +08:00
// Beta - allow team members to request access to a model
2024-02-20 08:53:40 +08:00
model LiteLLM_UserNotifications {
2024-03-03 03:59:17 +08:00
request_id String @id
2025-06-26 13:37:45 +08:00
user_id String
2024-02-20 08:35:20 +08:00
models String[]
justification String
2024-02-20 08:53:40 +08:00
status String // approved, disapproved, pending
2024-05-23 08:16:02 +08:00
}
model LiteLLM_TeamMembership {
// Use this table to track the Internal User's Spend within a Team + Set Budgets, rpm limits for the user within the team
user_id String
team_id String
spend Float @default(0.0)
2026-04-22 04:56:44 +08:00
total_spend Float @default(0.0)
2024-05-23 08:16:02 +08:00
budget_id String?
2025-06-26 13:37:45 +08:00
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
2024-05-23 08:16:02 +08:00
@@id([user_id, team_id])
}
2024-05-28 11:32:25 +08:00
2024-10-09 17:48:18 +08:00
model LiteLLM_OrganizationMembership {
// Use this table to track Internal User and Organization membership. Helps tracking a users role within an Organization
2024-10-09 17:55:27 +08:00
user_id String
organization_id String
2024-10-09 17:48:18 +08:00
user_role String?
spend Float? @default(0.0)
budget_id String?
created_at DateTime? @default(now()) @map("created_at")
updated_at DateTime? @default(now()) @updatedAt @map("updated_at")
// relations
user LiteLLM_UserTable @relation(fields: [user_id], references: [user_id])
2025-01-05 09:31:24 +08:00
organization LiteLLM_OrganizationTable @relation("OrganizationToMembership", fields: [organization_id], references: [organization_id])
2024-10-09 17:48:18 +08:00
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
2025-06-26 13:37:45 +08:00
2025-01-05 09:31:24 +08:00
2024-10-09 17:48:18 +08:00
@@id([user_id, organization_id])
@@unique([user_id, organization_id])
}
2024-05-28 11:32:25 +08:00
model LiteLLM_InvitationLink {
// use this table to track invite links sent by admin for people to join the proxy
id String @id @default(uuid())
user_id String
is_accepted Boolean @default(false)
accepted_at DateTime? // when link is claimed (user successfully onboards via link)
expires_at DateTime // till when is link valid
created_at DateTime // when did admin create the link
created_by String // who created the link
updated_at DateTime // when was invite status updated
updated_by String // who updated the status (admin/user who accepted invite)
// Relations
liteLLM_user_table_user LiteLLM_UserTable @relation("UserId", fields: [user_id], references: [user_id])
liteLLM_user_table_created LiteLLM_UserTable @relation("CreatedBy", fields: [created_by], references: [user_id])
liteLLM_user_table_updated LiteLLM_UserTable @relation("UpdatedBy", fields: [updated_by], references: [user_id])
2024-06-06 07:19:38 +08:00
}
2024-06-06 08:50:27 +08:00
model LiteLLM_AuditLog {
2024-06-09 07:02:18 +08:00
id String @id @default(uuid())
updated_at DateTime @default(now())
changed_by String @default("") // user or system that performed the action
changed_by_api_key String @default("") // api key hash that performed the action
action String // create, update, delete
table_name String // on of LitellmTableNames.TEAM_TABLE_NAME, LitellmTableNames.USER_TABLE_NAME, LitellmTableNames.PROXY_MODEL_TABLE_NAME,
object_id String // id of the object being audited. This can be the key id, team id, user id, model id
2025-06-26 13:37:45 +08:00
before_value Json? // value of the row
2024-06-09 07:02:18 +08:00
updated_values Json? // value of the row after change
2024-07-09 01:16:58 +08:00
}
2025-03-27 07:36:36 +08:00
// Track daily user spend metrics per model and key
model LiteLLM_DailyUserSpend {
id String @id @default(uuid())
2025-06-26 13:37:45 +08:00
user_id String?
2025-03-27 07:36:36 +08:00
date String
2025-06-26 13:37:45 +08:00
api_key String
2025-07-09 13:08:16 +08:00
model String?
2025-06-26 13:37:45 +08:00
model_group String?
2025-07-09 13:08:16 +08:00
custom_llm_provider String?
mcp_namespaced_tool_name String?
2026-01-07 07:54:03 +08:00
endpoint String?
2025-05-10 05:14:39 +08:00
prompt_tokens BigInt @default(0)
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
2025-03-27 07:36:36 +08:00
spend Float @default(0.0)
2025-05-10 05:14:39 +08:00
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
failed_requests BigInt @default(0)
2025-03-27 07:36:36 +08:00
created_at DateTime @default(now())
updated_at DateTime @updatedAt
2026-01-07 07:54:03 +08:00
@@unique([user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
2025-03-27 07:36:36 +08:00
@@index([date])
2026-02-20 06:36:28 +08:00
@@index([user_id, date])
2025-03-27 07:36:36 +08:00
@@index([api_key])
@@index([model])
2025-07-09 13:08:16 +08:00
@@index([mcp_namespaced_tool_name])
2026-01-07 07:54:03 +08:00
@@index([endpoint])
2025-03-27 07:36:36 +08:00
}
2025-03-28 12:10:26 +08:00
2025-11-13 10:22:26 +08:00
// Track daily organization spend metrics per model and key
model LiteLLM_DailyOrganizationSpend {
id String @id @default(uuid())
organization_id String?
date String
api_key String
model String?
model_group String?
custom_llm_provider String?
mcp_namespaced_tool_name String?
2026-01-07 07:54:03 +08:00
endpoint String?
2025-11-13 10:22:26 +08:00
prompt_tokens BigInt @default(0)
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
failed_requests BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
2026-01-07 07:54:03 +08:00
@@unique([organization_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
2025-11-13 10:22:26 +08:00
@@index([date])
2026-02-20 06:36:28 +08:00
@@index([organization_id, date])
2025-11-13 10:22:26 +08:00
@@index([api_key])
@@index([model])
@@index([mcp_namespaced_tool_name])
2026-01-07 07:54:03 +08:00
@@index([endpoint])
2025-12-05 04:30:08 +08:00
}
// Track daily end user (customer) spend metrics per model and key
model LiteLLM_DailyEndUserSpend {
id String @id @default(uuid())
end_user_id String?
date String
api_key String
model String?
model_group String?
custom_llm_provider String?
mcp_namespaced_tool_name String?
2026-01-07 07:54:03 +08:00
endpoint String?
2025-12-05 04:30:08 +08:00
prompt_tokens BigInt @default(0)
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
failed_requests BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
2026-01-07 07:54:03 +08:00
@@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
2025-12-05 04:30:08 +08:00
@@index([date])
2026-02-20 06:36:28 +08:00
@@index([end_user_id, date])
2025-12-05 04:30:08 +08:00
@@index([api_key])
@@index([model])
@@index([mcp_namespaced_tool_name])
2026-01-07 07:54:03 +08:00
@@index([endpoint])
2025-11-13 10:22:26 +08:00
}
2025-12-11 03:50:52 +08:00
// Track daily agent spend metrics per model and key
model LiteLLM_DailyAgentSpend {
id String @id @default(uuid())
agent_id String?
date String
api_key String
model String?
model_group String?
custom_llm_provider String?
mcp_namespaced_tool_name String?
2026-01-07 07:54:03 +08:00
endpoint String?
2025-12-11 03:50:52 +08:00
prompt_tokens BigInt @default(0)
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
failed_requests BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
2026-01-07 07:54:03 +08:00
@@unique([agent_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
2025-12-11 03:50:52 +08:00
@@index([date])
2026-02-20 06:36:28 +08:00
@@index([agent_id, date])
2025-12-11 03:50:52 +08:00
@@index([api_key])
@@index([model])
@@index([mcp_namespaced_tool_name])
2026-01-07 07:54:03 +08:00
@@index([endpoint])
2025-12-11 03:50:52 +08:00
}
2025-04-16 11:58:48 +08:00
// Track daily team spend metrics per model and key
model LiteLLM_DailyTeamSpend {
id String @id @default(uuid())
2025-05-27 13:03:18 +08:00
team_id String?
2025-04-16 11:58:48 +08:00
date String
2025-06-26 13:37:45 +08:00
api_key String
2025-07-09 13:08:16 +08:00
model String?
2025-06-26 13:37:45 +08:00
model_group String?
2025-07-09 13:08:16 +08:00
custom_llm_provider String?
mcp_namespaced_tool_name String?
2026-01-07 07:54:03 +08:00
endpoint String?
2025-05-10 05:14:39 +08:00
prompt_tokens BigInt @default(0)
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
2025-04-16 11:58:48 +08:00
spend Float @default(0.0)
2025-05-10 05:14:39 +08:00
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
failed_requests BigInt @default(0)
2025-04-16 11:58:48 +08:00
created_at DateTime @default(now())
updated_at DateTime @updatedAt
2026-01-07 07:54:03 +08:00
@@unique([team_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
2025-04-16 11:58:48 +08:00
@@index([date])
2026-02-20 06:36:28 +08:00
@@index([team_id, date])
2025-04-16 11:58:48 +08:00
@@index([api_key])
@@index([model])
2025-07-09 13:08:16 +08:00
@@index([mcp_namespaced_tool_name])
2026-01-07 07:54:03 +08:00
@@index([endpoint])
2025-04-16 11:58:48 +08:00
}
2025-04-17 03:26:21 +08:00
// Track daily team spend metrics per model and key
model LiteLLM_DailyTagSpend {
id String @id @default(uuid())
2025-11-12 10:53:48 +08:00
request_id String?
2025-06-26 13:37:45 +08:00
tag String?
2025-04-17 03:26:21 +08:00
date String
2025-06-26 13:37:45 +08:00
api_key String
2025-07-09 13:08:16 +08:00
model String?
2025-06-26 13:37:45 +08:00
model_group String?
2025-07-09 13:08:16 +08:00
custom_llm_provider String?
mcp_namespaced_tool_name String?
2026-01-07 07:54:03 +08:00
endpoint String?
2025-05-10 05:14:39 +08:00
prompt_tokens BigInt @default(0)
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
2025-04-17 03:26:21 +08:00
spend Float @default(0.0)
2025-05-10 05:14:39 +08:00
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
failed_requests BigInt @default(0)
2025-04-17 03:26:21 +08:00
created_at DateTime @default(now())
updated_at DateTime @updatedAt
2026-01-07 07:54:03 +08:00
@@unique([tag, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
2025-04-17 03:26:21 +08:00
@@index([date])
2026-02-20 06:36:28 +08:00
@@index([tag, date])
2025-04-17 03:26:21 +08:00
@@index([api_key])
@@index([model])
2025-07-09 13:08:16 +08:00
@@index([mcp_namespaced_tool_name])
2026-01-07 07:54:03 +08:00
@@index([endpoint])
2025-04-17 03:26:21 +08:00
}
2025-03-28 12:10:26 +08:00
// Track the status of cron jobs running. Only allow one pod to run the job at a time
2025-03-28 14:13:01 +08:00
model LiteLLM_CronJob {
2025-03-28 13:54:46 +08:00
cronjob_id String @id @default(cuid()) // Unique ID for the record
pod_id String // Unique identifier for the pod acting as the leader
2025-03-28 12:10:26 +08:00
status JobStatus @default(INACTIVE) // Status of the cron job (active or inactive)
2025-03-28 13:54:46 +08:00
last_updated DateTime @default(now()) // Timestamp for the last update of the cron job record
2025-03-28 12:10:26 +08:00
ttl DateTime // Time when the leader's lease expires
}
enum JobStatus {
ACTIVE
INACTIVE
}
2025-04-01 13:48:43 +08:00
2025-04-12 23:24:46 +08:00
model LiteLLM_ManagedFileTable {
id String @id @default(uuid())
unified_file_id String @unique // The base64 encoded unified file ID
2025-06-26 13:37:45 +08:00
file_object Json? // Stores the OpenAIFileObject
2025-06-04 06:57:33 +08:00
model_mappings Json
2025-05-23 14:05:45 +08:00
flat_model_file_ids String[] @default([]) // Flat list of model file id's - for faster querying of model id -> unified file id
2025-12-11 17:38:17 +08:00
storage_backend String? // Storage backend name (e.g., "azure_storage", "gcs", "default")
storage_url String? // The actual storage URL where the file is stored
2025-04-12 23:24:46 +08:00
created_at DateTime @default(now())
2025-06-26 13:37:45 +08:00
created_by String?
2025-04-12 23:24:46 +08:00
updated_at DateTime @updatedAt
2025-05-23 14:05:45 +08:00
updated_by String?
2025-04-12 23:24:46 +08:00
@@index([unified_file_id])
}
2025-06-26 13:37:45 +08:00
model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the
2025-05-23 14:05:45 +08:00
id String @id @default(uuid())
2025-05-27 13:03:18 +08:00
unified_object_id String @unique // The base64 encoded unified file ID
2025-06-26 13:37:45 +08:00
model_object_id String @unique // the id returned by the backend API provider
2025-05-23 14:05:45 +08:00
file_object Json // Stores the OpenAIFileObject
2025-06-04 06:57:33 +08:00
file_purpose String // either 'batch' or 'fine-tune'
2025-06-26 13:37:45 +08:00
status String? // check if batch cost has been tracked
2026-02-23 20:12:32 +08:00
batch_processed Boolean @default(false) // set to true by CheckBatchCost after cost is computed
2025-05-23 14:05:45 +08:00
created_at DateTime @default(now())
created_by String?
updated_at DateTime @updatedAt
2025-06-26 13:37:45 +08:00
updated_by String?
2025-05-23 14:05:45 +08:00
@@index([unified_object_id])
@@index([model_object_id])
}
2025-05-01 12:49:59 +08:00
2026-02-15 01:49:11 +08:00
model LiteLLM_ManagedVectorStoreTable {
id String @id @default(uuid())
unified_resource_id String @unique // The base64 encoded unified vector store ID
resource_object Json? // Stores the VectorStoreCreateResponse
model_mappings Json // Maps model_id -> provider_vector_store_id
flat_model_resource_ids String[] @default([]) // Flat list of provider vector store IDs for faster querying
storage_backend String? // Storage backend name (if applicable)
storage_url String? // Storage URL (if applicable)
created_at DateTime @default(now())
created_by String?
updated_at DateTime @updatedAt
updated_by String?
@@index([unified_resource_id])
}
2025-05-01 12:49:59 +08:00
model LiteLLM_ManagedVectorStoresTable {
vector_store_id String @id
custom_llm_provider String
vector_store_name String?
vector_store_description String?
vector_store_metadata Json?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
litellm_credential_name String?
2025-07-18 23:41:18 +08:00
litellm_params Json?
2026-01-29 10:55:40 +08:00
team_id String?
user_id String?
@@index([team_id])
@@index([user_id])
2025-05-15 05:19:51 +08:00
}
// Guardrails table for storing guardrail configurations
model LiteLLM_GuardrailsTable {
guardrail_id String @id @default(uuid())
guardrail_name String @unique
litellm_params Json
guardrail_info Json?
2026-02-15 01:49:11 +08:00
team_id String?
2025-05-15 05:19:51 +08:00
created_at DateTime @default(now())
updated_at DateTime @updatedAt
2026-03-03 14:06:49 +08:00
// Submission lifecycle. Possible values: pending_review (team-registered, awaiting approval), active (approved), rejected
status String @default("active")
submitted_at DateTime?
reviewed_at DateTime?
// submitted_by_user_id and submitted_by_email live in guardrail_info JSON
@@index([status])
2025-06-18 23:37:40 +08:00
}
2026-02-22 11:14:04 +08:00
// Daily guardrail metrics for usage dashboard (one row per guardrail per day)
model LiteLLM_DailyGuardrailMetrics {
guardrail_id String // logical id; may not FK if guardrail from config
date String // YYYY-MM-DD
requests_evaluated BigInt @default(0)
passed_count BigInt @default(0)
blocked_count BigInt @default(0)
flagged_count BigInt @default(0)
avg_score Float?
avg_latency_ms Float?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@id([guardrail_id, date])
@@index([date])
@@index([guardrail_id])
}
// Daily policy metrics for usage dashboard (one row per policy per day)
model LiteLLM_DailyPolicyMetrics {
policy_id String
date String // YYYY-MM-DD
requests_evaluated BigInt @default(0)
passed_count BigInt @default(0)
blocked_count BigInt @default(0)
flagged_count BigInt @default(0)
avg_score Float?
avg_latency_ms Float?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@id([policy_id, date])
@@index([date])
@@index([policy_id])
}
// Index for fast "last N logs for guardrail/policy" from SpendLogs
model LiteLLM_SpendLogGuardrailIndex {
request_id String
guardrail_id String
policy_id String? // set when run as part of a policy pipeline
start_time DateTime
@@id([request_id, guardrail_id])
@@index([guardrail_id, start_time])
@@index([policy_id, start_time])
}
2026-03-04 12:22:20 +08:00
// Index for fast "last N logs for tool" from SpendLogs – see how a tool is called in production
model LiteLLM_SpendLogToolIndex {
request_id String
tool_name String // matches LiteLLM_ToolTable.tool_name; join for input_policy/output_policy etc.
start_time DateTime
@@id([request_id, tool_name])
@@index([tool_name, start_time])
}
2025-08-03 13:33:37 +08:00
// Prompt table for storing prompt configurations
model LiteLLM_PromptTable {
id String @id @default(uuid())
2025-11-20 05:19:56 +08:00
prompt_id String
version Int @default(1)
Litellm ishaan april1 try2 (#25110)
* Litellm ishaan april1 (#25103)
* fix(proxy): enforce upperbound key params on key/update and add custom_key_update hook
The /key/update endpoint did not enforce upperbound_key_generate_params,
allowing users to bypass configured limits (tpm_limit, rpm_limit,
max_budget, duration, budget_duration) by updating an existing key
instead of generating a new one.
Extract the upperbound enforcement logic from _common_key_generation_helper()
into a standalone _enforce_upperbound_key_params() function and call it from
both the generate and update paths. For updates, None values are skipped
(not filled with defaults) since they mean "don't change this field".
Also adds a custom_key_update config option and user_custom_key_update global,
mirroring the existing custom_key_generate pattern, so custom key validation
logic can fire during key updates as well.
* fix(proxy): invoke custom_key_update hook in bulk update path
The user_custom_key_update hook was only called in update_key_fn
(single key update) but not in _process_single_key_update (bulk
update path), allowing custom validation to be bypassed via the
/key/update/bulk endpoint. Mirror the hook invocation in both paths.
* fix(proxy): pass UpdateKeyRequest to hook in bulk path, not BulkUpdateKeyRequestItem
Move the custom_key_update hook invocation to after UpdateKeyRequest
is constructed so the hook receives the same type in both single and
bulk update paths. Previously the bulk path passed
BulkUpdateKeyRequestItem (5 fields only), which would cause
AttributeError for hooks accessing fields like tpm_limit or models.
* fix(bedrock): promote cache usage to message_delta for Claude Code (#24850)
Ensure Bedrock/Anthropic-compatible streaming exposes cache usage where Claude Code reads it by promoting message_stop usage onto message_delta and preserving usage fields in fake-streamed message_delta events.
Made-with: Cursor
* fix(search): Support self-hosted Firecrawl response format in search transform (#24866)
The `transform_search_response` method only handled Firecrawl Cloud (v2)
response format where `data` is a dict with `web`/`news` keys. Self-hosted
Firecrawl (v1) returns `data` as a flat list of result objects, causing an
`AttributeError: 'list' object has no attribute 'get'`.
Detect the response format by checking if `data` is a list (self-hosted)
or dict (cloud) and handle both cases.
Cloud format: {"data": {"web": [...], "news": [...]}}
Self-hosted: {"success": true, "data": [{"url": "...", "title": "...", ...}]}
Co-authored-by: Synergy <synergyoclaw@gmail.com>
* feat: add environment and user tracking to prompt management (#24855)
* feat: add environment and user tracking to prompt management
- Add environment (development/staging/production) and created_by columns to LiteLLM_PromptTable
- Update unique constraint to [prompt_id, version, environment]
- All CRUD endpoints support environment filtering and user tracking
- Redesigned prompt detail page with environment tabs and version history
- UI: environment filter on list page, environment selector in editor
- 8 new tests for environment and user tracking
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* fix: Black formatting and add environments to PromptInfoResponse TypeScript type
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* fix: address Greptile review findings
- P1: delete_prompt scopes in-memory cleanup to environment when provided
- P2: dotprompt_content parsed directly regardless of environment flag
- P2: use distinct for environments query
- P2: fix double-fetch on initial mount in prompt_info.tsx
- fix: remove unsupported select kwarg from find_many
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* fix: address remaining Greptile review comments
- Remove unused useCallback import (index.tsx)
- Remove unused ENV_COLORS variable (prompt_info.tsx)
- P1: in-memory fallback in get_prompt_versions now respects environment filter
- P1: reset selectedEnv when promptId changes to avoid stale state
- Cyclic imports are pre-existing pattern, not introduced by this PR
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* fix: scope patch_prompt to environment using primary key
- Add environment query param to patch_prompt endpoint
- Look up target row by composite key (prompt_id + version + environment)
- Update by primary key (id) to target exactly one row
- Fixes Greptile finding: patch with multiple environments no longer ambiguous
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---------
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* fix: use actual start_time for failed request spend logs (#24906)
async_post_call_failure_hook set both start_time and end_time to
datetime.now(), making all failed requests show duration=0. Use the
actual start_time from litellm_logging_obj instead, so spend logs
reflect the real request duration on timeout and other failures.
Fixes #24888
* feat(bedrock): add nova canvas image edit support (#24869)
* feat(bedrock): add nova canvas image edit support
* fix(bedrock): support PathLike inputs for nova image edit
* chore: sync schema.prisma copies from root
* fix(mypy): correct type-ignore code for delta_usage arg-type
* fix(mypy): cast status_code to str, suppress intentional str yield
* fix(lint): extract _create_content_block_chunks to fix PLR0915
* fix(lint): extract helpers to fix PLR0915 in prompt endpoints
---------
Co-authored-by: michelligabriele <gabriele.michelli@icloud.com>
Co-authored-by: Sameer Kankute <sameer@berri.ai>
Co-authored-by: redhelix <amin.lalji@gmail.com>
Co-authored-by: Synergy <synergyoclaw@gmail.com>
Co-authored-by: Talha Anwar <37379131+talhaanwarch@users.noreply.github.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: madhu19991 <madhu@thunkai.com>
Co-authored-by: Srikanth @adobe <devarakondasrikanth@users.noreply.github.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
* fix(test): update model armor streaming test to handle string or int error code
---------
Co-authored-by: michelligabriele <gabriele.michelli@icloud.com>
Co-authored-by: Sameer Kankute <sameer@berri.ai>
Co-authored-by: redhelix <amin.lalji@gmail.com>
Co-authored-by: Synergy <synergyoclaw@gmail.com>
Co-authored-by: Talha Anwar <37379131+talhaanwarch@users.noreply.github.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: madhu19991 <madhu@thunkai.com>
Co-authored-by: Srikanth @adobe <devarakondasrikanth@users.noreply.github.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
2026-04-04 05:57:44 +08:00
environment String @default("development")
created_by String?
2025-08-03 13:33:37 +08:00
litellm_params Json
prompt_info Json?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
2025-11-20 05:19:56 +08:00
Litellm ishaan april1 try2 (#25110)
* Litellm ishaan april1 (#25103)
* fix(proxy): enforce upperbound key params on key/update and add custom_key_update hook
The /key/update endpoint did not enforce upperbound_key_generate_params,
allowing users to bypass configured limits (tpm_limit, rpm_limit,
max_budget, duration, budget_duration) by updating an existing key
instead of generating a new one.
Extract the upperbound enforcement logic from _common_key_generation_helper()
into a standalone _enforce_upperbound_key_params() function and call it from
both the generate and update paths. For updates, None values are skipped
(not filled with defaults) since they mean "don't change this field".
Also adds a custom_key_update config option and user_custom_key_update global,
mirroring the existing custom_key_generate pattern, so custom key validation
logic can fire during key updates as well.
* fix(proxy): invoke custom_key_update hook in bulk update path
The user_custom_key_update hook was only called in update_key_fn
(single key update) but not in _process_single_key_update (bulk
update path), allowing custom validation to be bypassed via the
/key/update/bulk endpoint. Mirror the hook invocation in both paths.
* fix(proxy): pass UpdateKeyRequest to hook in bulk path, not BulkUpdateKeyRequestItem
Move the custom_key_update hook invocation to after UpdateKeyRequest
is constructed so the hook receives the same type in both single and
bulk update paths. Previously the bulk path passed
BulkUpdateKeyRequestItem (5 fields only), which would cause
AttributeError for hooks accessing fields like tpm_limit or models.
* fix(bedrock): promote cache usage to message_delta for Claude Code (#24850)
Ensure Bedrock/Anthropic-compatible streaming exposes cache usage where Claude Code reads it by promoting message_stop usage onto message_delta and preserving usage fields in fake-streamed message_delta events.
Made-with: Cursor
* fix(search): Support self-hosted Firecrawl response format in search transform (#24866)
The `transform_search_response` method only handled Firecrawl Cloud (v2)
response format where `data` is a dict with `web`/`news` keys. Self-hosted
Firecrawl (v1) returns `data` as a flat list of result objects, causing an
`AttributeError: 'list' object has no attribute 'get'`.
Detect the response format by checking if `data` is a list (self-hosted)
or dict (cloud) and handle both cases.
Cloud format: {"data": {"web": [...], "news": [...]}}
Self-hosted: {"success": true, "data": [{"url": "...", "title": "...", ...}]}
Co-authored-by: Synergy <synergyoclaw@gmail.com>
* feat: add environment and user tracking to prompt management (#24855)
* feat: add environment and user tracking to prompt management
- Add environment (development/staging/production) and created_by columns to LiteLLM_PromptTable
- Update unique constraint to [prompt_id, version, environment]
- All CRUD endpoints support environment filtering and user tracking
- Redesigned prompt detail page with environment tabs and version history
- UI: environment filter on list page, environment selector in editor
- 8 new tests for environment and user tracking
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* fix: Black formatting and add environments to PromptInfoResponse TypeScript type
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* fix: address Greptile review findings
- P1: delete_prompt scopes in-memory cleanup to environment when provided
- P2: dotprompt_content parsed directly regardless of environment flag
- P2: use distinct for environments query
- P2: fix double-fetch on initial mount in prompt_info.tsx
- fix: remove unsupported select kwarg from find_many
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* fix: address remaining Greptile review comments
- Remove unused useCallback import (index.tsx)
- Remove unused ENV_COLORS variable (prompt_info.tsx)
- P1: in-memory fallback in get_prompt_versions now respects environment filter
- P1: reset selectedEnv when promptId changes to avoid stale state
- Cyclic imports are pre-existing pattern, not introduced by this PR
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* fix: scope patch_prompt to environment using primary key
- Add environment query param to patch_prompt endpoint
- Look up target row by composite key (prompt_id + version + environment)
- Update by primary key (id) to target exactly one row
- Fixes Greptile finding: patch with multiple environments no longer ambiguous
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---------
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* fix: use actual start_time for failed request spend logs (#24906)
async_post_call_failure_hook set both start_time and end_time to
datetime.now(), making all failed requests show duration=0. Use the
actual start_time from litellm_logging_obj instead, so spend logs
reflect the real request duration on timeout and other failures.
Fixes #24888
* feat(bedrock): add nova canvas image edit support (#24869)
* feat(bedrock): add nova canvas image edit support
* fix(bedrock): support PathLike inputs for nova image edit
* chore: sync schema.prisma copies from root
* fix(mypy): correct type-ignore code for delta_usage arg-type
* fix(mypy): cast status_code to str, suppress intentional str yield
* fix(lint): extract _create_content_block_chunks to fix PLR0915
* fix(lint): extract helpers to fix PLR0915 in prompt endpoints
---------
Co-authored-by: michelligabriele <gabriele.michelli@icloud.com>
Co-authored-by: Sameer Kankute <sameer@berri.ai>
Co-authored-by: redhelix <amin.lalji@gmail.com>
Co-authored-by: Synergy <synergyoclaw@gmail.com>
Co-authored-by: Talha Anwar <37379131+talhaanwarch@users.noreply.github.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: madhu19991 <madhu@thunkai.com>
Co-authored-by: Srikanth @adobe <devarakondasrikanth@users.noreply.github.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
* fix(test): update model armor streaming test to handle string or int error code
---------
Co-authored-by: michelligabriele <gabriele.michelli@icloud.com>
Co-authored-by: Sameer Kankute <sameer@berri.ai>
Co-authored-by: redhelix <amin.lalji@gmail.com>
Co-authored-by: Synergy <synergyoclaw@gmail.com>
Co-authored-by: Talha Anwar <37379131+talhaanwarch@users.noreply.github.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: madhu19991 <madhu@thunkai.com>
Co-authored-by: Srikanth @adobe <devarakondasrikanth@users.noreply.github.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
2026-04-04 05:57:44 +08:00
@@unique([prompt_id, version, environment])
@@index([prompt_id, environment])
2025-11-20 05:19:56 +08:00
@@index([prompt_id])
2025-08-03 13:33:37 +08:00
}
2025-06-18 23:37:40 +08:00
model LiteLLM_HealthCheckTable {
health_check_id String @id @default(uuid())
model_name String
model_id String?
status String
healthy_count Int @default(0)
unhealthy_count Int @default(0)
error_message String?
response_time_ms Float?
details Json?
checked_by String?
checked_at DateTime @default(now())
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@index([model_name])
@@index([checked_at])
@@index([status])
2026-04-15 11:41:52 +08:00
@@index([model_id, model_name, checked_at(sort: Desc)], map: "LiteLLM_HealthCheckTable_model_id_model_name_checked_at_idx")
2025-10-24 08:57:49 +08:00
}
// Search Tools table for storing search tool configurations
model LiteLLM_SearchToolsTable {
search_tool_id String @id @default(uuid())
search_tool_name String @unique
litellm_params Json
search_tool_info Json?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
2025-10-31 05:32:08 +08:00
}
// SSO configuration table
model LiteLLM_SSOConfig {
id String @id @default("sso_config")
sso_settings Json
created_at DateTime @default(now())
updated_at DateTime @updatedAt
2025-11-01 08:43:59 +08:00
}
2025-11-02 04:45:32 +08:00
model LiteLLM_ManagedVectorStoreIndexTable {
2025-11-02 03:01:32 +08:00
id String @id @default(uuid())
index_name String @unique
litellm_params Json
index_info Json?
created_at DateTime @default(now())
created_by String?
updated_at DateTime @updatedAt
updated_by String?
}
2025-11-01 08:43:59 +08:00
// Cache configuration table
model LiteLLM_CacheConfig {
id String @id @default("cache_config")
cache_settings Json
created_at DateTime @default(now())
updated_at DateTime @updatedAt
2025-12-10 03:19:53 +08:00
}
// UI Settings configuration table
model LiteLLM_UISettings {
id String @id @default("ui_settings")
ui_settings Json
created_at DateTime @default(now())
updated_at DateTime @updatedAt
2025-12-19 21:25:59 +08:00
}
2026-03-07 15:39:08 +08:00
// Generic config overrides table - one row per config_type
model LiteLLM_ConfigOverrides {
config_type String @id
config_value Json
created_at DateTime @default(now())
updated_at DateTime @updatedAt
}
2025-12-19 21:25:59 +08:00
// Skills table for storing LiteLLM-managed skills
model LiteLLM_SkillsTable {
skill_id String @id @default(uuid())
display_title String?
description String?
instructions String? // The skill instructions/prompt (from SKILL.md)
source String @default("custom") // "custom" or "anthropic"
latest_version String?
file_content Bytes? // Binary content of the skill files (zip)
file_name String? // Original filename
file_type String? // MIME type (e.g., "application/zip")
metadata Json? @default("{}")
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
2026-01-05 14:49:09 +08:00
}
2026-01-24 05:16:58 +08:00
2026-02-22 12:14:31 +08:00
// Policy table for storing guardrail policies (versioned)
2026-01-24 05:16:58 +08:00
model LiteLLM_PolicyTable {
2026-02-22 12:14:31 +08:00
policy_id String @id @default(uuid())
policy_name String // No longer @unique; use @@unique([policy_name, version_number])
version_number Int @default(1)
version_status String @default("production") // "draft" | "published" | "production"
parent_version_id String?
is_latest Boolean @default(true)
published_at DateTime?
production_at DateTime?
inherit String? // Name of parent policy to inherit from
description String?
guardrails_add String[] @default([])
guardrails_remove String[] @default([])
condition Json? @default("{}") // Policy conditions (e.g., model matching)
pipeline Json? // Optional guardrail pipeline (mode + steps[])
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
@@unique([policy_name, version_number])
@@index([policy_name, version_status])
2026-01-24 05:16:58 +08:00
}
// Policy attachment table for defining where policies apply
model LiteLLM_PolicyAttachmentTable {
attachment_id String @id @default(uuid())
policy_name String // Name of the policy to attach
scope String? // Use '*' for global scope
teams String[] @default([]) // Team aliases or patterns
keys String[] @default([]) // Key aliases or patterns
models String[] @default([]) // Model names or patterns
2026-02-11 09:50:37 +08:00
tags String[] @default([]) // Tag patterns (e.g., ["healthcare", "prod-*"])
2026-01-24 05:16:58 +08:00
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
}
2026-02-12 12:44:30 +08:00
2026-03-04 12:22:20 +08:00
// Global tool registry - auto-discovered from LLM responses; admins set input/output policies here
2026-02-26 03:44:30 +08:00
model LiteLLM_ToolTable {
2026-03-04 12:22:20 +08:00
tool_id String @id @default(uuid())
tool_name String @unique // e.g. "huggingface_remote-mcp__dynamic_space"
origin String? // MCP server name or "user_defined"
input_policy String @default("untrusted") // "trusted" | "untrusted" | "blocked"
output_policy String @default("untrusted") // "trusted" | "untrusted"
call_count Int @default(0) // cumulative number of times this tool was seen
assignments Json? @default("{}")
key_hash String? // hash of the virtual key that first called this tool
team_id String? // team that first called this tool
key_alias String? // human-readable alias of the virtual key
user_agent String? // user-agent of the first request that discovered this tool
last_used_at DateTime? // timestamp of the most recent call
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
@@index([input_policy])
@@index([output_policy])
2026-02-26 03:44:30 +08:00
@@index([team_id])
}
2026-03-07 15:39:08 +08:00
// Per-(tool, team/key) policy overrides. When present, override replaces global tool policy for that scope.
2026-02-12 12:44:30 +08:00
//Unified Access Groups table for storing unified access groups
2026-02-13 04:30:22 +08:00
model LiteLLM_AccessGroupTable {
2026-02-12 12:44:30 +08:00
access_group_id String @id @default(uuid())
access_group_name String @unique
description String?
// Resource memberships - explicit arrays per type
2026-02-14 08:44:49 +08:00
access_model_names String[] @default([])
2026-02-12 12:44:30 +08:00
access_mcp_server_ids String[] @default([])
access_agent_ids String[] @default([])
assigned_team_ids String[] @default([])
assigned_key_ids String[] @default([])
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
2026-02-28 07:59:37 +08:00
}
// Claude Code Plugin Marketplace table
model LiteLLM_ClaudeCodePluginTable {
id String @id @default(uuid())
name String @unique
version String?
description String?
manifest_json String?
files_json String? @default("{}")
enabled Boolean @default(true)
created_at DateTime? @default(now())
updated_at DateTime? @default(now()) @updatedAt
created_by String?
@@map("LiteLLM_ClaudeCodePluginTable")
}
2026-04-19 07:35:17 +08:00
feat(proxy): add /v1/memory CRUD endpoints (#26218)
* feat(proxy): add /v1/memory CRUD endpoints with user/team scoping
New LiteLLM_MemoryTable stores user/team-scoped key/value entries with
optional JSON metadata. Value is a String (LLM-readable text) and metadata
is an optional Json? envelope, matching the Letta + mem0 hybrid model so
future structured fields can be added without a schema migration.
Endpoints:
POST /v1/memory - create
GET /v1/memory - list (caller-scoped; admins see all)
GET /v1/memory/{key} - fetch one
PUT /v1/memory/{key} - upsert
DELETE /v1/memory/{key} - delete
Non-admin callers cannot set a user_id/team_id other than their own.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
* fix(proxy/memory): omit metadata field when None on create
Prisma's Python client rejects `metadata=None` on a `Json?` field with
"A value is required but not set" — the field must be omitted from the
`data` dict entirely to store SQL NULL. Build the create payload
conditionally in both `create_memory` and the PUT-create branch of
`upsert_memory`.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
* feat(ui): add Memory page to view/manage /v1/memory entries
Adds a new "Memory" sidebar item under Tools so users can see what their
agents have stored. Lists all memories visible to the caller (scoped by
the backend), with a key-search filter, preview column, scope tags, and
view/edit/delete actions. Create modal accepts optional JSON metadata.
- networking.tsx: fetchMemoryList / createMemory / updateMemory / deleteMemory
wired to the /v1/memory CRUD endpoints.
- MemoryView + MemoryEditModal: new antd-based components (per CLAUDE.md:
use antd for new UI, not tremor).
- page.tsx + leftnav.tsx: wire the "memory" route + sidebar entry.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
* feat(memory): add key_prefix filter + promote Memory to AI GATEWAY nav
Backend:
- GET /v1/memory now accepts `key_prefix` for Redis-style namespace
scans (e.g. `?key_prefix=user:`). When both `key` and `key_prefix`
are passed, `key_prefix` wins.
- Prefix filter sits under the visibility filter in the Prisma where
clause, so it can never leak rows across user/team scopes.
- New tests: prefix match, and cross-scope isolation (another user's
`user:*` rows must not appear in the caller's results).
UI:
- Memory moved from a Tools submenu to a top-level AI GATEWAY item
(alongside Agents, MCP Servers, Skills) — it's an API primitive,
not a tool-management surface.
- Search box now drives prefix search, matching the Redis mental
model ("type the namespace, see everything under it").
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
* fix(memory): enforce unique key per scope by using NULLS NOT DISTINCT
The unique constraint `(key, user_id, team_id)` on LiteLLM_MemoryTable
silently allowed duplicates when user_id or team_id was NULL, because
Postgres treats every NULL as distinct by default (ANSI semantics). A
caller with no team_id could POST the same key three times and get
three rows.
Migration:
1. Dedupe existing rows, keeping the most recent per (key, user_id,
team_id), using `IS NOT DISTINCT FROM` so NULL == NULL.
2. Drop the old unique index.
3. Recreate it with `NULLS NOT DISTINCT` (Postgres 15+).
No code change: POST already returns 409 on unique-violation error
messages — it just wasn't firing before because the constraint didn't
catch the NULL-team case.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
* fix(memory): make key globally unique, 409 on any duplicate
Switches from the compound unique `(key, user_id, team_id)` to a simple
`key @unique`. The compound form silently allowed duplicates when
user_id or team_id was NULL (Postgres treats each NULL as distinct), so
callers could POST the same key repeatedly. Globally-unique key means
one row per key, period — any duplicate create → 409.
- schema.prisma (×3): `key String @unique`, drop `@@unique(...)`.
- initial add_memory_table migration: unique index on (key) only.
- Remove the now-unused follow-up NULLS NOT DISTINCT migration.
- Endpoint error message simplified ("already exists" — no "for this scope").
- Test fake's create() now enforces global key uniqueness.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
* fix(ui/memory): full-width layout + user/teams-style columns
- Add `w-full` to the MemoryView outer div so the page fills the
flex-flex-1 container (was collapsing to intrinsic width).
- Replace the combined "Scope" column with separate User ID / Team ID
columns, matching the layout of the Users / Teams pages: ID, Name,
Preview, User ID, Team ID, Updated, Actions.
- IDs render with a truncated mono label + copy-to-clipboard button,
same pattern as view_users.
- Detail drawer now shows Memory ID / User ID / Team ID as separate
fields instead of stacked color tags.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
* fix(ui/memory): use clean MCP-style ID pill, drop copy icons
The ID / User ID / Team ID columns showed a mono text blob with a
copy-to-clipboard icon next to each value — too busy compared to the
MCP Servers page. Swap the renderer for MCP's pill style:
- Truncated mono ID inside a blue Tailwind pill
(`font-mono text-blue-600 bg-blue-50 ... rounded-md border`).
- No copy icon. Full ID surfaces via tooltip.
- ID column is a button that opens the detail drawer on click;
user/team ID pills are static (not clickable).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
* fix(memory): address greptile review feedback
Addresses 5 greptile findings (3/5 → higher confidence target):
1. Identity-less orphan rows (P1): non-admin callers with no user_id AND
no team_id could create rows that the visibility filter would never
match again. Now rejected up front with 400 — caller must authenticate
with a scoped key or act as PROXY_ADMIN.
2. Upsert race returning 500 (P1): PUT's check-then-create isn't atomic;
a concurrent writer could slip a row in between the 404-check and the
create call. Now catch unique-violation on create, re-read, and fall
through to update — PUT stays idempotent. If the conflicting row
belongs to a different scope, surface a 409 instead of 500.
3. PUT-create scope inconsistency (P2): PUT's create branch always used
the caller's own user_id/team_id, so admins couldn't bootstrap rows
scoped elsewhere via PUT (only POST). Now PUT-create calls the shared
`_resolve_scope()` helper, matching POST semantics.
4. Stale schema comment (P2): schema said "Keyed by (key, user_id,
team_id)" but `key` is globally unique. Updated all three schema
copies to reflect the actual design.
5. UI silently truncated at 200 (P2): MemoryView fetched pageSize=200
with no load-more. Swapped to real server-side pagination driven by
`data.total`; page size is now 50 and the pager is a real AntD
control.
Also extracts a shared `_resolve_scope()` helper and `_is_unique_violation()`
from create_memory so POST and PUT don't drift on the scope/error logic.
Tests: +3 new (identity-less 400, PUT admin bootstrap, PUT race →
update), 18/18 pass.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
* fix(memory): typed Prisma error + explicit-null metadata on PUT
Two more greptile threads from the last review:
- Unique-violation detection was string-matching "Unique"/"UniqueViolation"
in the exception message, fragile across Prisma/driver versions. Now
check the typed error `code == "P2002"` first, with string fallback.
- PUT could not distinguish "metadata omitted" from "metadata: null" —
both parsed as `None`, so callers had no way to clear stored metadata.
Switch to Pydantic v2's `model_fields_set` to tell which fields the
caller actually sent; explicit null now clears the column.
New tests:
- explicit null clears metadata
- omitted metadata preserves existing value
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
* fix(ui/memory): send explicit null when user clears metadata
Addresses the remaining P1 from the last greptile review:
When the edit modal's metadata textarea was cleared and saved,
`metadataParsed` stayed `undefined`, `JSON.stringify` dropped the key
entirely, and the backend's `model_fields_set` guard therefore left
the stored metadata untouched — UI showed success but nothing changed.
Now: empty textarea on edit → send explicit `null` so the backend
sees `metadata` in `model_fields_set` and clears the column.
Empty textarea on create still maps to `undefined` (field omitted)
to avoid Prisma's `Json? = None` quirk on insert.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
* fix(ui/memory): preserve slashes in key path encoding
The backend route `/v1/memory/{key:path}` supports keys with slashes,
but `encodeURIComponent` encoded `/` as `%2F`. Some proxies (nginx
default, CloudFlare, AWS ALB) reject or re-decode `%2F` mid-flight,
so UI update/delete calls on slash-containing keys could fail or
silently misroute.
New helper `encodeMemoryKeyForPath` splits by `/`, URL-encodes each
segment, then rejoins with literal `/`. Every other unsafe char
(spaces, `?`, `#`, `%`) stays encoded per-segment; slashes stay as
path delimiters, matching what the `:path` converter expects.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
* fix(ui/memory): drop misleading client-side column sorters
With server-side pagination, client sorters on `key` and `updated_at`
only reorder the current page while pretending to sort the full
dataset — users would see "sorted by name" but only the visible 50
rows would actually be sorted.
Remove the sorters. The backend already returns rows in
`updated_at DESC` order (sensible default for a memory view), and
users can narrow the result with the key-prefix filter.
Greptile also flagged missing `@@map` on the new model as a
"consistency" issue, but only 1 of 59 tables in this repo uses
`@@map` — the dominant pattern is to rely on Prisma's default
(model name == table name). Skipping that finding as a
false-positive on convention.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
* fix(memory): compose visibility + key filters via explicit AND
Greptile P1 (filter-fragility): `where.update(vis)` was semantically
correct today, but dict-merging by key meant any future visibility
filter that grew a new top-level "OR" would silently clobber the
existing key filter.
Compose explicitly instead:
where = {"AND": [key_filter, vis]}
Applied to both `list_memory` and `_find_memory_for_caller`. When
either side is empty (admin has no visibility filter; list has no
key filter), skip the wrapper and use the non-empty side directly
to keep the generated SQL clean.
Test fake's `_matches` now understands top-level `AND` too.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
* refactor(ui/memory): wrap write helpers with react-query useMutation
Previously the Memory view read via `useQuery` but called the raw
create/update/delete fetch helpers directly in handlers, tracking
loading state with a local `submitting` flag and invalidating state
via `refetch()`. That mixes two concerns:
- it skips react-query's mutation state (isPending / isError / isSuccess)
- `refetch()` only retouches the currently-mounted query instance, not
other cached pages, so navigating back to an older page could show
stale rows
Switch the three write paths to `useMutation`:
- `createMutation`, `updateMutation`, `deleteMutation` — each owns
the mutation fn, success toast, and error toast.
- Success handlers invalidate the whole `["memoryList", ...]` prefix
via `queryClient.invalidateQueries`, so every cached page refetches
(pagination + filter-aware).
- Refresh button now invalidates instead of `refetch()`, keeping all
behavior consistent.
- handleSave/handleDelete become thin adapters that call `.mutateAsync`;
their errors are swallowed locally since the mutation's onError has
already surfaced the toast.
Also tightened the edit modal's key-field tooltip to reflect the
actual global-unique semantics (was "Unique per user/team scope").
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
* fix(memory): close cross-user write gap + sanitize 500 errors (Veria)
Addresses two Veria findings:
**High — cross-user memory tampering via team membership.** The
visibility filter uses an OR (`user_id == caller OR team_id == caller`)
so team members can SEE each other's team-scoped rows. That's
intentional for list/get. But because PUT/DELETE used the same filter
to find the target row, any team member could overwrite or delete a
teammate's *personal* row whenever both `user_id` and `team_id` were
stamped on it — broader visibility was being silently treated as
broader authority.
New `_assert_write_access(row, caller)` enforces ownership for
mutations. Non-admin rules:
- The row's `user_id` must match the caller (personal ownership), OR
- The row has no `user_id` and its `team_id` matches the caller's
team (a "pure team row" intended for shared writes).
Admins bypass the check. The same gate runs in PUT (both regular
and post-race-recovery branches) and DELETE.
**Medium — DB internals leaked through 500 detail.** Every `except`
block was raising `HTTPException(500, detail=str(e))`, which surfaces
Prisma error strings (table/column names, host:port, error class
names) to API callers. New `_internal_error()` helper logs the real
exception server-side and returns a generic, caller-safe `detail`.
Applied to create, list, upsert (general fallthrough), and delete.
Also tightened the race-recovery 409 message to drop the "in a
different scope" wording — the caller never needs to know whose
scope it lives in.
Tests (+5):
- teammate cannot overwrite personal row → 403
- teammate cannot delete personal row → 403
- teammate CAN modify pure team row (no user_id stamped) → 200
- admin bypasses write-auth → 200
- 500 response never echoes Prisma internals (table/host/class names)
25/25 unit tests pass.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
* fix(memory): require team admin to modify pure team rows
Tightens the write-authorization rule for "pure team rows" (rows with
no user_id stamped, only team_id) to match the pattern used by
team-management endpoints (`_is_user_team_admin` + `_is_user_org_admin_for_team`):
- Plain team members can READ team rows via the OR visibility filter
(intentional, unchanged).
- Only PROXY_ADMIN, team admins of the row's team_id, or org admins
for the team's organization may MODIFY them. Plain members get 403.
`_assert_write_access` is now async and takes the prisma_client so it
can fetch the team and run the existing `_is_user_team_admin` /
`_is_user_org_admin_for_team` helpers from
`litellm.proxy.management_endpoints.common_utils`. The org-admin path
is best-effort: it calls `get_user_object`, which depends on the
proxy_server module being initialized, so any exception there is
treated as "not an org admin" rather than crashing the request.
Tests:
- team admin can modify pure team row → 200
- plain team member cannot modify pure team row → 403
- plain team member cannot delete pure team row → 403
Updates the test fake to add a tiny `litellm_teamtable.find_unique`
implementation and a `_make_team(team_id, admin_user_ids=[...])`
helper.
27/27 unit tests pass.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
* fix: mypy + UI page-metadata sync for memory page
Two CI failures:
1. mypy: `_find_memory_for_caller` had `key_filter` inferred as
`dict[str, str]` (literal type) and the conditional `{"AND": [key_filter, vis]}`
returned `dict[str, list[...]]`, so the join site failed
`dict-item` typing. Annotate both intermediates as `dict` so mypy
widens the value type.
2. UI test (`page_utils.test.ts > should have descriptions for all
pages`): every leftnav entry must have a description in
`page_metadata.ts`, and `memory` was missing. Added a one-line
description, matching the style of neighboring entries.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
* [Feat] Day-0 support for GPT-5.5 and GPT-5.5 Pro (#26449)
* feat(openai): day-0 support for GPT-5.5 and GPT-5.5 Pro
Add pricing + capability entries for the new GPT-5.5 family launched by
OpenAI on 2026-04-24:
- gpt-5.5 / gpt-5.5-2026-04-23 (chat): $5/$30/$0.50 per 1M
input/output/cached input
- gpt-5.5-pro / gpt-5.5-pro-2026-04-23 (responses-only): $60/$360/$6
per 1M input/output/cached input
Other fees (long-context >272k, flex, batches, priority, cache
discounts) follow the same ratios as GPT-5.4, with context window
retained at 1.05M input / 128K output.
No transformation / classifier code changes are required:
OpenAIGPT5Config.is_model_gpt_5_4_plus_model() already matches 5.5+ via
numeric version parsing, and model registration is driven from the
JSON. The existing responses-API bridge for tools + reasoning_effort
(litellm/main.py:970) already covers gpt-5.5-pro.
Tests:
- GPT5_MODELS regression list now covers gpt-5.5-pro and dated variants
- New test_generic_cost_per_token_gpt55_pro cost-calc test
- Updated test_generic_cost_per_token_gpt55 for long-context fields
* fix(openai): mirror reasoning_effort flags onto gpt-5.5 dated variants
gpt-5.5-2026-04-23 and gpt-5.5-pro-2026-04-23 were missing the
supports_none_reasoning_effort, supports_xhigh_reasoning_effort, and
supports_minimal_reasoning_effort flags that their non-dated
counterparts define. Reasoning-effort routing in OpenAIGPT5Config is
fully capability-driven from these JSON flags — since an absent flag
is treated as False for opt-in levels (xhigh), users pinning to a
dated snapshot would silently lose xhigh support and diverge from the
base alias on logprobs + flexible temperature handling.
Copy the flags onto both dated variants so every dated snapshot
inherits the base model's reasoning-effort capability profile.
Adds a parametrized regression test that asserts
supports_{none,minimal,xhigh}_reasoning_effort parity between each
dated variant and its non-dated counterpart, preventing future drift
when new snapshots are added.
* fix(schema): close LiteLLM_MemoryTable model brace dropped during merge
The rebase against `litellm_internal_staging` (which added
`LiteLLM_AdaptiveRouterState` / `LiteLLM_AdaptiveRouterSession`) left
the closing brace of `LiteLLM_MemoryTable` missing in all three
schema copies — the next model declaration ended up parsed as a field
of the memory table, surfacing as the CI prisma error:
error: This line is not a valid field or attribute definition.
--> schema.prisma:1250
|
1249 | // Per-(router, request_type, model) Beta posterior for the adaptive router.
1250 | model LiteLLM_AdaptiveRouterState {
Add the missing `}` (and the standard blank line) after the memory
table's `@@index([team_id])` in `schema.prisma`,
`litellm/proxy/schema.prisma`, and
`litellm-proxy-extras/litellm_proxy_extras/schema.prisma`.
`prisma generate --schema litellm/proxy/schema.prisma` now runs clean;
27/27 memory unit tests pass.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---------
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Co-authored-by: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
2026-04-25 09:38:07 +08:00
// User/team-scoped memory store with a GLOBAL unique key.
// `value` is a string (typically markdown/text meant for LLM context);
// `metadata` is an optional JSON envelope for structured tags without schema changes.
// Note: `key` is globally unique across all users/teams — callers should
// namespace their keys (e.g. `user:123:notes`) if they need per-user isolation.
// `user_id` / `team_id` stamp ownership for visibility filtering, but do NOT
// participate in the unique constraint.
model LiteLLM_MemoryTable {
memory_id String @id @default(uuid())
key String @unique
value String
metadata Json?
user_id String?
team_id String?
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
@@index([user_id])
@@index([team_id])
}
2026-04-19 07:35:17 +08:00
// Per-(router, request_type, model) Beta posterior for the adaptive router.
model LiteLLM_AdaptiveRouterState {
router_name String
request_type String
model_name String
alpha Float
beta Float
total_samples Int @default(0)
2026-04-22 07:27:01 +08:00
last_updated_at DateTime @default(now()) @updatedAt
2026-04-19 07:35:17 +08:00
@@id([router_name, request_type, model_name])
}
// Per-(session, router, model) signal counters for the adaptive router.
model LiteLLM_AdaptiveRouterSession {
session_id String
router_name String
model_name String
classified_type String
misalignment_count Int @default(0)
stagnation_count Int @default(0)
disengagement_count Int @default(0)
satisfaction_count Int @default(0)
failure_count Int @default(0)
loop_count Int @default(0)
exhaustion_count Int @default(0)
last_user_content String?
last_assistant_content String?
tool_call_history Json @default("[]")
pending_tool_calls Json @default("{}")
turn_count Int @default(0)
last_processed_turn Int @default(-1)
clean_credit_awarded Boolean @default(false)
terminal_status Int?
2026-04-22 07:27:01 +08:00
last_activity_at DateTime @default(now()) @updatedAt
2026-04-19 07:35:17 +08:00
@@id([session_id, router_name, model_name])
2026-04-22 08:49:38 +08:00
@@index([last_activity_at], map: "idx_adaptive_router_session_activity")
2026-04-19 07:35:17 +08:00
}
2026-04-30 08:12:18 +08:00
// ---------------------------------------------------------------------------
// Workflow Run Tracking
//
// Generic durable state tracking for any agent or automated workflow.
// Design: three tables — run (header + materialized status), event (append-only
// source of truth for state transitions), message (conversation inbox/outbox).
//
// Usage:
// - Set `workflow_type` to identify the owning system (e.g. "shin-builder").
// - Store domain-specific fields in `metadata` (worktree_path, pr_url, etc.).
// - `session_id` on WorkflowRun matches `x-litellm-session-id` header sent to
// the proxy — all spend logs for this run are automatically tagged.
// ---------------------------------------------------------------------------
// One instance of work being done. `status` is a materialized cache of the
// latest event; the event log is the authoritative source of truth.
model LiteLLM_WorkflowRun {
run_id String @id @default(uuid())
session_id String @unique @default(uuid())
workflow_type String
status String @default("pending")
created_by String? // user_id of the key that created this run; null = created by master key
created_at DateTime @default(now())
updated_at DateTime @updatedAt
input Json?
output Json?
metadata Json?
events LiteLLM_WorkflowEvent[]
messages LiteLLM_WorkflowMessage[]
@@index([workflow_type, status])
@@index([session_id])
@@index([created_at])
@@index([created_by])
}
// Append-only log of state transitions. Never mutate rows here.
// `step_name` and `event_type` are caller-defined strings — no hardcoded enums.
// Status auto-update rules (applied by the append endpoint):
// step.started → run.status = running
// step.failed → run.status = failed
// hook.waiting → run.status = paused
// hook.received → run.status = running
model LiteLLM_WorkflowEvent {
event_id String @id @default(uuid())
run_id String
event_type String
step_name String
sequence_number Int
data Json?
created_at DateTime @default(now())
run LiteLLM_WorkflowRun @relation(fields: [run_id], references: [run_id])
@@unique([run_id, sequence_number])
@@index([run_id])
}
// Conversation inbox/outbox — full message content, separate from the durable
// event log. Spend logs truncate messages; this table stores them in full.
// `session_id` here is the Claude --resume session ID (or similar).
model LiteLLM_WorkflowMessage {
message_id String @id @default(uuid())
run_id String
role String
content String
sequence_number Int
session_id String?
created_at DateTime @default(now())
run LiteLLM_WorkflowRun @relation(fields: [run_id], references: [run_id])
@@unique([run_id, sequence_number])
@@index([run_id])
}