litellm/schema.prisma

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

1370 lines
53 KiB
Plaintext
Raw Normal View History

datasource client {
provider = "postgresql"
url = env("DATABASE_URL")
}
generator client {
provider = "prisma-client-py"
2026-02-01 07:07:28 +08:00
binaryTargets = ["native", "debian-openssl-1.1.x", "debian-openssl-3.0.x", "linux-musl", "linux-musl-openssl-3.0.x"]
}
2024-03-03 03:59:17 +08:00
// Budget / Rate Limits for an org
model LiteLLM_BudgetTable {
budget_id String @id @default(uuid())
max_budget Float?
2024-03-03 04:25:40 +08:00
soft_budget Float?
2024-03-03 03:59:17 +08:00
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
model_max_budget Json?
feat(teams): per-member model scope + team default_team_member_models (#24950) * fix(bedrock): strip [1m]/[200k] context window suffixes before cost lookup * test(bedrock): add test for [1m] context window suffix stripping in cost lookup * schema: add allowed_models to BudgetTable, default_team_member_models to TeamTable * migration: add allowed_models and default_team_member_models columns * types: add allowed_models to TeamMemberAddRequest, TeamMemberUpdateRequest, UpdateTeamRequest * utils: add allowed_models param to add_new_member, persist to budget table * common_utils: add allowed_models to _upsert_budget_and_membership * team endpoints: seed allowed_models on member_add, persist on member_update and team/update * auth: enforce per-member allowed_models at request time * networking: add allowed_models to Member type and teamMemberUpdateCall * TeamMemberTab: add Model Scope column showing per-member allowed_models * EditMembership: add Allowed Models multi-select field * TeamInfo: add default_team_member_models field in Settings tab * chore: sync schema.prisma copies from root * fix(team_member_update): update existing budget in-place instead of creating new one When a member already has a budget_id, patch only the fields the caller provided rather than always creating a fresh budget record. The old code ignored existing_budget_id entirely, so updating only allowed_models silently dropped the stored max_budget / tpm_limit / rpm_limit values. * fix(auth): pass llm_router to _check_team_member_model_access Without the router, _can_object_call_model cannot resolve wildcard model names (e.g. openai/*) or access-group names in allowed_models, causing legitimate requests to be denied. Thread the existing llm_router from _run_common_checks through to the new member-scope check. * feat(ui): add Team Member Settings accordion to Create Team modal Groups default_team_member_models, member budget/key duration, and tpm/rpm defaults into a single collapsible section. The model picker is filtered to only show the models selected for the team, and the copy distinguishes it from the team-level Models field. * feat(ui): consolidate Team Member Settings into accordion in edit team form Moves default_team_member_models + per-member budget/key/tpm/rpm fields into a collapsible "Team Member Settings" panel. Keeps the top-level form focused on team-wide settings (team models, team budget, tpm/rpm). * fix(ui): use tremor Accordion for Team Member Settings in edit team form * fix(ui): move Team Member Settings accordion above budget fields in Create Team * chore: fixes --------- Co-authored-by: Ishaan Jaffer <ishaanjaffer0324@gmail.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Yuneng Jiang <yuneng@berri.ai>
2026-04-07 04:48:43 +08:00
budget_duration String?
2024-03-03 03:59:17 +08:00
budget_reset_at DateTime?
feat(teams): per-member model scope + team default_team_member_models (#24950) * fix(bedrock): strip [1m]/[200k] context window suffixes before cost lookup * test(bedrock): add test for [1m] context window suffix stripping in cost lookup * schema: add allowed_models to BudgetTable, default_team_member_models to TeamTable * migration: add allowed_models and default_team_member_models columns * types: add allowed_models to TeamMemberAddRequest, TeamMemberUpdateRequest, UpdateTeamRequest * utils: add allowed_models param to add_new_member, persist to budget table * common_utils: add allowed_models to _upsert_budget_and_membership * team endpoints: seed allowed_models on member_add, persist on member_update and team/update * auth: enforce per-member allowed_models at request time * networking: add allowed_models to Member type and teamMemberUpdateCall * TeamMemberTab: add Model Scope column showing per-member allowed_models * EditMembership: add Allowed Models multi-select field * TeamInfo: add default_team_member_models field in Settings tab * chore: sync schema.prisma copies from root * fix(team_member_update): update existing budget in-place instead of creating new one When a member already has a budget_id, patch only the fields the caller provided rather than always creating a fresh budget record. The old code ignored existing_budget_id entirely, so updating only allowed_models silently dropped the stored max_budget / tpm_limit / rpm_limit values. * fix(auth): pass llm_router to _check_team_member_model_access Without the router, _can_object_call_model cannot resolve wildcard model names (e.g. openai/*) or access-group names in allowed_models, causing legitimate requests to be denied. Thread the existing llm_router from _run_common_checks through to the new member-scope check. * feat(ui): add Team Member Settings accordion to Create Team modal Groups default_team_member_models, member budget/key duration, and tpm/rpm defaults into a single collapsible section. The model picker is filtered to only show the models selected for the team, and the copy distinguishes it from the team-level Models field. * feat(ui): consolidate Team Member Settings into accordion in edit team form Moves default_team_member_models + per-member budget/key/tpm/rpm fields into a collapsible "Team Member Settings" panel. Keeps the top-level form focused on team-wide settings (team models, team budget, tpm/rpm). * fix(ui): use tremor Accordion for Team Member Settings in edit team form * fix(ui): move Team Member Settings accordion above budget fields in Create Team * chore: fixes --------- Co-authored-by: Ishaan Jaffer <ishaanjaffer0324@gmail.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Yuneng Jiang <yuneng@berri.ai>
2026-04-07 04:48:43 +08:00
allowed_models String[] @default([]) // per-member model scope; empty = inherit team models
2024-03-03 03:59:17 +08:00
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
organization LiteLLM_OrganizationTable[] // multiple orgs can have the same budget
2025-11-13 06:38:15 +08:00
projects LiteLLM_ProjectTable[] // multiple projects can have the same budget
keys LiteLLM_VerificationToken[] // multiple keys can have the same budget
end_users LiteLLM_EndUserTable[] // multiple end-users can have the same budget
tags LiteLLM_TagTable[] // multiple tags can have the same budget
team_membership LiteLLM_TeamMembership[] // budgets of Users within a Team
organization_membership LiteLLM_OrganizationMembership[] // budgets of Users within a Organization
2024-03-03 03:59:17 +08:00
}
// Models on proxy
model LiteLLM_CredentialsTable {
credential_id String @id @default(uuid())
credential_name String @unique
credential_values Json
credential_info Json?
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
}
// Models on proxy
model LiteLLM_ProxyModelTable {
model_id String @id @default(uuid())
model_name String
litellm_params Json
model_info Json?
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
}
// Agents on proxy
model LiteLLM_AgentsTable {
agent_id String @id @default(uuid())
agent_name String @unique
litellm_params Json?
agent_card_params Json
2026-03-07 15:39:08 +08:00
static_headers Json? @default("{}")
extra_headers String[] @default([])
agent_access_groups String[] @default([])
Agents - assign tools (#22064) * feat(proxy): add max_iterations limiter for agent session loops (#22058) Adds a new proxy hook that enforces a per-session cap on the number of LLM calls an agentic loop can make. Callers send a session_id with each request, and the hook counts calls per session, returning 429 when the configured max_iterations limit is exceeded. - Uses Redis Lua script for atomic increment (multi-instance safe) - Falls back to in-memory cache when Redis unavailable - Follows parallel_request_limiter_v3 pattern - Configurable via key metadata: {"max_iterations": 25} - Session counters auto-expire via TTL (default 1hr) Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> * feat: add new code execution dataset * feat(agent_endpoints/): allow giving agents keys * fix: ui fixes * feat: allow assigning mcp servers to agents * fix: eliminate duplicate DB queries in MCP agent auth and N+1 in agent listing (#22110) - Extract _get_agent_object_permission helper so _get_allowed_mcp_servers_for_agent and _get_agent_tool_permissions_for_server share a single DB fetch instead of each independently querying the same agent row (was 1+N queries per MCP request) - Use include={"object_permission": True} on find_many in get_all_agents_from_db to eagerly load permissions in one query instead of N+1 - Use include={"object_permission": True} on create/update/find_unique in all agent CRUD operations, removing attach_object_permission_to_dict follow-up calls Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-26 03:44:30 +08:00
object_permission_id String?
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
spend Float @default(0.0)
tpm_limit Int?
rpm_limit Int?
session_tpm_limit Int?
session_rpm_limit Int?
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
}
2024-03-03 03:59:17 +08:00
model LiteLLM_OrganizationTable {
organization_id String @id @default(uuid())
organization_alias String
2024-03-03 03:59:17 +08:00
budget_id String
metadata Json @default("{}")
models String[]
spend Float @default(0.0)
model_spend Json @default("{}")
object_permission_id String?
2024-03-03 03:59:17 +08:00
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
teams LiteLLM_TeamTable[]
users LiteLLM_UserTable[]
keys LiteLLM_VerificationToken[]
members LiteLLM_OrganizationMembership[] @relation("OrganizationToMembership")
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
2024-03-03 03:59:17 +08:00
}
// Model info for teams, just has model aliases for now.
model LiteLLM_ModelTable {
id Int @id @default(autoincrement())
model_aliases Json? @map("aliases")
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
team LiteLLM_TeamTable?
}
// Assign prod keys to groups, not individuals
model LiteLLM_TeamTable {
2024-03-03 03:59:17 +08:00
team_id String @id @default(uuid())
team_alias String?
2024-03-03 03:59:17 +08:00
organization_id String?
object_permission_id String?
admins String[]
members String[]
members_with_roles Json @default("{}")
metadata Json @default("{}")
max_budget Float?
soft_budget Float?
spend Float @default(0.0)
models String[]
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
budget_duration String?
budget_reset_at DateTime?
blocked Boolean @default(false)
created_at DateTime @default(now()) @map("created_at")
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
model_spend Json @default("{}")
model_max_budget Json @default("{}")
router_settings Json? @default("{}")
team_member_permissions String[] @default([])
access_group_ids String[] @default([])
2026-01-24 05:16:58 +08:00
policies String[] @default([])
feat(teams): per-member model scope + team default_team_member_models (#24950) * fix(bedrock): strip [1m]/[200k] context window suffixes before cost lookup * test(bedrock): add test for [1m] context window suffix stripping in cost lookup * schema: add allowed_models to BudgetTable, default_team_member_models to TeamTable * migration: add allowed_models and default_team_member_models columns * types: add allowed_models to TeamMemberAddRequest, TeamMemberUpdateRequest, UpdateTeamRequest * utils: add allowed_models param to add_new_member, persist to budget table * common_utils: add allowed_models to _upsert_budget_and_membership * team endpoints: seed allowed_models on member_add, persist on member_update and team/update * auth: enforce per-member allowed_models at request time * networking: add allowed_models to Member type and teamMemberUpdateCall * TeamMemberTab: add Model Scope column showing per-member allowed_models * EditMembership: add Allowed Models multi-select field * TeamInfo: add default_team_member_models field in Settings tab * chore: sync schema.prisma copies from root * fix(team_member_update): update existing budget in-place instead of creating new one When a member already has a budget_id, patch only the fields the caller provided rather than always creating a fresh budget record. The old code ignored existing_budget_id entirely, so updating only allowed_models silently dropped the stored max_budget / tpm_limit / rpm_limit values. * fix(auth): pass llm_router to _check_team_member_model_access Without the router, _can_object_call_model cannot resolve wildcard model names (e.g. openai/*) or access-group names in allowed_models, causing legitimate requests to be denied. Thread the existing llm_router from _run_common_checks through to the new member-scope check. * feat(ui): add Team Member Settings accordion to Create Team modal Groups default_team_member_models, member budget/key duration, and tpm/rpm defaults into a single collapsible section. The model picker is filtered to only show the models selected for the team, and the copy distinguishes it from the team-level Models field. * feat(ui): consolidate Team Member Settings into accordion in edit team form Moves default_team_member_models + per-member budget/key/tpm/rpm fields into a collapsible "Team Member Settings" panel. Keeps the top-level form focused on team-wide settings (team models, team budget, tpm/rpm). * fix(ui): use tremor Accordion for Team Member Settings in edit team form * fix(ui): move Team Member Settings accordion above budget fields in Create Team * chore: fixes --------- Co-authored-by: Ishaan Jaffer <ishaanjaffer0324@gmail.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Yuneng Jiang <yuneng@berri.ai>
2026-04-07 04:48:43 +08:00
default_team_member_models String[] @default([]) // default allowed_models for newly added team members; empty = no per-member restriction
budget_limits Json? // per-model budget limits for the team
model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases
allow_team_guardrail_config Boolean @default(false) // if true, team admin can configure guardrails for this team
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
litellm_model_table LiteLLM_ModelTable? @relation(fields: [model_id], references: [id])
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
2025-11-13 06:38:15 +08:00
projects LiteLLM_ProjectTable[]
@@index([organization_id])
@@index([team_alias])
@@index([created_at])
2025-11-13 06:38:15 +08:00
}
// Projects sit between teams and keys for use-case management
model LiteLLM_ProjectTable {
project_id String @id @default(uuid())
project_alias String?
2025-11-13 06:58:38 +08:00
description String?
2025-11-13 06:38:15 +08:00
team_id String?
budget_id String?
metadata Json @default("{}")
models String[]
spend Float @default(0.0)
model_spend Json @default("{}")
2025-11-13 06:58:38 +08:00
model_rpm_limit Json @default("{}")
model_tpm_limit Json @default("{}")
2025-11-13 06:38:15 +08:00
blocked Boolean @default(false)
object_permission_id String?
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
// Relations
litellm_team_table LiteLLM_TeamTable? @relation(fields: [team_id], references: [team_id])
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
keys LiteLLM_VerificationToken[]
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
}
// Audit table for deleted teams - preserves spend and team information for historical tracking
model LiteLLM_DeletedTeamTable {
id String @id @default(uuid())
team_id String // Original team_id
team_alias String?
organization_id String?
object_permission_id String?
admins String[]
members String[]
members_with_roles Json @default("{}")
metadata Json @default("{}")
max_budget Float?
soft_budget Float?
spend Float @default(0.0)
models String[]
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
budget_duration String?
budget_reset_at DateTime?
blocked Boolean @default(false)
model_spend Json @default("{}")
model_max_budget Json @default("{}")
router_settings Json? @default("{}")
team_member_permissions String[] @default([])
access_group_ids String[] @default([])
2026-01-24 05:16:58 +08:00
policies String[] @default([])
model_id Int? // id for LiteLLM_ModelTable -> stores team-level model aliases
allow_team_guardrail_config Boolean @default(false)
2026-02-15 01:49:11 +08:00
// Original timestamps from team creation/updates
created_at DateTime? @map("created_at")
updated_at DateTime? @map("updated_at")
// Deletion metadata
deleted_at DateTime @default(now()) @map("deleted_at")
deleted_by String? @map("deleted_by") // User who deleted the team
deleted_by_api_key String? @map("deleted_by_api_key") // API key hash that performed the deletion
litellm_changed_by String? @map("litellm_changed_by") // From litellm-changed-by header if provided
@@index([team_id])
@@index([deleted_at])
@@index([organization_id])
@@index([team_alias])
@@index([created_at])
}
2024-01-25 09:15:01 +08:00
// Track spend, rate limit, budget Users
model LiteLLM_UserTable {
2024-03-03 03:59:17 +08:00
user_id String @id
user_alias String?
2024-01-19 06:42:46 +08:00
team_id String?
2025-02-01 15:04:51 +08:00
sso_user_id String? @unique
organization_id String?
object_permission_id String?
password String?
teams String[] @default([])
2024-02-04 02:23:50 +08:00
user_role String?
max_budget Float?
spend Float @default(0.0)
user_email String?
models String[]
metadata Json @default("{}")
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
budget_duration String?
budget_reset_at DateTime?
allowed_cache_controls String[] @default([])
2026-01-24 05:16:58 +08:00
policies String[] @default([])
model_spend Json @default("{}")
model_max_budget Json @default("{}")
created_at DateTime? @default(now()) @map("created_at")
updated_at DateTime? @default(now()) @updatedAt @map("updated_at")
2024-10-09 17:48:18 +08:00
// relations
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
organization_memberships LiteLLM_OrganizationMembership[]
invitations_created LiteLLM_InvitationLink[] @relation("CreatedBy")
invitations_updated LiteLLM_InvitationLink[] @relation("UpdatedBy")
invitations_user LiteLLM_InvitationLink[] @relation("UserId")
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
}
model LiteLLM_ObjectPermissionTable {
object_permission_id String @id @default(uuid())
mcp_servers String[] @default([])
mcp_access_groups String[] @default([])
mcp_tool_permissions Json? // Tool-level permissions for MCP servers. Format: {"server_id": ["tool_name_1", "tool_name_2"]}
vector_stores String[] @default([])
agents String[] @default([])
agent_access_groups String[] @default([])
models String[] @default([])
[Feat] Add Tool Policies for AI Gateway (#22732) * fix: fix ui render * fix: fix minor bugs * refactor: use prisma functions instead of raw sql (safer) * fix(add-new-tiles-to-tool-policies): allow developer to see what's available * feat: ensure tool allowlist runs correctly for tool names + mcp's * refactor: more ui improvements * feat: working key tool blocking * feat(tools): show tool logs * refactor: backend code improvements * refactor: improve log viewer for tools * fix: address PR review feedback for tool access control - Add missing blocked_tools column to root schema.prisma (schema drift) - Invalidate ToolPolicyRegistry after policy mutations so changes take effect immediately - Remove dead code: unused get_effective_policies, get_tool_policies_cached, and helpers Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix: race condition in permission resolution and remove duplicate allowlist check - Use atomic update_many with object_permission_id=None to prevent concurrent requests from creating orphaned permission rows and losing tool blocks - Remove duplicate allowed_tools enforcement from guardrail (already enforced in auth layer via check_tools_allowlist) - Move inline uuid import to module level Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * update to account for userAgent * UI - Add ToolDetails * input/output policy * LiteLLM_PolicyAttachmentTable * LiteLLM_PolicyAttachmentTable * fix: add _enqueue_tool_registry_upsert * fix: tool mgmt endpoints * tool mgmt endpoints * Update tests/test_litellm/proxy/db/test_tool_registry_writer.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * Update tests/test_litellm/proxy/db/test_tool_registry_writer.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * Update tests/test_litellm/proxy/db/test_tool_registry_writer.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * fix: sync root schema.prisma and fix test_tool_registry_writer for input/output policy - Migrate root schema.prisma LiteLLM_ToolTable from call_policy to input_policy/output_policy, add missing user_agent and last_used_at columns (now consistent with litellm/proxy/schema.prisma and litellm-proxy-extras) - Fix SpendLogToolIndex comment across all three schema files - Fix all call_policy references in test_tool_registry_writer.py: swapped update_tool_policy arguments, wrong get_tools_by_names return type assertions, _mock_tool_row setting call_policy instead of input_policy Addresses Greptile review feedback on PR #22732. Made-with: Cursor --------- Co-authored-by: Krrish Dholakia <krrishdholakia@gmail.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
2026-03-04 12:22:20 +08:00
blocked_tools String[] @default([]) // Tool names blocked for any key/team/user with this permission
Litellm ishaan march23 - MCP Toolsets + GCP Caching fix (#25146) (#25155) * Litellm ishaan march23 - MCP Toolsets + GCP Caching fix (#25146) * feat(mcp): MCP Toolsets — curated tool subsets from one or more MCP servers (#24335) * feat(mcp): add LiteLLM_MCPToolsetTable and mcp_toolsets to ObjectPermissionTable * feat(mcp): add prisma migration for MCPToolset table * feat(mcp): add MCPToolset Python types * feat(mcp): add toolset_db.py with CRUD helpers for MCPToolset * feat(mcp): add toolset CRUD endpoints to mcp_management_endpoints * fix(mcp): skip allow_all_keys servers when explicit mcp_servers permission is set (toolset scope fix) * feat(mcp): add _apply_toolset_scope and toolset route handling in server.py * fix(mcp): resolve toolset names in responses API before fetching tools * feat(mcp): add mcp_toolsets field to LiteLLM_ObjectPermissionTable type * feat(mcp): register LiteLLM_MCPToolsetTable in prisma client initialization * feat(mcp): validate mcp_toolsets in key-vs-team permission check * feat(mcp): register toolset routes in proxy_server.py * feat(mcp): add MCPToolset and MCPToolsetTool TypeScript types * feat(mcp): add fetchMCPToolsets, createMCPToolset, updateMCPToolset, deleteMCPToolset API functions * feat(mcp): add useMCPToolsets React Query hook * feat(mcp): add toolsets (purple) as third option type in MCPServerSelector * feat(mcp): extract toolsets from combined MCP field in key form * feat(mcp): extract toolsets from combined MCP field in team form * feat(mcp): show toolsets section in MCPServerPermissions read view * feat(mcp): pass mcp_toolsets through object_permissions_view * feat(mcp): add MCPToolsetsTab component for creating and managing toolsets * feat(mcp): add Toolsets tab to mcp_servers.tsx * feat(mcp): pass mcpToolsets to playground chat and responses API calls * feat(mcp): generate correct server_url for toolsets in playground API calls * docs(mcp): add MCP Toolsets documentation * docs(mcp): add mcp_toolsets to sidebar * fix(mcp): replace x-mcp-toolset-id header with ContextVar to prevent client forgery * fix(mcp): use ContextVar + StreamingResponse for toolset MCP routes (fixes SSE streaming) * fix(mcp): cache toolset permission lookups to avoid per-request DB calls * test(mcp): add tests for toolset scope enforcement, ContextVar isolation, and access control * fix(mcp): cache toolset name lookups in MCPServerManager to avoid per-request DB calls * fix(mcp): prevent body_iter deadlock + use cached toolset lookup in responses API - _stream_mcp_asgi_response: add done callback to handler_task that puts the EOF sentinel on body_queue when the task exits, preventing body_iter from hanging forever if the handler raises after headers are sent. - litellm_proxy_mcp_handler: replace raw get_mcp_toolset_by_name() DB call with global_mcp_server_manager.get_toolset_by_name_cached() so toolset resolution uses the 60s TTL cache added for this purpose instead of hitting the DB on every responses-API request. * fix(mcp): toolset access control, asyncio fix, and real unit tests - server.py: _apply_toolset_scope now enforces that non-admin keys must have the requested toolset_id in their mcp_toolsets grant list; admin keys always bypass the check. - mcp_management_endpoints.py: three access-control fixes: * fetch_mcp_toolsets: non-admin keys with mcp_toolsets=None now return [] instead of all toolsets (only admins get 'all' when the field is absent) * fetch_mcp_toolset: non-admin keys that haven't been granted the requested toolset_id now get 403 instead of the full result * add_mcp_toolset: duplicate toolset_name now returns 409 Conflict instead of an opaque 500 - proxy_server.py: use asyncio.get_running_loop() instead of get_event_loop() inside an already-running coroutine (Python 3.10+). - test_mcp_toolset_scope.py: replace four hollow tests that only asserted local variable properties with real tests that call the production fetch_mcp_toolsets() and handle_streamable_http_mcp() functions with mocked dependencies. * fix(mcp): add mcp_toolsets to ObjectPermissionBase, fix multi-toolset overwrite, fix delete 404, allow standalone key toolsets * fix(mcp): add auth check on toolset resolution in responses API; union mcp_servers in _merge_toolset_permissions * fix(mcp): handle RecordNotFoundError in update_mcp_toolset; union direct servers with toolset servers * fix(mcp): use _user_has_admin_view; deny None mcp_toolsets for non-admin; use direct RecordNotFoundError import; fix docstring * fix(mcp): add @default(now()) to MCPToolsetTable.updated_at; fix test for non-admin toolset access * fix: use UniqueViolationError import; guard _ensure_eof for error/cancel only * fix(mcp): preserve mcp_access_groups in toolset scope, use shared Redis cache for toolset perms - Remove mcp_access_groups=[] from _apply_toolset_scope (server.py) and the responses API toolset path (litellm_proxy_mcp_handler.py). A key's access-group grants remain valid even when the request is scoped to a single toolset; clearing them silently revoked legitimate entitlements. - Switch resolve_toolset_tool_permissions and get_toolset_by_name_cached to use user_api_key_cache (Redis-backed DualCache in production) instead of per-instance in-memory dicts. Cache entries are now shared across workers, eliminating the per-worker stale-toolset-permission window flagged as a P1 by Greptile. - Use union merge (set union of tool names per server) when applying toolset permissions in the responses API path so direct-server tool restrictions are not overwritten by toolset permissions. * fix(mcp): return 404 when edit_mcp_toolset target does not exist * fix(mcp): align mcp_toolsets default to None in LiteLLM_ObjectPermissionTable * fix(mcp): admin toolset visibility, in-place tool name mutation, test helper coercion * fix(mcp): treat None/[] team mcp_toolsets as no restriction in key validation * fix(mcp): allow_all_keys backward compat, blocked_tools API write-path, efficient startup query * fix(mcp): use _mcp_active_toolset_id ContextVar to detect toolset scope, avoiding DB-default false-positive * fix(mcp): remove dead toolset cache stubs, log invalidation failures, align schema updated_at defaults * fix(mcp): deserialise MCPToolset from Redis cache hit, replace fastapi import in test * fix(mcp): evict name-cache on toolset mutation, 409 on rename conflict, warning-level list errors * fix(redis): regenerate GCP IAM token per connection for async cluster (#24426) * fix(redis): regenerate GCP IAM token per connection for async cluster clients Async RedisCluster was generating the IAM token once at startup and storing it as a static password. After the 1-hour GCP token TTL, any new connection (including to newly-discovered cluster nodes) would fail to authenticate. Fix: introduce GCPIAMCredentialProvider that implements redis-py's CredentialProvider protocol. It calls _generate_gcp_iam_access_token() on every new connection, matching what the sync redis_connect_func already does. async_redis.RedisCluster accepts a credential_provider kwarg which is invoked per-connection. * refactor(redis): move GCPIAMCredentialProvider to its own file Extract GCPIAMCredentialProvider and _generate_gcp_iam_access_token into litellm/_redis_credential_provider.py. _redis.py imports them from there, keeping the public API unchanged. * fix: address Greptile review issues - GCPIAMCredentialProvider now inherits from redis.credentials.CredentialProvider so redis-py's async path calls get_credentials_async() properly - move _redis_credential_provider import to top of _redis.py (PEP 8) - remove dead else-branch that silently no-oped (gcp_service_account from redis_kwargs.get() was always None since it's popped by _get_redis_client_logic) - remove mid-function 'from litellm import get_secret_str' inline import - remove unused 'call' import from test_redis.py * chore: retrigger CI/review * chore: sync schema.prisma copies from root * chore: sync schema.prisma copies from root * fix(proxy_server): use bounded asyncio.Queue with maxsize to prevent unbounded growth * fix(a2a/pydantic_ai): make api_base Optional to match base class signature * fix(a2a/pydantic_ai): make api_base Optional in handler and guard against None * fix(mcp): remove unused get_all_mcp_servers import * fix(mcp): remove unused MCPToolset import * refactor(mcp): extract toolset permission logic to reduce statement count below PLR0915 limit * fix(tests): update reload_servers_from_database tests to mock prisma directly --------- Co-authored-by: Ishaan Jaff <ishaanjaffer0324@gmail.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(toolset_db): lazy-import prisma to avoid ImportError when prisma not installed * fix(tests): update UI tests for toolset tab and updated empty state text * fix(tests): add get_mcp_server_by_name to fake_manager stub --------- Co-authored-by: Ishaan Jaff <ishaanjaffer0324@gmail.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
2026-04-05 07:23:21 +08:00
mcp_toolsets String[] @default([]) // Toolset IDs granted to this key/team/user
teams LiteLLM_TeamTable[]
2025-11-13 06:38:15 +08:00
projects LiteLLM_ProjectTable[]
verification_tokens LiteLLM_VerificationToken[]
organizations LiteLLM_OrganizationTable[]
users LiteLLM_UserTable[]
Mcp user permissions (#21462) * feat(schema.prisma): add object permissions for end users allows controlling if end user can call specific mcp servers * feat: cleanup for customer_endpoints support of object permission id * fix: cleanup str * feat(customers/): enforce end user can only call allowed mcps - if configured * docs: document customer/end user object permission usage * feat: enforce end user permissions on MCP tool calls This commit implements end user permission enforcement for MCP servers: 1. Always add server prefixes to MCP tool names - Removed conditional logic that only added prefixes when multiple servers existed - Now always adds server prefix for consistent tool naming across all scenarios - Updated 5 locations in server.py (list_tools, get_prompts, get_resources, get_resource_templates, get_prompt) 2. Created MCP End User Permission Guardrail Hook - New guardrail hook: litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission.py - Runs on post_call to validate tool calls in LLM responses - Extracts MCP server name from tool names (splits on first '-') - Checks if end_user_id has permissions for the MCP server - Raises GuardrailRaisedException if end user lacks permission - Supports both streaming and non-streaming responses 3. Added comprehensive tests - Test file: tests/test_litellm/proxy/guardrails/guardrail_hooks/test_mcp_end_user_permission.py - Tests cover: authorized/unauthorized tools, non-MCP tools, no end_user scenarios - Tests permission checking logic and exception raising The hook integrates with the existing MCPRequestHandler._get_allowed_mcp_servers_for_end_user to fetch end user permissions and enforce access control at the response level. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com> * refactor: remove redundant add_prefix variable assignments Simplified the code by removing intermediate `add_prefix` variable assignments and passing `True` directly to function calls since we now always add server prefixes. Changes: - Removed `add_prefix = True` variable assignments in 5 locations - Changed `add_prefix=add_prefix` to `add_prefix=True` in function calls - Added inline comments to clarify the behavior This makes the code more concise and clearer in intent. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com> * feat(auth_utils.py): support safety_identifier as a valid way of passing the end user id for responses api * feat(llms): ensure 'tools' is correctly updated for responses api * fix: fix greptile feedback * feat: transformation.py proper responses api tool handling for guardrail translation layer --------- Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>
2026-02-19 10:53:59 +08:00
end_users LiteLLM_EndUserTable[]
Agents - assign tools (#22064) * feat(proxy): add max_iterations limiter for agent session loops (#22058) Adds a new proxy hook that enforces a per-session cap on the number of LLM calls an agentic loop can make. Callers send a session_id with each request, and the hook counts calls per session, returning 429 when the configured max_iterations limit is exceeded. - Uses Redis Lua script for atomic increment (multi-instance safe) - Falls back to in-memory cache when Redis unavailable - Follows parallel_request_limiter_v3 pattern - Configurable via key metadata: {"max_iterations": 25} - Session counters auto-expire via TTL (default 1hr) Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> * feat: add new code execution dataset * feat(agent_endpoints/): allow giving agents keys * fix: ui fixes * feat: allow assigning mcp servers to agents * fix: eliminate duplicate DB queries in MCP agent auth and N+1 in agent listing (#22110) - Extract _get_agent_object_permission helper so _get_allowed_mcp_servers_for_agent and _get_agent_tool_permissions_for_server share a single DB fetch instead of each independently querying the same agent row (was 1+N queries per MCP request) - Use include={"object_permission": True} on find_many in get_all_agents_from_db to eagerly load permissions in one query instead of N+1 - Use include={"object_permission": True} on create/update/find_unique in all agent CRUD operations, removing attach_object_permission_to_dict follow-up calls Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-26 03:44:30 +08:00
agents_table LiteLLM_AgentsTable[]
}
// Holds the MCP server configuration
model LiteLLM_MCPServerTable {
2025-07-12 11:25:26 +08:00
server_id String @id @default(uuid())
server_name String?
2025-07-12 11:25:26 +08:00
alias String?
description String?
instructions String?
2025-07-12 11:25:26 +08:00
url String?
spec_path String?
2025-07-12 11:25:26 +08:00
transport String @default("sse")
auth_type String?
credentials Json? @default("{}")
2025-07-12 11:25:26 +08:00
created_at DateTime? @default(now()) @map("created_at")
created_by String?
updated_at DateTime? @default(now()) @updatedAt @map("updated_at")
updated_by String?
mcp_info Json? @default("{}")
mcp_access_groups String[]
allowed_tools String[] @default([])
2026-03-07 15:39:08 +08:00
tool_name_to_display_name Json? @default("{}")
tool_name_to_description Json? @default("{}")
2025-10-04 10:07:50 +08:00
extra_headers String[] @default([])
static_headers Json? @default("{}")
// Health check status
status String? @default("unknown")
last_health_check DateTime?
health_check_error String?
2025-07-12 11:25:26 +08:00
// Stdio-specific fields
command String?
args String[] @default([])
env Json? @default("{}")
authorization_url String?
token_url String?
registration_url String?
2026-01-05 14:49:09 +08:00
allow_all_keys Boolean @default(false)
fix(mcp): default available_on_public_internet to true (#22331) * fix(mcp): default available_on_public_internet to true MCPs were defaulting to private (available_on_public_internet=false) which was a breaking change. This reverts the default to public (true) across: - Pydantic models (AddMCPServerRequest, UpdateMCPServerRequest, LiteLLM_MCPServerTable) - Prisma schema @default - mcp_server_manager.py YAML config + DB loading fallbacks - UI form initialValue and setFieldValue defaults * fix(ui): add forceRender to Collapse.Panel so toggle defaults render correctly Ant Design's Collapse.Panel lazy-renders children by default. Without forceRender, the Form.Item for 'Available on Public Internet' isn't mounted when the useEffect fires form.setFieldValue, causing the Switch to visually show OFF even though the intended default is true. Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix(mcp): update remaining schema copies and MCPServer type default to true Missed in previous commit per Greptile review: - schema.prisma (root) - litellm-proxy-extras/litellm_proxy_extras/schema.prisma - litellm/types/mcp_server/mcp_server_manager.py MCPServer class * ui(mcp): reframe network access as 'Internal network only' restriction Replace scary 'Available on Public Internet' toggle with 'Internal network only' opt-in restriction. Toggle OFF (default) = all networks allowed. Toggle ON = restricted to internal network only. Auth is always required either way. - MCPPermissionManagement: new label/tooltip/description, invert display via getValueProps/getValueFromEvent so underlying available_on_public_internet value is unchanged - mcp_server_view: 'Public' → 'All networks', 'Internal' → 'Internal only' (orange) - mcp_server_columns: same badge updates --------- Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
2026-02-28 12:06:07 +08:00
available_on_public_internet Boolean @default(true)
2026-03-07 15:39:08 +08:00
is_byok Boolean @default(false)
byok_description String[] @default([])
byok_api_key_help_url String?
source_url String?
// BYOM submission lifecycle
approval_status String? @default("active")
submitted_by String?
submitted_at DateTime?
reviewed_at DateTime?
review_notes String?
@@index([approval_status])
2026-03-07 15:39:08 +08:00
}
Litellm ishaan march23 - MCP Toolsets + GCP Caching fix (#25146) (#25155) * Litellm ishaan march23 - MCP Toolsets + GCP Caching fix (#25146) * feat(mcp): MCP Toolsets — curated tool subsets from one or more MCP servers (#24335) * feat(mcp): add LiteLLM_MCPToolsetTable and mcp_toolsets to ObjectPermissionTable * feat(mcp): add prisma migration for MCPToolset table * feat(mcp): add MCPToolset Python types * feat(mcp): add toolset_db.py with CRUD helpers for MCPToolset * feat(mcp): add toolset CRUD endpoints to mcp_management_endpoints * fix(mcp): skip allow_all_keys servers when explicit mcp_servers permission is set (toolset scope fix) * feat(mcp): add _apply_toolset_scope and toolset route handling in server.py * fix(mcp): resolve toolset names in responses API before fetching tools * feat(mcp): add mcp_toolsets field to LiteLLM_ObjectPermissionTable type * feat(mcp): register LiteLLM_MCPToolsetTable in prisma client initialization * feat(mcp): validate mcp_toolsets in key-vs-team permission check * feat(mcp): register toolset routes in proxy_server.py * feat(mcp): add MCPToolset and MCPToolsetTool TypeScript types * feat(mcp): add fetchMCPToolsets, createMCPToolset, updateMCPToolset, deleteMCPToolset API functions * feat(mcp): add useMCPToolsets React Query hook * feat(mcp): add toolsets (purple) as third option type in MCPServerSelector * feat(mcp): extract toolsets from combined MCP field in key form * feat(mcp): extract toolsets from combined MCP field in team form * feat(mcp): show toolsets section in MCPServerPermissions read view * feat(mcp): pass mcp_toolsets through object_permissions_view * feat(mcp): add MCPToolsetsTab component for creating and managing toolsets * feat(mcp): add Toolsets tab to mcp_servers.tsx * feat(mcp): pass mcpToolsets to playground chat and responses API calls * feat(mcp): generate correct server_url for toolsets in playground API calls * docs(mcp): add MCP Toolsets documentation * docs(mcp): add mcp_toolsets to sidebar * fix(mcp): replace x-mcp-toolset-id header with ContextVar to prevent client forgery * fix(mcp): use ContextVar + StreamingResponse for toolset MCP routes (fixes SSE streaming) * fix(mcp): cache toolset permission lookups to avoid per-request DB calls * test(mcp): add tests for toolset scope enforcement, ContextVar isolation, and access control * fix(mcp): cache toolset name lookups in MCPServerManager to avoid per-request DB calls * fix(mcp): prevent body_iter deadlock + use cached toolset lookup in responses API - _stream_mcp_asgi_response: add done callback to handler_task that puts the EOF sentinel on body_queue when the task exits, preventing body_iter from hanging forever if the handler raises after headers are sent. - litellm_proxy_mcp_handler: replace raw get_mcp_toolset_by_name() DB call with global_mcp_server_manager.get_toolset_by_name_cached() so toolset resolution uses the 60s TTL cache added for this purpose instead of hitting the DB on every responses-API request. * fix(mcp): toolset access control, asyncio fix, and real unit tests - server.py: _apply_toolset_scope now enforces that non-admin keys must have the requested toolset_id in their mcp_toolsets grant list; admin keys always bypass the check. - mcp_management_endpoints.py: three access-control fixes: * fetch_mcp_toolsets: non-admin keys with mcp_toolsets=None now return [] instead of all toolsets (only admins get 'all' when the field is absent) * fetch_mcp_toolset: non-admin keys that haven't been granted the requested toolset_id now get 403 instead of the full result * add_mcp_toolset: duplicate toolset_name now returns 409 Conflict instead of an opaque 500 - proxy_server.py: use asyncio.get_running_loop() instead of get_event_loop() inside an already-running coroutine (Python 3.10+). - test_mcp_toolset_scope.py: replace four hollow tests that only asserted local variable properties with real tests that call the production fetch_mcp_toolsets() and handle_streamable_http_mcp() functions with mocked dependencies. * fix(mcp): add mcp_toolsets to ObjectPermissionBase, fix multi-toolset overwrite, fix delete 404, allow standalone key toolsets * fix(mcp): add auth check on toolset resolution in responses API; union mcp_servers in _merge_toolset_permissions * fix(mcp): handle RecordNotFoundError in update_mcp_toolset; union direct servers with toolset servers * fix(mcp): use _user_has_admin_view; deny None mcp_toolsets for non-admin; use direct RecordNotFoundError import; fix docstring * fix(mcp): add @default(now()) to MCPToolsetTable.updated_at; fix test for non-admin toolset access * fix: use UniqueViolationError import; guard _ensure_eof for error/cancel only * fix(mcp): preserve mcp_access_groups in toolset scope, use shared Redis cache for toolset perms - Remove mcp_access_groups=[] from _apply_toolset_scope (server.py) and the responses API toolset path (litellm_proxy_mcp_handler.py). A key's access-group grants remain valid even when the request is scoped to a single toolset; clearing them silently revoked legitimate entitlements. - Switch resolve_toolset_tool_permissions and get_toolset_by_name_cached to use user_api_key_cache (Redis-backed DualCache in production) instead of per-instance in-memory dicts. Cache entries are now shared across workers, eliminating the per-worker stale-toolset-permission window flagged as a P1 by Greptile. - Use union merge (set union of tool names per server) when applying toolset permissions in the responses API path so direct-server tool restrictions are not overwritten by toolset permissions. * fix(mcp): return 404 when edit_mcp_toolset target does not exist * fix(mcp): align mcp_toolsets default to None in LiteLLM_ObjectPermissionTable * fix(mcp): admin toolset visibility, in-place tool name mutation, test helper coercion * fix(mcp): treat None/[] team mcp_toolsets as no restriction in key validation * fix(mcp): allow_all_keys backward compat, blocked_tools API write-path, efficient startup query * fix(mcp): use _mcp_active_toolset_id ContextVar to detect toolset scope, avoiding DB-default false-positive * fix(mcp): remove dead toolset cache stubs, log invalidation failures, align schema updated_at defaults * fix(mcp): deserialise MCPToolset from Redis cache hit, replace fastapi import in test * fix(mcp): evict name-cache on toolset mutation, 409 on rename conflict, warning-level list errors * fix(redis): regenerate GCP IAM token per connection for async cluster (#24426) * fix(redis): regenerate GCP IAM token per connection for async cluster clients Async RedisCluster was generating the IAM token once at startup and storing it as a static password. After the 1-hour GCP token TTL, any new connection (including to newly-discovered cluster nodes) would fail to authenticate. Fix: introduce GCPIAMCredentialProvider that implements redis-py's CredentialProvider protocol. It calls _generate_gcp_iam_access_token() on every new connection, matching what the sync redis_connect_func already does. async_redis.RedisCluster accepts a credential_provider kwarg which is invoked per-connection. * refactor(redis): move GCPIAMCredentialProvider to its own file Extract GCPIAMCredentialProvider and _generate_gcp_iam_access_token into litellm/_redis_credential_provider.py. _redis.py imports them from there, keeping the public API unchanged. * fix: address Greptile review issues - GCPIAMCredentialProvider now inherits from redis.credentials.CredentialProvider so redis-py's async path calls get_credentials_async() properly - move _redis_credential_provider import to top of _redis.py (PEP 8) - remove dead else-branch that silently no-oped (gcp_service_account from redis_kwargs.get() was always None since it's popped by _get_redis_client_logic) - remove mid-function 'from litellm import get_secret_str' inline import - remove unused 'call' import from test_redis.py * chore: retrigger CI/review * chore: sync schema.prisma copies from root * chore: sync schema.prisma copies from root * fix(proxy_server): use bounded asyncio.Queue with maxsize to prevent unbounded growth * fix(a2a/pydantic_ai): make api_base Optional to match base class signature * fix(a2a/pydantic_ai): make api_base Optional in handler and guard against None * fix(mcp): remove unused get_all_mcp_servers import * fix(mcp): remove unused MCPToolset import * refactor(mcp): extract toolset permission logic to reduce statement count below PLR0915 limit * fix(tests): update reload_servers_from_database tests to mock prisma directly --------- Co-authored-by: Ishaan Jaff <ishaanjaffer0324@gmail.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(toolset_db): lazy-import prisma to avoid ImportError when prisma not installed * fix(tests): update UI tests for toolset tab and updated empty state text * fix(tests): add get_mcp_server_by_name to fake_manager stub --------- Co-authored-by: Ishaan Jaff <ishaanjaffer0324@gmail.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
2026-04-05 07:23:21 +08:00
// Named collection of {server_id, tool_name} pairs that can be granted to keys/teams
model LiteLLM_MCPToolsetTable {
toolset_id String @id @default(uuid())
toolset_name String @unique
description String?
tools Json @default("[]") // [{server_id: string, tool_name: string}]
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
}
2026-03-07 15:39:08 +08:00
// Per-user BYOK credentials for MCP servers
model LiteLLM_MCPUserCredentials {
id String @id @default(uuid())
user_id String
server_id String
credential_b64 String
created_at DateTime @default(now()) @map("created_at")
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
@@unique([user_id, server_id])
}
2024-01-25 09:15:01 +08:00
// Generate Tokens for Proxy
model LiteLLM_VerificationToken {
2024-03-03 03:59:17 +08:00
token String @id
2024-01-27 12:53:03 +08:00
key_name String?
key_alias String?
soft_budget_cooldown Boolean @default(false) // key-level state on if budget alerts need to be cooled down
spend Float @default(0.0)
expires DateTime?
models String[]
aliases Json @default("{}")
config Json @default("{}")
router_settings Json? @default("{}")
user_id String?
2024-01-19 05:34:51 +08:00
team_id String?
Agents - assign tools (#22064) * feat(proxy): add max_iterations limiter for agent session loops (#22058) Adds a new proxy hook that enforces a per-session cap on the number of LLM calls an agentic loop can make. Callers send a session_id with each request, and the hook counts calls per session, returning 429 when the configured max_iterations limit is exceeded. - Uses Redis Lua script for atomic increment (multi-instance safe) - Falls back to in-memory cache when Redis unavailable - Follows parallel_request_limiter_v3 pattern - Configurable via key metadata: {"max_iterations": 25} - Session counters auto-expire via TTL (default 1hr) Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> * feat: add new code execution dataset * feat(agent_endpoints/): allow giving agents keys * fix: ui fixes * feat: allow assigning mcp servers to agents * fix: eliminate duplicate DB queries in MCP agent auth and N+1 in agent listing (#22110) - Extract _get_agent_object_permission helper so _get_allowed_mcp_servers_for_agent and _get_agent_tool_permissions_for_server share a single DB fetch instead of each independently querying the same agent row (was 1+N queries per MCP request) - Use include={"object_permission": True} on find_many in get_all_agents_from_db to eagerly load permissions in one query instead of N+1 - Use include={"object_permission": True} on create/update/find_unique in all agent CRUD operations, removing attach_object_permission_to_dict follow-up calls Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-26 03:44:30 +08:00
agent_id String?
2025-11-13 06:38:15 +08:00
project_id String?
permissions Json @default("{}")
max_parallel_requests Int?
metadata Json @default("{}")
blocked Boolean?
tpm_limit BigInt?
rpm_limit BigInt?
max_budget Float?
budget_duration String?
budget_reset_at DateTime?
allowed_cache_controls String[] @default([])
allowed_routes String[] @default([])
2026-01-24 05:16:58 +08:00
policies String[] @default([])
access_group_ids String[] @default([])
2024-02-17 07:44:34 +08:00
model_spend Json @default("{}")
model_max_budget Json @default("{}")
2024-03-03 04:25:40 +08:00
budget_id String?
organization_id String?
object_permission_id String?
2024-10-25 11:19:29 +08:00
created_at DateTime? @default(now()) @map("created_at")
created_by String?
2024-10-25 11:19:29 +08:00
updated_at DateTime? @default(now()) @updatedAt @map("updated_at")
updated_by String?
last_active DateTime? // When this key was last used
2025-09-25 12:37:56 +08:00
rotation_count Int? @default(0) // Number of times key has been rotated
auto_rotate Boolean? @default(false) // Whether this key should be auto-rotated
rotation_interval String? // How often to rotate (e.g., "30d", "90d")
last_rotation_at DateTime? // When this key was last rotated
key_rotation_at DateTime? // When this key should next be rotated
budget_limits Json? // per-model budget limits for the key
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
2025-11-13 06:38:15 +08:00
litellm_project_table LiteLLM_ProjectTable? @relation(fields: [project_id], references: [project_id])
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
2026-03-07 15:39:08 +08:00
jwt_key_mappings LiteLLM_JWTKeyMapping[]
// SELECT COUNT(*) FROM (SELECT "public"."LiteLLM_VerificationToken"."token" FROM "public"."LiteLLM_VerificationToken" WHERE ("public"."LiteLLM_VerificationToken"."user_id" = $1 AND ("public"."LiteLLM_VerificationToken"."team_id" IS NULL OR "public"."LiteLLM_VerificationToken"."team_id" <> $2)) OFFSET $3 ) AS "sub"
// SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE "public"."LiteLLM_VerificationToken"."user_id" = $1 OFFSET $2
@@index([user_id, team_id])
// SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE "public"."LiteLLM_VerificationToken"."team_id" = $1 OFFSET $2
@@index([team_id])
// SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE (("public"."LiteLLM_VerificationToken"."expires" IS NULL OR "public"."LiteLLM_VerificationToken"."expires" > $1) AND "public"."LiteLLM_VerificationToken"."budget_reset_at" < $2) OFFSET $3
@@index([budget_reset_at, expires])
}
2026-03-07 15:39:08 +08:00
model LiteLLM_JWTKeyMapping {
id String @id @default(uuid())
jwt_claim_name String // e.g. "sub", "email"
jwt_claim_value String // The claim value to match
token String // Hashed virtual key (FK)
description String?
is_active Boolean @default(true)
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token])
@@unique([jwt_claim_name, jwt_claim_value])
@@index([jwt_claim_name, jwt_claim_value, is_active])
}
// Deprecated keys during grace period - allows old key to work until revoke_at
model LiteLLM_DeprecatedVerificationToken {
id String @id @default(uuid())
token String // Hashed old key
active_token_id String // Current token hash in LiteLLM_VerificationToken
revoke_at DateTime // When the old key stops working
created_at DateTime @default(now()) @map("created_at")
@@unique([token])
@@index([token, revoke_at])
@@index([revoke_at])
}
// Audit table for deleted keys - preserves spend and key information for historical tracking
model LiteLLM_DeletedVerificationToken {
id String @id @default(uuid())
token String // Original token (hashed)
key_name String?
key_alias String?
soft_budget_cooldown Boolean @default(false)
spend Float @default(0.0)
expires DateTime?
models String[]
aliases Json @default("{}")
config Json @default("{}")
user_id String?
team_id String?
2026-02-27 12:29:43 +08:00
agent_id String?
2026-02-20 02:50:23 +08:00
project_id String?
permissions Json @default("{}")
max_parallel_requests Int?
metadata Json @default("{}")
blocked Boolean?
tpm_limit BigInt?
rpm_limit BigInt?
max_budget Float?
budget_duration String?
budget_reset_at DateTime?
allowed_cache_controls String[] @default([])
allowed_routes String[] @default([])
2026-01-24 05:16:58 +08:00
policies String[] @default([])
access_group_ids String[] @default([])
model_spend Json @default("{}")
model_max_budget Json @default("{}")
router_settings Json? @default("{}")
budget_id String?
organization_id String?
object_permission_id String?
created_at DateTime? // Original creation timestamp
created_by String? // Original creator
updated_at DateTime? // Last update timestamp before deletion
updated_by String? // Last user who updated before deletion
last_active DateTime? // When this key was last used before deletion
rotation_count Int? @default(0)
auto_rotate Boolean? @default(false)
rotation_interval String?
last_rotation_at DateTime?
key_rotation_at DateTime?
2026-02-15 01:49:11 +08:00
// Deletion metadata
deleted_at DateTime @default(now()) @map("deleted_at")
deleted_by String? @map("deleted_by") // User who deleted the key
deleted_by_api_key String? @map("deleted_by_api_key") // API key hash that performed the deletion
litellm_changed_by String? @map("litellm_changed_by") // From litellm-changed-by header if provided
@@index([token])
@@index([deleted_at])
@@index([user_id])
@@index([team_id])
@@index([organization_id])
@@index([key_alias])
@@index([created_at])
}
model LiteLLM_EndUserTable {
2024-03-17 05:15:01 +08:00
user_id String @id
alias String? // admin-facing alias
spend Float @default(0.0)
allowed_model_region String? // require all user requests to use models in this specific region
default_model String? // use along with 'allowed_model_region'. if no available model in region, default to this model.
budget_id String?
Mcp user permissions (#21462) * feat(schema.prisma): add object permissions for end users allows controlling if end user can call specific mcp servers * feat: cleanup for customer_endpoints support of object permission id * fix: cleanup str * feat(customers/): enforce end user can only call allowed mcps - if configured * docs: document customer/end user object permission usage * feat: enforce end user permissions on MCP tool calls This commit implements end user permission enforcement for MCP servers: 1. Always add server prefixes to MCP tool names - Removed conditional logic that only added prefixes when multiple servers existed - Now always adds server prefix for consistent tool naming across all scenarios - Updated 5 locations in server.py (list_tools, get_prompts, get_resources, get_resource_templates, get_prompt) 2. Created MCP End User Permission Guardrail Hook - New guardrail hook: litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission.py - Runs on post_call to validate tool calls in LLM responses - Extracts MCP server name from tool names (splits on first '-') - Checks if end_user_id has permissions for the MCP server - Raises GuardrailRaisedException if end user lacks permission - Supports both streaming and non-streaming responses 3. Added comprehensive tests - Test file: tests/test_litellm/proxy/guardrails/guardrail_hooks/test_mcp_end_user_permission.py - Tests cover: authorized/unauthorized tools, non-MCP tools, no end_user scenarios - Tests permission checking logic and exception raising The hook integrates with the existing MCPRequestHandler._get_allowed_mcp_servers_for_end_user to fetch end user permissions and enforce access control at the response level. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com> * refactor: remove redundant add_prefix variable assignments Simplified the code by removing intermediate `add_prefix` variable assignments and passing `True` directly to function calls since we now always add server prefixes. Changes: - Removed `add_prefix = True` variable assignments in 5 locations - Changed `add_prefix=add_prefix` to `add_prefix=True` in function calls - Added inline comments to clarify the behavior This makes the code more concise and clearer in intent. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com> * feat(auth_utils.py): support safety_identifier as a valid way of passing the end user id for responses api * feat(llms): ensure 'tools' is correctly updated for responses api * fix: fix greptile feedback * feat: transformation.py proper responses api tool handling for guardrail translation layer --------- Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>
2026-02-19 10:53:59 +08:00
object_permission_id String?
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
Mcp user permissions (#21462) * feat(schema.prisma): add object permissions for end users allows controlling if end user can call specific mcp servers * feat: cleanup for customer_endpoints support of object permission id * fix: cleanup str * feat(customers/): enforce end user can only call allowed mcps - if configured * docs: document customer/end user object permission usage * feat: enforce end user permissions on MCP tool calls This commit implements end user permission enforcement for MCP servers: 1. Always add server prefixes to MCP tool names - Removed conditional logic that only added prefixes when multiple servers existed - Now always adds server prefix for consistent tool naming across all scenarios - Updated 5 locations in server.py (list_tools, get_prompts, get_resources, get_resource_templates, get_prompt) 2. Created MCP End User Permission Guardrail Hook - New guardrail hook: litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission.py - Runs on post_call to validate tool calls in LLM responses - Extracts MCP server name from tool names (splits on first '-') - Checks if end_user_id has permissions for the MCP server - Raises GuardrailRaisedException if end user lacks permission - Supports both streaming and non-streaming responses 3. Added comprehensive tests - Test file: tests/test_litellm/proxy/guardrails/guardrail_hooks/test_mcp_end_user_permission.py - Tests cover: authorized/unauthorized tools, non-MCP tools, no end_user scenarios - Tests permission checking logic and exception raising The hook integrates with the existing MCPRequestHandler._get_allowed_mcp_servers_for_end_user to fetch end user permissions and enforce access control at the response level. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com> * refactor: remove redundant add_prefix variable assignments Simplified the code by removing intermediate `add_prefix` variable assignments and passing `True` directly to function calls since we now always add server prefixes. Changes: - Removed `add_prefix = True` variable assignments in 5 locations - Changed `add_prefix=add_prefix` to `add_prefix=True` in function calls - Added inline comments to clarify the behavior This makes the code more concise and clearer in intent. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com> * feat(auth_utils.py): support safety_identifier as a valid way of passing the end user id for responses api * feat(llms): ensure 'tools' is correctly updated for responses api * fix: fix greptile feedback * feat: transformation.py proper responses api tool handling for guardrail translation layer --------- Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>
2026-02-19 10:53:59 +08:00
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
blocked Boolean @default(false)
}
// Track tags with budgets and spend
model LiteLLM_TagTable {
tag_name String @id
description String?
models String[]
model_info Json? // maps model_id to model_name
spend Float @default(0.0)
budget_id String?
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
created_at DateTime @default(now()) @map("created_at")
created_by String?
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
}
2024-01-25 09:15:01 +08:00
// store proxy config.yaml
model LiteLLM_Config {
param_name String @id
param_value Json?
2024-01-19 02:04:34 +08:00
}
2024-01-25 09:15:01 +08:00
// View spend, model, api_key per request
2024-01-19 02:04:34 +08:00
model LiteLLM_SpendLogs {
2024-03-03 03:59:17 +08:00
request_id String @id
2024-01-19 02:04:34 +08:00
call_type String
api_key String @default ("") // Hashed API Token. Not the actual Virtual Key. Equivalent to 'token' column in LiteLLM_VerificationToken
2024-01-19 05:16:25 +08:00
spend Float @default(0.0)
2024-01-27 05:23:51 +08:00
total_tokens Int @default(0)
prompt_tokens Int @default(0)
completion_tokens Int @default(0)
2024-01-19 02:04:34 +08:00
startTime DateTime // Assuming start_time is a DateTime field
endTime DateTime // Assuming end_time is a DateTime field
request_duration_ms Int?
completionStartTime DateTime? // Assuming completionStartTime is a DateTime field
2024-01-19 02:04:34 +08:00
model String @default("")
model_id String? @default("") // the model id stored in proxy model db
model_group String? @default("") // public model_name / model_group
custom_llm_provider String? @default("") // litellm used custom_llm_provider
api_base String? @default("")
user String? @default("")
2025-11-13 06:38:15 +08:00
metadata Json? @default("{}") // project_id stored here
cache_hit String? @default("")
cache_key String? @default("")
request_tags Json? @default("[]")
team_id String?
organization_id String?
2024-03-01 11:21:57 +08:00
end_user String?
2024-07-09 01:16:58 +08:00
requester_ip_address String?
messages Json? @default("{}")
response Json? @default("{}")
session_id String?
status String?
mcp_namespaced_tool_name String?
agent_id String?
proxy_server_request Json? @default("{}")
@@index([startTime])
[Release Fix] (#22411) * fix(lint): suppress PLR0915 for 3 complex methods that exceed 50-statement limit - streaming_iterator.py: _process_event (84 statements) - transformation.py: translate_messages_to_responses_input (51 statements) - transformation.py: transform_realtime_response (54 statements) Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix(mypy): resolve type errors in public_endpoints, user_api_key_auth, common_utils, transformation - public_endpoints.py: fix _cached_endpoints type annotation - user_api_key_auth.py: accept Optional[str] for end_user_id parameter - common_utils.py: add NewProjectRequest/UpdateProjectRequest to Union type - transformation.py: add ChatCompletionRedactedThinkingBlock and list[Any] to content type Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix(proxy-extras): bump version to 0.4.50 and sync schema - Bump litellm-proxy-extras from 0.4.49 to 0.4.50 - Sync schema.prisma with main proxy schema - Includes new LiteLLM_ClaudeCodePluginTable model - Includes new @@index([startTime, request_id]) on SpendLogs - Update version references in requirements.txt and pyproject.toml Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix(router): use string id in test_add_deployment and add defensive str() in register_model - Change test to use string '100' instead of int 100 for model_info.id - Add str() conversion in register_model to prevent AttributeError on non-string keys Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix(security): update minimatch to 10.2.4 to fix CVE-2026-27903 and CVE-2026-27904 - Run npm audit fix in docs/my-website - Updates minimatch from 10.2.1 to 10.2.4 (fixes HIGH severity ReDoS vulnerabilities) Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix(test): update realtime guardrail test assertions to match actual guardrail behavior - test_text_message_blocked_by_guardrail_no_ai_response: allow guardrail's own block message text in response.done (previously expected empty content) - test_voice_transcript_blocked_by_guardrail: allow guardrail to send response.cancel + block message + response.create flow (previously expected no response.create) Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix: revert proxy-extras version in requirements.txt and pyproject.toml The litellm-proxy-extras 0.4.50 is not published to PyPI yet, so consumer references must stay at 0.4.49. Only the source package pyproject.toml should be bumped to 0.4.50 for the publish_proxy_extras CI job. Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix: make transcript delta check optional in voice guardrail test The guardrail sends an error event (guardrail_violation) when blocking voice transcripts; it does not always produce transcript deltas. Remove the assertion requiring response.audio_transcript.delta since the error event is the primary signal that blocked content was handled. Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * Add missing env keys to documentation: LITELLM_MAX_STREAMING_DURATION_SECONDS and LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES These two environment variables were used in code but not documented in the environment variables reference section of config_settings.md, causing the test_env_keys.py CI test to fail. Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * Fix 13 mypy type errors across 6 files - in_flight_requests_middleware.py: Fix type: ignore error codes from [union-attr] to [attr-defined], add [arg-type] for Gauge **kwargs - transformation.py: Add [assignment] ignore for output_format reassignment, add fallback empty string for tool use id to fix arg-type - responses/main.py: Remove redundant type annotation on second secret_fields assignment to fix no-redef - streaming_iterator.py: Add [assignment] ignores for intermediate cache token assignments - handler.py: Add [typeddict-item] ignore for AnthropicMessagesRequest construction from dict - public_endpoints.py: Add [arg-type] ignore for _load_endpoints() return type mismatch with SupportedEndpoint model Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix: add auth overrides to spend tracking tests, fix realtime guardrail assertion, update UI minimatch - Add app.dependency_overrides for user_api_key_auth in 4 spend tracking tests that were returning 401 Unauthorized (error_code, error_message, error_code_and_key_alias, key_hash) - Fix realtime guardrail test to check ANY error event for guardrail_violation instead of just the first (OpenAI may send its own errors first) - Update ui/litellm-dashboard/package-lock.json to fix minimatch vulnerability Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * Fix failing MCP e2e and create_mcp_server UI tests Test 1 (test_independent_clients_no_shared_session): - Add allow_all_keys: true to MCP servers in test config. With master_key and no DB, get_allowed_mcp_servers returned empty, causing 0 tools and 403 on tool calls. allow_all_keys bypasses per-key restrictions. - Add asyncio.sleep(0.5) between client connections to allow MCP SDK TaskGroup cleanup and avoid ExceptionGroup on connection close (MCP #915). Test 2 (create_mcp_server 'auth value is provided'): - Use userEvent.setup({ delay: null }) for instant keystrokes to avoid timeout from default typing delay on CI. - Increase per-test timeout to 15000ms for CI environments. Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix: stabilize proxy unit tests for parallel execution - test_response_polling_handler: add xdist_group to prevent heavy import OOM - test_db_schema_migration: use temp dir for worker isolation, sync schema.prisma index - test_custom_tokenizer_bug: use lighter tokenizer to prevent OOM in parallel Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix: add auth overrides to more spend tracking and model info tests - Fix test_ui_view_spend_logs_pagination missing auth override (401) - Fix test_view_spend_tags missing auth override (401) - Fix test_view_spend_tags_no_database missing auth override (401) - Fix test_empty_model_list.py to use app.dependency_overrides instead of patch() for FastAPI dependency injection auth Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix(test): use patch.object for aiohttp transport test to work in parallel execution The @patch decorator was not intercepting the static method call in parallel xdist workers. Using patch.object on the directly-imported class is more reliable. Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix(security): update minimatch from 10.2.1 to 10.2.4 in Dockerfile The Docker image was explicitly pinning minimatch@10.2.1 which has HIGH severity ReDoS vulnerabilities (GHSA-7r86-cg39-jmmj, GHSA-23c5-xmqv-rm74). Update to 10.2.4 which includes fixes for both CVEs. Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix(ui): prevent MCP and TeamInfo test timeouts on CI - Add userEvent.setup({ delay: null }) to all tests using userEvent in both files - Add timeout: 15000 to tests with significant user interaction (typing, multiple clicks) - Fixes: create_mcp_server Bearer Token test, TeamInfo cancel button test Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix: stabilize parallel test execution and aiohttp transport test - test_aiohttp_handler: rewrite transport test to not rely on static method mock (consistently fails in parallel xdist workers) - test_proxy_cli: add xdist_group to prevent timeout during heavy imports - test_swagger_chat_completions: add xdist_group to prevent timeout Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix(security): add serialize-javascript override to fix GHSA-5c6j-r48x-rmvq Add npm override for serialize-javascript>=7.0.3 in docs/my-website to fix HIGH severity RCE vulnerability via RegExp.flags. Also bump minimatch override to >=10.2.4. Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * Fix flaky tests: remove broken Vertex model, add retries for Anthropic - Remove vertex_ai/meta/llama-4-scout-17b-16e-instruct-maas from test_partner_models_httpx_streaming - consistently returns 400 BadRequest - Add @pytest.mark.flaky(retries=6, delay=10) to test_function_call_parsing for transient Anthropic API overload errors - Add @pytest.mark.flaky(retries=6, delay=10) to test_openai_stream_options_call for transient Anthropic InternalServerError Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix(ci): add xdist_group(proxy_heavy) to prevent OOM in parallel proxy tests - Add pytestmark = pytest.mark.xdist_group('proxy_heavy') to test_proxy_utils.py - Change test_db_schema_migration.py from schema_migration to proxy_heavy group - Add @pytest.mark.xdist_group('proxy_heavy') to test_proxy_server.py::test_health Groups heavy proxy tests to run on same worker, avoiding worker OOM crashes. Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * Fix vertex AI qwen global endpoint test to mock vertexai module import The test_vertex_ai_qwen_global_endpoint_url test was failing because the VertexAIPartnerModels.completion() method tries to 'import vertexai' before any of the mocked code runs. In environments without google-cloud-aiplatform installed, this import fails with a VertexAIError(status_code=400). Fix by: - Adding patch.dict('sys.modules', {'vertexai': MagicMock()}) to mock the vertexai module import - Adding vertex_ai_location parameter to the acompletion call for completeness Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix(ci): add xdist_group to health endpoint and watsonx tests for parallel stability - test_health_liveliness_endpoint: add xdist_group('proxy_health') to prevent timeout - test_watsonx_gpt_oss tests: add xdist_group('watsonx_heavy') to prevent mock interference Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix(test): pre-populate WatsonX IAM token cache to prevent parallel test interference The watsonx prompt transformation test was failing in parallel execution because litellm.module_level_client.post mock was being interfered with by other tests. Pre-populating the IAM token cache avoids the HTTP call entirely. Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix(test): add spend data polling with retries for e2e pass-through tests - test_vertex_with_spend.test.js: Replace 15s fixed wait with polling loop (up to 6 attempts, 10s apart) for spend data to appear in DB - Increase test timeout from 25s to 90s to accommodate polling - base_anthropic_messages_tool_search_test.py: Add flaky(retries=3) for streaming test that depends on live Anthropic API Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix(ci): reduce parallel workers from 8 to 4 for proxy tests to prevent OOM - litellm_proxy_unit_testing_part2: -n 8 -> -n 4 - litellm_mapped_tests_proxy_part2: -n 8 -> -n 4, timeout 60 -> 120 - Worker crashes consistently caused by too many parallel proxy tests each loading the full FastAPI app and heavy dependency tree Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix(db): add migration for SpendLogs composite index (startTime, request_id) The @@index([startTime, request_id]) was added to schema.prisma but had no corresponding migration. This caused test_aaaasschema_migration_check to fail because prisma migrate diff detected the missing index. Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix(db): add migration for MCP available_on_public_internet default change to true The schema.prisma changed the default for available_on_public_internet from false to true, but no migration was created. This caused the schema migration test to detect drift. Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix(test): increase server wait time and add retry to flaky external API tests - test_basic_python_version.py: increase server startup wait from 60s to 90s for slower CI environments (fixes installing_litellm_on_python_3_13) - test_a2a_agent.py: add flaky(retries=3, delay=5) for non-streaming test that depends on live A2A agent endpoint Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix(test): add flaky retries to all intermittent external API tests for 0-fail CI Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> * fix(test): add auth overrides to file endpoint tests that return 500 The test_target_storage tests were getting 500 because the FastAPI auth dependency wasn't overridden. Added app.dependency_overrides for proper auth bypass in test environment. Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com> --------- Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
2026-03-01 01:46:35 +08:00
@@index([startTime, request_id])
@@index([end_user])
@@index([session_id])
}
2024-05-01 02:42:17 +08:00
// View spend, model, api_key per request
model LiteLLM_ErrorLogs {
request_id String @id @default(uuid())
startTime DateTime // Assuming start_time is a DateTime field
endTime DateTime // Assuming end_time is a DateTime field
api_base String @default("")
2024-05-01 04:11:09 +08:00
model_group String @default("") // public model_name / model_group
litellm_model_name String @default("") // model passed to litellm
2024-05-01 02:42:17 +08:00
model_id String @default("") // ID of model in ProxyModelTable
request_kwargs Json @default("{}")
2024-05-01 03:31:19 +08:00
exception_type String @default("")
exception_string String @default("")
status_code String @default("")
2024-05-01 02:42:17 +08:00
}
// Beta - allow team members to request access to a model
2024-02-20 08:53:40 +08:00
model LiteLLM_UserNotifications {
2024-03-03 03:59:17 +08:00
request_id String @id
user_id String
models String[]
justification String
2024-02-20 08:53:40 +08:00
status String // approved, disapproved, pending
}
model LiteLLM_TeamMembership {
// Use this table to track the Internal User's Spend within a Team + Set Budgets, rpm limits for the user within the team
user_id String
team_id String
spend Float @default(0.0)
total_spend Float @default(0.0)
budget_id String?
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
@@id([user_id, team_id])
}
2024-10-09 17:48:18 +08:00
model LiteLLM_OrganizationMembership {
// Use this table to track Internal User and Organization membership. Helps tracking a users role within an Organization
2024-10-09 17:55:27 +08:00
user_id String
organization_id String
2024-10-09 17:48:18 +08:00
user_role String?
spend Float? @default(0.0)
budget_id String?
created_at DateTime? @default(now()) @map("created_at")
updated_at DateTime? @default(now()) @updatedAt @map("updated_at")
// relations
user LiteLLM_UserTable @relation(fields: [user_id], references: [user_id])
organization LiteLLM_OrganizationTable @relation("OrganizationToMembership", fields: [organization_id], references: [organization_id])
2024-10-09 17:48:18 +08:00
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
2024-10-09 17:48:18 +08:00
@@id([user_id, organization_id])
@@unique([user_id, organization_id])
}
model LiteLLM_InvitationLink {
// use this table to track invite links sent by admin for people to join the proxy
id String @id @default(uuid())
user_id String
is_accepted Boolean @default(false)
accepted_at DateTime? // when link is claimed (user successfully onboards via link)
expires_at DateTime // till when is link valid
created_at DateTime // when did admin create the link
created_by String // who created the link
updated_at DateTime // when was invite status updated
updated_by String // who updated the status (admin/user who accepted invite)
// Relations
liteLLM_user_table_user LiteLLM_UserTable @relation("UserId", fields: [user_id], references: [user_id])
liteLLM_user_table_created LiteLLM_UserTable @relation("CreatedBy", fields: [created_by], references: [user_id])
liteLLM_user_table_updated LiteLLM_UserTable @relation("UpdatedBy", fields: [updated_by], references: [user_id])
2024-06-06 07:19:38 +08:00
}
model LiteLLM_AuditLog {
2024-06-09 07:02:18 +08:00
id String @id @default(uuid())
updated_at DateTime @default(now())
changed_by String @default("") // user or system that performed the action
changed_by_api_key String @default("") // api key hash that performed the action
action String // create, update, delete
table_name String // on of LitellmTableNames.TEAM_TABLE_NAME, LitellmTableNames.USER_TABLE_NAME, LitellmTableNames.PROXY_MODEL_TABLE_NAME,
object_id String // id of the object being audited. This can be the key id, team id, user id, model id
before_value Json? // value of the row
2024-06-09 07:02:18 +08:00
updated_values Json? // value of the row after change
2024-07-09 01:16:58 +08:00
}
// Track daily user spend metrics per model and key
model LiteLLM_DailyUserSpend {
id String @id @default(uuid())
user_id String?
date String
api_key String
model String?
model_group String?
custom_llm_provider String?
mcp_namespaced_tool_name String?
endpoint String?
prompt_tokens BigInt @default(0)
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
failed_requests BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
@@index([date])
@@index([user_id, date])
@@index([api_key])
@@index([model])
@@index([mcp_namespaced_tool_name])
@@index([endpoint])
}
2025-03-28 12:10:26 +08:00
// Track daily organization spend metrics per model and key
model LiteLLM_DailyOrganizationSpend {
id String @id @default(uuid())
organization_id String?
date String
api_key String
model String?
model_group String?
custom_llm_provider String?
mcp_namespaced_tool_name String?
endpoint String?
prompt_tokens BigInt @default(0)
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
failed_requests BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([organization_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
@@index([date])
@@index([organization_id, date])
@@index([api_key])
@@index([model])
@@index([mcp_namespaced_tool_name])
@@index([endpoint])
}
// Track daily end user (customer) spend metrics per model and key
model LiteLLM_DailyEndUserSpend {
id String @id @default(uuid())
end_user_id String?
date String
api_key String
model String?
model_group String?
custom_llm_provider String?
mcp_namespaced_tool_name String?
endpoint String?
prompt_tokens BigInt @default(0)
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
failed_requests BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
@@index([date])
@@index([end_user_id, date])
@@index([api_key])
@@index([model])
@@index([mcp_namespaced_tool_name])
@@index([endpoint])
}
2025-12-11 03:50:52 +08:00
// Track daily agent spend metrics per model and key
model LiteLLM_DailyAgentSpend {
id String @id @default(uuid())
agent_id String?
date String
api_key String
model String?
model_group String?
custom_llm_provider String?
mcp_namespaced_tool_name String?
endpoint String?
2025-12-11 03:50:52 +08:00
prompt_tokens BigInt @default(0)
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
failed_requests BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([agent_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
2025-12-11 03:50:52 +08:00
@@index([date])
@@index([agent_id, date])
2025-12-11 03:50:52 +08:00
@@index([api_key])
@@index([model])
@@index([mcp_namespaced_tool_name])
@@index([endpoint])
2025-12-11 03:50:52 +08:00
}
// Track daily team spend metrics per model and key
model LiteLLM_DailyTeamSpend {
id String @id @default(uuid())
team_id String?
date String
api_key String
model String?
model_group String?
custom_llm_provider String?
mcp_namespaced_tool_name String?
endpoint String?
prompt_tokens BigInt @default(0)
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
failed_requests BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([team_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
@@index([date])
@@index([team_id, date])
@@index([api_key])
@@index([model])
@@index([mcp_namespaced_tool_name])
@@index([endpoint])
}
// Track daily team spend metrics per model and key
model LiteLLM_DailyTagSpend {
id String @id @default(uuid())
request_id String?
tag String?
date String
api_key String
model String?
model_group String?
custom_llm_provider String?
mcp_namespaced_tool_name String?
endpoint String?
prompt_tokens BigInt @default(0)
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
failed_requests BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([tag, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
@@index([date])
@@index([tag, date])
@@index([api_key])
@@index([model])
@@index([mcp_namespaced_tool_name])
@@index([endpoint])
}
2025-03-28 12:10:26 +08:00
// Track the status of cron jobs running. Only allow one pod to run the job at a time
2025-03-28 14:13:01 +08:00
model LiteLLM_CronJob {
cronjob_id String @id @default(cuid()) // Unique ID for the record
pod_id String // Unique identifier for the pod acting as the leader
2025-03-28 12:10:26 +08:00
status JobStatus @default(INACTIVE) // Status of the cron job (active or inactive)
last_updated DateTime @default(now()) // Timestamp for the last update of the cron job record
2025-03-28 12:10:26 +08:00
ttl DateTime // Time when the leader's lease expires
}
enum JobStatus {
ACTIVE
INACTIVE
}
model LiteLLM_ManagedFileTable {
id String @id @default(uuid())
unified_file_id String @unique // The base64 encoded unified file ID
file_object Json? // Stores the OpenAIFileObject
model_mappings Json
flat_model_file_ids String[] @default([]) // Flat list of model file id's - for faster querying of model id -> unified file id
2025-12-11 17:38:17 +08:00
storage_backend String? // Storage backend name (e.g., "azure_storage", "gcs", "default")
storage_url String? // The actual storage URL where the file is stored
created_at DateTime @default(now())
created_by String?
updated_at DateTime @updatedAt
updated_by String?
@@index([unified_file_id])
}
model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the
id String @id @default(uuid())
unified_object_id String @unique // The base64 encoded unified file ID
model_object_id String @unique // the id returned by the backend API provider
file_object Json // Stores the OpenAIFileObject
file_purpose String // either 'batch' or 'fine-tune'
status String? // check if batch cost has been tracked
batch_processed Boolean @default(false) // set to true by CheckBatchCost after cost is computed
created_at DateTime @default(now())
created_by String?
updated_at DateTime @updatedAt
updated_by String?
@@index([unified_object_id])
@@index([model_object_id])
}
2026-02-15 01:49:11 +08:00
model LiteLLM_ManagedVectorStoreTable {
id String @id @default(uuid())
unified_resource_id String @unique // The base64 encoded unified vector store ID
resource_object Json? // Stores the VectorStoreCreateResponse
model_mappings Json // Maps model_id -> provider_vector_store_id
flat_model_resource_ids String[] @default([]) // Flat list of provider vector store IDs for faster querying
storage_backend String? // Storage backend name (if applicable)
storage_url String? // Storage URL (if applicable)
created_at DateTime @default(now())
created_by String?
updated_at DateTime @updatedAt
updated_by String?
@@index([unified_resource_id])
}
model LiteLLM_ManagedVectorStoresTable {
vector_store_id String @id
custom_llm_provider String
vector_store_name String?
vector_store_description String?
vector_store_metadata Json?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
litellm_credential_name String?
litellm_params Json?
team_id String?
user_id String?
@@index([team_id])
@@index([user_id])
}
// Guardrails table for storing guardrail configurations
model LiteLLM_GuardrailsTable {
guardrail_id String @id @default(uuid())
guardrail_name String @unique
litellm_params Json
guardrail_info Json?
2026-02-15 01:49:11 +08:00
team_id String?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
feat(guardrails): team-based guardrail registration and approval workflow (#22459) * feat(guardrails): team-based guardrail registration and approval workflow Add team-based guardrail submission system where teams can register Generic Guardrail API guardrails for admin review. Includes: - POST /guardrails/register endpoint for team-scoped submissions - Admin review endpoints (list/get/approve/reject submissions) - Team Guardrails tab in the UI dashboard - extra_headers support for forwarding client headers to guardrail APIs - Prisma schema migration for status, submitted_at, reviewed_at fields - Documentation for team-based guardrails and static/dynamic headers Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix(guardrails): address review feedback - SSRF, silent failure, redundant query - Validate api_base URL scheme (http/https only) and hostname in register_guardrail to prevent SSRF via team submissions - Return warning field in approve response when in-memory initialization fails so admins know the guardrail won't work until next sync cycle - Eliminate redundant DB query in list_guardrail_submissions by fetching all team guardrails once and deriving both filtered list and summary counts from the single result set Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix(guardrails): add pending_review status guard to reject endpoint Prevent rejecting already-active or already-rejected guardrails, which would create a DB/memory inconsistency (active in memory but rejected in DB). Now mirrors the approve endpoint's status check. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-03 14:06:49 +08:00
// Submission lifecycle. Possible values: pending_review (team-registered, awaiting approval), active (approved), rejected
status String @default("active")
submitted_at DateTime?
reviewed_at DateTime?
// submitted_by_user_id and submitted_by_email live in guardrail_info JSON
@@index([status])
}
// Daily guardrail metrics for usage dashboard (one row per guardrail per day)
model LiteLLM_DailyGuardrailMetrics {
guardrail_id String // logical id; may not FK if guardrail from config
date String // YYYY-MM-DD
requests_evaluated BigInt @default(0)
passed_count BigInt @default(0)
blocked_count BigInt @default(0)
flagged_count BigInt @default(0)
avg_score Float?
avg_latency_ms Float?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@id([guardrail_id, date])
@@index([date])
@@index([guardrail_id])
}
// Daily policy metrics for usage dashboard (one row per policy per day)
model LiteLLM_DailyPolicyMetrics {
policy_id String
date String // YYYY-MM-DD
requests_evaluated BigInt @default(0)
passed_count BigInt @default(0)
blocked_count BigInt @default(0)
flagged_count BigInt @default(0)
avg_score Float?
avg_latency_ms Float?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@id([policy_id, date])
@@index([date])
@@index([policy_id])
}
// Index for fast "last N logs for guardrail/policy" from SpendLogs
model LiteLLM_SpendLogGuardrailIndex {
request_id String
guardrail_id String
policy_id String? // set when run as part of a policy pipeline
start_time DateTime
@@id([request_id, guardrail_id])
@@index([guardrail_id, start_time])
@@index([policy_id, start_time])
}
[Feat] Add Tool Policies for AI Gateway (#22732) * fix: fix ui render * fix: fix minor bugs * refactor: use prisma functions instead of raw sql (safer) * fix(add-new-tiles-to-tool-policies): allow developer to see what's available * feat: ensure tool allowlist runs correctly for tool names + mcp's * refactor: more ui improvements * feat: working key tool blocking * feat(tools): show tool logs * refactor: backend code improvements * refactor: improve log viewer for tools * fix: address PR review feedback for tool access control - Add missing blocked_tools column to root schema.prisma (schema drift) - Invalidate ToolPolicyRegistry after policy mutations so changes take effect immediately - Remove dead code: unused get_effective_policies, get_tool_policies_cached, and helpers Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix: race condition in permission resolution and remove duplicate allowlist check - Use atomic update_many with object_permission_id=None to prevent concurrent requests from creating orphaned permission rows and losing tool blocks - Remove duplicate allowed_tools enforcement from guardrail (already enforced in auth layer via check_tools_allowlist) - Move inline uuid import to module level Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * update to account for userAgent * UI - Add ToolDetails * input/output policy * LiteLLM_PolicyAttachmentTable * LiteLLM_PolicyAttachmentTable * fix: add _enqueue_tool_registry_upsert * fix: tool mgmt endpoints * tool mgmt endpoints * Update tests/test_litellm/proxy/db/test_tool_registry_writer.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * Update tests/test_litellm/proxy/db/test_tool_registry_writer.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * Update tests/test_litellm/proxy/db/test_tool_registry_writer.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * fix: sync root schema.prisma and fix test_tool_registry_writer for input/output policy - Migrate root schema.prisma LiteLLM_ToolTable from call_policy to input_policy/output_policy, add missing user_agent and last_used_at columns (now consistent with litellm/proxy/schema.prisma and litellm-proxy-extras) - Fix SpendLogToolIndex comment across all three schema files - Fix all call_policy references in test_tool_registry_writer.py: swapped update_tool_policy arguments, wrong get_tools_by_names return type assertions, _mock_tool_row setting call_policy instead of input_policy Addresses Greptile review feedback on PR #22732. Made-with: Cursor --------- Co-authored-by: Krrish Dholakia <krrishdholakia@gmail.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
2026-03-04 12:22:20 +08:00
// Index for fast "last N logs for tool" from SpendLogs see how a tool is called in production
model LiteLLM_SpendLogToolIndex {
request_id String
tool_name String // matches LiteLLM_ToolTable.tool_name; join for input_policy/output_policy etc.
start_time DateTime
@@id([request_id, tool_name])
@@index([tool_name, start_time])
}
Prompt Management - add prompts on UI (#13240) * fix(create_key_button.tsx): add prompts on UI * feat(key_management_endpoints.py): support adding prompt to key via `/key/update` * fix(key_info_view.tsx): show existing prompts on key in key_info_view.tsx * fix(key_edit_view.tsx): UX - disable premium feature for non-premium users prevent accidental clicking * fix(create_key_button.tsx): disable premium features behind flag, prevent errors * feat(prompts.tsx): add new ui component to view created prompts enables viewing prompts created on config * feat(prompt_info.tsx): add component for viewing the prompt information * feat(prompt_endpoints.py): support converting dotprompt to json structure + accept json structure in promptmanager allows prompt manager to work with api endpoints * test(test_prompt_manager.py): add unit tests for json data input * feat(dotprompt/__init__.py): add prompt data to dotpromptmanager * fix(prompt_endpoints.py): working crud endpoints for prompt management * feat(prompts/): support `prompt_file` for dotprompt allows to precisely point to the prompt file a prompt should use * feat(proxy/utils.py): resolve prompt id correctly resolves user sent prompt id with internal prompt id * feat(schema.prisma): initial pr with db schema for prompt management table allows post endpoints to work with backend * feat(prompt_endpoints.py): use db in patch_prompt endpoint * feat(prompt_endpoints.py): use db for update_prompt endpoint * feat(prompt_endpoints.py): use db on prompt delete endpoint * build(schema.prisma): add prompt tale to schema.prisma in litellm-proxy-extras * build(migration.sql): add new sql migration file * fix(init_prompts.py): fix init * feat(prompt_info_view.tsx): show the raw prompt template on ui allows developer to know the prompt template they'll be calling * feat(add_prompt_form.tsx): working ui add prompt flow allows user to add prompts to litellm via ui * build(ui/): styling fixes * build(ui/): prompts.tsx styling improvements * fix(add_prompt_form.tsx): styling improvements * build(prompts.tsx): styling improvements * build(ui/): styling improvements * build(ui/): fix ui error * fix: fix ruff check * docs: document new api params * test: update tests
2025-08-03 13:33:37 +08:00
// Prompt table for storing prompt configurations
model LiteLLM_PromptTable {
id String @id @default(uuid())
prompt_id String
version Int @default(1)
Litellm ishaan april1 try2 (#25110) * Litellm ishaan april1 (#25103) * fix(proxy): enforce upperbound key params on key/update and add custom_key_update hook The /key/update endpoint did not enforce upperbound_key_generate_params, allowing users to bypass configured limits (tpm_limit, rpm_limit, max_budget, duration, budget_duration) by updating an existing key instead of generating a new one. Extract the upperbound enforcement logic from _common_key_generation_helper() into a standalone _enforce_upperbound_key_params() function and call it from both the generate and update paths. For updates, None values are skipped (not filled with defaults) since they mean "don't change this field". Also adds a custom_key_update config option and user_custom_key_update global, mirroring the existing custom_key_generate pattern, so custom key validation logic can fire during key updates as well. * fix(proxy): invoke custom_key_update hook in bulk update path The user_custom_key_update hook was only called in update_key_fn (single key update) but not in _process_single_key_update (bulk update path), allowing custom validation to be bypassed via the /key/update/bulk endpoint. Mirror the hook invocation in both paths. * fix(proxy): pass UpdateKeyRequest to hook in bulk path, not BulkUpdateKeyRequestItem Move the custom_key_update hook invocation to after UpdateKeyRequest is constructed so the hook receives the same type in both single and bulk update paths. Previously the bulk path passed BulkUpdateKeyRequestItem (5 fields only), which would cause AttributeError for hooks accessing fields like tpm_limit or models. * fix(bedrock): promote cache usage to message_delta for Claude Code (#24850) Ensure Bedrock/Anthropic-compatible streaming exposes cache usage where Claude Code reads it by promoting message_stop usage onto message_delta and preserving usage fields in fake-streamed message_delta events. Made-with: Cursor * fix(search): Support self-hosted Firecrawl response format in search transform (#24866) The `transform_search_response` method only handled Firecrawl Cloud (v2) response format where `data` is a dict with `web`/`news` keys. Self-hosted Firecrawl (v1) returns `data` as a flat list of result objects, causing an `AttributeError: 'list' object has no attribute 'get'`. Detect the response format by checking if `data` is a list (self-hosted) or dict (cloud) and handle both cases. Cloud format: {"data": {"web": [...], "news": [...]}} Self-hosted: {"success": true, "data": [{"url": "...", "title": "...", ...}]} Co-authored-by: Synergy <synergyoclaw@gmail.com> * feat: add environment and user tracking to prompt management (#24855) * feat: add environment and user tracking to prompt management - Add environment (development/staging/production) and created_by columns to LiteLLM_PromptTable - Update unique constraint to [prompt_id, version, environment] - All CRUD endpoints support environment filtering and user tracking - Redesigned prompt detail page with environment tabs and version history - UI: environment filter on list page, environment selector in editor - 8 new tests for environment and user tracking Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: Black formatting and add environments to PromptInfoResponse TypeScript type Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: address Greptile review findings - P1: delete_prompt scopes in-memory cleanup to environment when provided - P2: dotprompt_content parsed directly regardless of environment flag - P2: use distinct for environments query - P2: fix double-fetch on initial mount in prompt_info.tsx - fix: remove unsupported select kwarg from find_many Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: address remaining Greptile review comments - Remove unused useCallback import (index.tsx) - Remove unused ENV_COLORS variable (prompt_info.tsx) - P1: in-memory fallback in get_prompt_versions now respects environment filter - P1: reset selectedEnv when promptId changes to avoid stale state - Cyclic imports are pre-existing pattern, not introduced by this PR Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: scope patch_prompt to environment using primary key - Add environment query param to patch_prompt endpoint - Look up target row by composite key (prompt_id + version + environment) - Update by primary key (id) to target exactly one row - Fixes Greptile finding: patch with multiple environments no longer ambiguous Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: use actual start_time for failed request spend logs (#24906) async_post_call_failure_hook set both start_time and end_time to datetime.now(), making all failed requests show duration=0. Use the actual start_time from litellm_logging_obj instead, so spend logs reflect the real request duration on timeout and other failures. Fixes #24888 * feat(bedrock): add nova canvas image edit support (#24869) * feat(bedrock): add nova canvas image edit support * fix(bedrock): support PathLike inputs for nova image edit * chore: sync schema.prisma copies from root * fix(mypy): correct type-ignore code for delta_usage arg-type * fix(mypy): cast status_code to str, suppress intentional str yield * fix(lint): extract _create_content_block_chunks to fix PLR0915 * fix(lint): extract helpers to fix PLR0915 in prompt endpoints --------- Co-authored-by: michelligabriele <gabriele.michelli@icloud.com> Co-authored-by: Sameer Kankute <sameer@berri.ai> Co-authored-by: redhelix <amin.lalji@gmail.com> Co-authored-by: Synergy <synergyoclaw@gmail.com> Co-authored-by: Talha Anwar <37379131+talhaanwarch@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: madhu19991 <madhu@thunkai.com> Co-authored-by: Srikanth @adobe <devarakondasrikanth@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(test): update model armor streaming test to handle string or int error code --------- Co-authored-by: michelligabriele <gabriele.michelli@icloud.com> Co-authored-by: Sameer Kankute <sameer@berri.ai> Co-authored-by: redhelix <amin.lalji@gmail.com> Co-authored-by: Synergy <synergyoclaw@gmail.com> Co-authored-by: Talha Anwar <37379131+talhaanwarch@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: madhu19991 <madhu@thunkai.com> Co-authored-by: Srikanth @adobe <devarakondasrikanth@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
2026-04-04 05:57:44 +08:00
environment String @default("development")
created_by String?
Prompt Management - add prompts on UI (#13240) * fix(create_key_button.tsx): add prompts on UI * feat(key_management_endpoints.py): support adding prompt to key via `/key/update` * fix(key_info_view.tsx): show existing prompts on key in key_info_view.tsx * fix(key_edit_view.tsx): UX - disable premium feature for non-premium users prevent accidental clicking * fix(create_key_button.tsx): disable premium features behind flag, prevent errors * feat(prompts.tsx): add new ui component to view created prompts enables viewing prompts created on config * feat(prompt_info.tsx): add component for viewing the prompt information * feat(prompt_endpoints.py): support converting dotprompt to json structure + accept json structure in promptmanager allows prompt manager to work with api endpoints * test(test_prompt_manager.py): add unit tests for json data input * feat(dotprompt/__init__.py): add prompt data to dotpromptmanager * fix(prompt_endpoints.py): working crud endpoints for prompt management * feat(prompts/): support `prompt_file` for dotprompt allows to precisely point to the prompt file a prompt should use * feat(proxy/utils.py): resolve prompt id correctly resolves user sent prompt id with internal prompt id * feat(schema.prisma): initial pr with db schema for prompt management table allows post endpoints to work with backend * feat(prompt_endpoints.py): use db in patch_prompt endpoint * feat(prompt_endpoints.py): use db for update_prompt endpoint * feat(prompt_endpoints.py): use db on prompt delete endpoint * build(schema.prisma): add prompt tale to schema.prisma in litellm-proxy-extras * build(migration.sql): add new sql migration file * fix(init_prompts.py): fix init * feat(prompt_info_view.tsx): show the raw prompt template on ui allows developer to know the prompt template they'll be calling * feat(add_prompt_form.tsx): working ui add prompt flow allows user to add prompts to litellm via ui * build(ui/): styling fixes * build(ui/): prompts.tsx styling improvements * fix(add_prompt_form.tsx): styling improvements * build(prompts.tsx): styling improvements * build(ui/): styling improvements * build(ui/): fix ui error * fix: fix ruff check * docs: document new api params * test: update tests
2025-08-03 13:33:37 +08:00
litellm_params Json
prompt_info Json?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
Litellm ishaan april1 try2 (#25110) * Litellm ishaan april1 (#25103) * fix(proxy): enforce upperbound key params on key/update and add custom_key_update hook The /key/update endpoint did not enforce upperbound_key_generate_params, allowing users to bypass configured limits (tpm_limit, rpm_limit, max_budget, duration, budget_duration) by updating an existing key instead of generating a new one. Extract the upperbound enforcement logic from _common_key_generation_helper() into a standalone _enforce_upperbound_key_params() function and call it from both the generate and update paths. For updates, None values are skipped (not filled with defaults) since they mean "don't change this field". Also adds a custom_key_update config option and user_custom_key_update global, mirroring the existing custom_key_generate pattern, so custom key validation logic can fire during key updates as well. * fix(proxy): invoke custom_key_update hook in bulk update path The user_custom_key_update hook was only called in update_key_fn (single key update) but not in _process_single_key_update (bulk update path), allowing custom validation to be bypassed via the /key/update/bulk endpoint. Mirror the hook invocation in both paths. * fix(proxy): pass UpdateKeyRequest to hook in bulk path, not BulkUpdateKeyRequestItem Move the custom_key_update hook invocation to after UpdateKeyRequest is constructed so the hook receives the same type in both single and bulk update paths. Previously the bulk path passed BulkUpdateKeyRequestItem (5 fields only), which would cause AttributeError for hooks accessing fields like tpm_limit or models. * fix(bedrock): promote cache usage to message_delta for Claude Code (#24850) Ensure Bedrock/Anthropic-compatible streaming exposes cache usage where Claude Code reads it by promoting message_stop usage onto message_delta and preserving usage fields in fake-streamed message_delta events. Made-with: Cursor * fix(search): Support self-hosted Firecrawl response format in search transform (#24866) The `transform_search_response` method only handled Firecrawl Cloud (v2) response format where `data` is a dict with `web`/`news` keys. Self-hosted Firecrawl (v1) returns `data` as a flat list of result objects, causing an `AttributeError: 'list' object has no attribute 'get'`. Detect the response format by checking if `data` is a list (self-hosted) or dict (cloud) and handle both cases. Cloud format: {"data": {"web": [...], "news": [...]}} Self-hosted: {"success": true, "data": [{"url": "...", "title": "...", ...}]} Co-authored-by: Synergy <synergyoclaw@gmail.com> * feat: add environment and user tracking to prompt management (#24855) * feat: add environment and user tracking to prompt management - Add environment (development/staging/production) and created_by columns to LiteLLM_PromptTable - Update unique constraint to [prompt_id, version, environment] - All CRUD endpoints support environment filtering and user tracking - Redesigned prompt detail page with environment tabs and version history - UI: environment filter on list page, environment selector in editor - 8 new tests for environment and user tracking Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: Black formatting and add environments to PromptInfoResponse TypeScript type Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: address Greptile review findings - P1: delete_prompt scopes in-memory cleanup to environment when provided - P2: dotprompt_content parsed directly regardless of environment flag - P2: use distinct for environments query - P2: fix double-fetch on initial mount in prompt_info.tsx - fix: remove unsupported select kwarg from find_many Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: address remaining Greptile review comments - Remove unused useCallback import (index.tsx) - Remove unused ENV_COLORS variable (prompt_info.tsx) - P1: in-memory fallback in get_prompt_versions now respects environment filter - P1: reset selectedEnv when promptId changes to avoid stale state - Cyclic imports are pre-existing pattern, not introduced by this PR Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: scope patch_prompt to environment using primary key - Add environment query param to patch_prompt endpoint - Look up target row by composite key (prompt_id + version + environment) - Update by primary key (id) to target exactly one row - Fixes Greptile finding: patch with multiple environments no longer ambiguous Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: use actual start_time for failed request spend logs (#24906) async_post_call_failure_hook set both start_time and end_time to datetime.now(), making all failed requests show duration=0. Use the actual start_time from litellm_logging_obj instead, so spend logs reflect the real request duration on timeout and other failures. Fixes #24888 * feat(bedrock): add nova canvas image edit support (#24869) * feat(bedrock): add nova canvas image edit support * fix(bedrock): support PathLike inputs for nova image edit * chore: sync schema.prisma copies from root * fix(mypy): correct type-ignore code for delta_usage arg-type * fix(mypy): cast status_code to str, suppress intentional str yield * fix(lint): extract _create_content_block_chunks to fix PLR0915 * fix(lint): extract helpers to fix PLR0915 in prompt endpoints --------- Co-authored-by: michelligabriele <gabriele.michelli@icloud.com> Co-authored-by: Sameer Kankute <sameer@berri.ai> Co-authored-by: redhelix <amin.lalji@gmail.com> Co-authored-by: Synergy <synergyoclaw@gmail.com> Co-authored-by: Talha Anwar <37379131+talhaanwarch@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: madhu19991 <madhu@thunkai.com> Co-authored-by: Srikanth @adobe <devarakondasrikanth@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(test): update model armor streaming test to handle string or int error code --------- Co-authored-by: michelligabriele <gabriele.michelli@icloud.com> Co-authored-by: Sameer Kankute <sameer@berri.ai> Co-authored-by: redhelix <amin.lalji@gmail.com> Co-authored-by: Synergy <synergyoclaw@gmail.com> Co-authored-by: Talha Anwar <37379131+talhaanwarch@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: madhu19991 <madhu@thunkai.com> Co-authored-by: Srikanth @adobe <devarakondasrikanth@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
2026-04-04 05:57:44 +08:00
@@unique([prompt_id, version, environment])
@@index([prompt_id, environment])
@@index([prompt_id])
Prompt Management - add prompts on UI (#13240) * fix(create_key_button.tsx): add prompts on UI * feat(key_management_endpoints.py): support adding prompt to key via `/key/update` * fix(key_info_view.tsx): show existing prompts on key in key_info_view.tsx * fix(key_edit_view.tsx): UX - disable premium feature for non-premium users prevent accidental clicking * fix(create_key_button.tsx): disable premium features behind flag, prevent errors * feat(prompts.tsx): add new ui component to view created prompts enables viewing prompts created on config * feat(prompt_info.tsx): add component for viewing the prompt information * feat(prompt_endpoints.py): support converting dotprompt to json structure + accept json structure in promptmanager allows prompt manager to work with api endpoints * test(test_prompt_manager.py): add unit tests for json data input * feat(dotprompt/__init__.py): add prompt data to dotpromptmanager * fix(prompt_endpoints.py): working crud endpoints for prompt management * feat(prompts/): support `prompt_file` for dotprompt allows to precisely point to the prompt file a prompt should use * feat(proxy/utils.py): resolve prompt id correctly resolves user sent prompt id with internal prompt id * feat(schema.prisma): initial pr with db schema for prompt management table allows post endpoints to work with backend * feat(prompt_endpoints.py): use db in patch_prompt endpoint * feat(prompt_endpoints.py): use db for update_prompt endpoint * feat(prompt_endpoints.py): use db on prompt delete endpoint * build(schema.prisma): add prompt tale to schema.prisma in litellm-proxy-extras * build(migration.sql): add new sql migration file * fix(init_prompts.py): fix init * feat(prompt_info_view.tsx): show the raw prompt template on ui allows developer to know the prompt template they'll be calling * feat(add_prompt_form.tsx): working ui add prompt flow allows user to add prompts to litellm via ui * build(ui/): styling fixes * build(ui/): prompts.tsx styling improvements * fix(add_prompt_form.tsx): styling improvements * build(prompts.tsx): styling improvements * build(ui/): styling improvements * build(ui/): fix ui error * fix: fix ruff check * docs: document new api params * test: update tests
2025-08-03 13:33:37 +08:00
}
model LiteLLM_HealthCheckTable {
health_check_id String @id @default(uuid())
model_name String
model_id String?
status String
healthy_count Int @default(0)
unhealthy_count Int @default(0)
error_message String?
response_time_ms Float?
details Json?
checked_by String?
checked_at DateTime @default(now())
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@index([model_name])
@@index([checked_at])
@@index([status])
@@index([model_id, model_name, checked_at(sort: Desc)], map: "LiteLLM_HealthCheckTable_model_id_model_name_checked_at_idx")
}
// Search Tools table for storing search tool configurations
model LiteLLM_SearchToolsTable {
search_tool_id String @id @default(uuid())
search_tool_name String @unique
litellm_params Json
search_tool_info Json?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
}
// SSO configuration table
model LiteLLM_SSOConfig {
id String @id @default("sso_config")
sso_settings Json
created_at DateTime @default(now())
updated_at DateTime @updatedAt
}
2025-11-02 04:45:32 +08:00
model LiteLLM_ManagedVectorStoreIndexTable {
id String @id @default(uuid())
index_name String @unique
litellm_params Json
index_info Json?
created_at DateTime @default(now())
created_by String?
updated_at DateTime @updatedAt
updated_by String?
}
// Cache configuration table
model LiteLLM_CacheConfig {
id String @id @default("cache_config")
cache_settings Json
created_at DateTime @default(now())
updated_at DateTime @updatedAt
}
// UI Settings configuration table
model LiteLLM_UISettings {
id String @id @default("ui_settings")
ui_settings Json
created_at DateTime @default(now())
updated_at DateTime @updatedAt
}
2026-03-07 15:39:08 +08:00
// Generic config overrides table - one row per config_type
model LiteLLM_ConfigOverrides {
config_type String @id
config_value Json
created_at DateTime @default(now())
updated_at DateTime @updatedAt
}
// Skills table for storing LiteLLM-managed skills
model LiteLLM_SkillsTable {
skill_id String @id @default(uuid())
display_title String?
description String?
instructions String? // The skill instructions/prompt (from SKILL.md)
source String @default("custom") // "custom" or "anthropic"
latest_version String?
file_content Bytes? // Binary content of the skill files (zip)
file_name String? // Original filename
file_type String? // MIME type (e.g., "application/zip")
metadata Json? @default("{}")
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
2026-01-05 14:49:09 +08:00
}
2026-01-24 05:16:58 +08:00
// Policy table for storing guardrail policies (versioned)
2026-01-24 05:16:58 +08:00
model LiteLLM_PolicyTable {
policy_id String @id @default(uuid())
policy_name String // No longer @unique; use @@unique([policy_name, version_number])
version_number Int @default(1)
version_status String @default("production") // "draft" | "published" | "production"
parent_version_id String?
is_latest Boolean @default(true)
published_at DateTime?
production_at DateTime?
inherit String? // Name of parent policy to inherit from
description String?
guardrails_add String[] @default([])
guardrails_remove String[] @default([])
condition Json? @default("{}") // Policy conditions (e.g., model matching)
pipeline Json? // Optional guardrail pipeline (mode + steps[])
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
@@unique([policy_name, version_number])
@@index([policy_name, version_status])
2026-01-24 05:16:58 +08:00
}
// Policy attachment table for defining where policies apply
model LiteLLM_PolicyAttachmentTable {
attachment_id String @id @default(uuid())
policy_name String // Name of the policy to attach
scope String? // Use '*' for global scope
teams String[] @default([]) // Team aliases or patterns
keys String[] @default([]) // Key aliases or patterns
models String[] @default([]) // Model names or patterns
[Feat] Policies - Allow connecting Policies to Tags, Simulating Policies, Viewing how many keys, teams it applies on (#20904) * init schema with TAGS * ui: add policy test * resolvePoliciesCall * add_policy_sources_to_metadata + headers * types Policy * preview Impact * def _describe_match_reason( * match based on TAGs * TestTagBasedAttachments * test fixes * add policy_resolve_router * add_guardrails_from_policy_engine * TestMatchAttribution * refactor * fix * fix: address Greptile review feedback on policy resolve endpoints - Track unnamed keys/teams as separate counts instead of inflating affected_keys_count with duplicate "(unnamed key)" placeholders. Added unnamed_keys_count and unnamed_teams_count to response. - Push alias pattern matching to DB via _build_alias_where() which converts exact patterns to Prisma "in" and suffix wildcards to "startsWith" filters. - Gate sync_policies_from_db/sync_attachments_from_db behind force_sync query param (default false) to avoid 2 DB round-trips on every /policies/resolve request. - Remove worktree-only conftest.py that cleared sys.modules at import time — no longer needed since code moved to main repo. - Rename MAX_ESTIMATE_IMPACT_ROWS → MAX_POLICY_ESTIMATE_IMPACT_ROWS. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix: eliminate duplicate DB queries and fix header delimiter ambiguity - Fetch teams table once in estimate_attachment_impact and reuse for both tag-based and alias-based lookups (was querying teams twice when both tag_patterns and team_patterns were provided). - Convert tag/team filter functions from async DB queries to sync filters that operate on pre-fetched data (_filter_keys_by_tags, _filter_teams_by_tags). - Fix comma ambiguity in x-litellm-policy-sources header: use '; ' as entry delimiter since matched_via values can contain commas. - Use '+' as the within-value separator in matched_via reason strings (e.g. "tag:healthcare+team:health-team") to avoid conflict with header delimiters. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * Update litellm/proxy/policy_engine/policy_resolve_endpoints.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
2026-02-11 09:50:37 +08:00
tags String[] @default([]) // Tag patterns (e.g., ["healthcare", "prod-*"])
2026-01-24 05:16:58 +08:00
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
}
2026-02-12 12:44:30 +08:00
[Feat] Add Tool Policies for AI Gateway (#22732) * fix: fix ui render * fix: fix minor bugs * refactor: use prisma functions instead of raw sql (safer) * fix(add-new-tiles-to-tool-policies): allow developer to see what's available * feat: ensure tool allowlist runs correctly for tool names + mcp's * refactor: more ui improvements * feat: working key tool blocking * feat(tools): show tool logs * refactor: backend code improvements * refactor: improve log viewer for tools * fix: address PR review feedback for tool access control - Add missing blocked_tools column to root schema.prisma (schema drift) - Invalidate ToolPolicyRegistry after policy mutations so changes take effect immediately - Remove dead code: unused get_effective_policies, get_tool_policies_cached, and helpers Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix: race condition in permission resolution and remove duplicate allowlist check - Use atomic update_many with object_permission_id=None to prevent concurrent requests from creating orphaned permission rows and losing tool blocks - Remove duplicate allowed_tools enforcement from guardrail (already enforced in auth layer via check_tools_allowlist) - Move inline uuid import to module level Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * update to account for userAgent * UI - Add ToolDetails * input/output policy * LiteLLM_PolicyAttachmentTable * LiteLLM_PolicyAttachmentTable * fix: add _enqueue_tool_registry_upsert * fix: tool mgmt endpoints * tool mgmt endpoints * Update tests/test_litellm/proxy/db/test_tool_registry_writer.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * Update tests/test_litellm/proxy/db/test_tool_registry_writer.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * Update tests/test_litellm/proxy/db/test_tool_registry_writer.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * fix: sync root schema.prisma and fix test_tool_registry_writer for input/output policy - Migrate root schema.prisma LiteLLM_ToolTable from call_policy to input_policy/output_policy, add missing user_agent and last_used_at columns (now consistent with litellm/proxy/schema.prisma and litellm-proxy-extras) - Fix SpendLogToolIndex comment across all three schema files - Fix all call_policy references in test_tool_registry_writer.py: swapped update_tool_policy arguments, wrong get_tools_by_names return type assertions, _mock_tool_row setting call_policy instead of input_policy Addresses Greptile review feedback on PR #22732. Made-with: Cursor --------- Co-authored-by: Krrish Dholakia <krrishdholakia@gmail.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
2026-03-04 12:22:20 +08:00
// Global tool registry - auto-discovered from LLM responses; admins set input/output policies here
Agents - assign tools (#22064) * feat(proxy): add max_iterations limiter for agent session loops (#22058) Adds a new proxy hook that enforces a per-session cap on the number of LLM calls an agentic loop can make. Callers send a session_id with each request, and the hook counts calls per session, returning 429 when the configured max_iterations limit is exceeded. - Uses Redis Lua script for atomic increment (multi-instance safe) - Falls back to in-memory cache when Redis unavailable - Follows parallel_request_limiter_v3 pattern - Configurable via key metadata: {"max_iterations": 25} - Session counters auto-expire via TTL (default 1hr) Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> * feat: add new code execution dataset * feat(agent_endpoints/): allow giving agents keys * fix: ui fixes * feat: allow assigning mcp servers to agents * fix: eliminate duplicate DB queries in MCP agent auth and N+1 in agent listing (#22110) - Extract _get_agent_object_permission helper so _get_allowed_mcp_servers_for_agent and _get_agent_tool_permissions_for_server share a single DB fetch instead of each independently querying the same agent row (was 1+N queries per MCP request) - Use include={"object_permission": True} on find_many in get_all_agents_from_db to eagerly load permissions in one query instead of N+1 - Use include={"object_permission": True} on create/update/find_unique in all agent CRUD operations, removing attach_object_permission_to_dict follow-up calls Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-26 03:44:30 +08:00
model LiteLLM_ToolTable {
[Feat] Add Tool Policies for AI Gateway (#22732) * fix: fix ui render * fix: fix minor bugs * refactor: use prisma functions instead of raw sql (safer) * fix(add-new-tiles-to-tool-policies): allow developer to see what's available * feat: ensure tool allowlist runs correctly for tool names + mcp's * refactor: more ui improvements * feat: working key tool blocking * feat(tools): show tool logs * refactor: backend code improvements * refactor: improve log viewer for tools * fix: address PR review feedback for tool access control - Add missing blocked_tools column to root schema.prisma (schema drift) - Invalidate ToolPolicyRegistry after policy mutations so changes take effect immediately - Remove dead code: unused get_effective_policies, get_tool_policies_cached, and helpers Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix: race condition in permission resolution and remove duplicate allowlist check - Use atomic update_many with object_permission_id=None to prevent concurrent requests from creating orphaned permission rows and losing tool blocks - Remove duplicate allowed_tools enforcement from guardrail (already enforced in auth layer via check_tools_allowlist) - Move inline uuid import to module level Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * update to account for userAgent * UI - Add ToolDetails * input/output policy * LiteLLM_PolicyAttachmentTable * LiteLLM_PolicyAttachmentTable * fix: add _enqueue_tool_registry_upsert * fix: tool mgmt endpoints * tool mgmt endpoints * Update tests/test_litellm/proxy/db/test_tool_registry_writer.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * Update tests/test_litellm/proxy/db/test_tool_registry_writer.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * Update tests/test_litellm/proxy/db/test_tool_registry_writer.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * fix: sync root schema.prisma and fix test_tool_registry_writer for input/output policy - Migrate root schema.prisma LiteLLM_ToolTable from call_policy to input_policy/output_policy, add missing user_agent and last_used_at columns (now consistent with litellm/proxy/schema.prisma and litellm-proxy-extras) - Fix SpendLogToolIndex comment across all three schema files - Fix all call_policy references in test_tool_registry_writer.py: swapped update_tool_policy arguments, wrong get_tools_by_names return type assertions, _mock_tool_row setting call_policy instead of input_policy Addresses Greptile review feedback on PR #22732. Made-with: Cursor --------- Co-authored-by: Krrish Dholakia <krrishdholakia@gmail.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
2026-03-04 12:22:20 +08:00
tool_id String @id @default(uuid())
tool_name String @unique // e.g. "huggingface_remote-mcp__dynamic_space"
origin String? // MCP server name or "user_defined"
input_policy String @default("untrusted") // "trusted" | "untrusted" | "blocked"
output_policy String @default("untrusted") // "trusted" | "untrusted"
call_count Int @default(0) // cumulative number of times this tool was seen
assignments Json? @default("{}")
key_hash String? // hash of the virtual key that first called this tool
team_id String? // team that first called this tool
key_alias String? // human-readable alias of the virtual key
user_agent String? // user-agent of the first request that discovered this tool
last_used_at DateTime? // timestamp of the most recent call
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
@@index([input_policy])
@@index([output_policy])
Agents - assign tools (#22064) * feat(proxy): add max_iterations limiter for agent session loops (#22058) Adds a new proxy hook that enforces a per-session cap on the number of LLM calls an agentic loop can make. Callers send a session_id with each request, and the hook counts calls per session, returning 429 when the configured max_iterations limit is exceeded. - Uses Redis Lua script for atomic increment (multi-instance safe) - Falls back to in-memory cache when Redis unavailable - Follows parallel_request_limiter_v3 pattern - Configurable via key metadata: {"max_iterations": 25} - Session counters auto-expire via TTL (default 1hr) Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> * feat: add new code execution dataset * feat(agent_endpoints/): allow giving agents keys * fix: ui fixes * feat: allow assigning mcp servers to agents * fix: eliminate duplicate DB queries in MCP agent auth and N+1 in agent listing (#22110) - Extract _get_agent_object_permission helper so _get_allowed_mcp_servers_for_agent and _get_agent_tool_permissions_for_server share a single DB fetch instead of each independently querying the same agent row (was 1+N queries per MCP request) - Use include={"object_permission": True} on find_many in get_all_agents_from_db to eagerly load permissions in one query instead of N+1 - Use include={"object_permission": True} on create/update/find_unique in all agent CRUD operations, removing attach_object_permission_to_dict follow-up calls Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-26 03:44:30 +08:00
@@index([team_id])
}
2026-03-07 15:39:08 +08:00
// Per-(tool, team/key) policy overrides. When present, override replaces global tool policy for that scope.
2026-02-12 12:44:30 +08:00
//Unified Access Groups table for storing unified access groups
model LiteLLM_AccessGroupTable {
2026-02-12 12:44:30 +08:00
access_group_id String @id @default(uuid())
access_group_name String @unique
description String?
// Resource memberships - explicit arrays per type
access_model_names String[] @default([])
2026-02-12 12:44:30 +08:00
access_mcp_server_ids String[] @default([])
access_agent_ids String[] @default([])
assigned_team_ids String[] @default([])
assigned_key_ids String[] @default([])
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
}
// Claude Code Plugin Marketplace table
model LiteLLM_ClaudeCodePluginTable {
id String @id @default(uuid())
name String @unique
version String?
description String?
manifest_json String?
files_json String? @default("{}")
enabled Boolean @default(true)
created_at DateTime? @default(now())
updated_at DateTime? @default(now()) @updatedAt
created_by String?
@@map("LiteLLM_ClaudeCodePluginTable")
}
feat(proxy): add /v1/memory CRUD endpoints (#26218) * feat(proxy): add /v1/memory CRUD endpoints with user/team scoping New LiteLLM_MemoryTable stores user/team-scoped key/value entries with optional JSON metadata. Value is a String (LLM-readable text) and metadata is an optional Json? envelope, matching the Letta + mem0 hybrid model so future structured fields can be added without a schema migration. Endpoints: POST /v1/memory - create GET /v1/memory - list (caller-scoped; admins see all) GET /v1/memory/{key} - fetch one PUT /v1/memory/{key} - upsert DELETE /v1/memory/{key} - delete Non-admin callers cannot set a user_id/team_id other than their own. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix(proxy/memory): omit metadata field when None on create Prisma's Python client rejects `metadata=None` on a `Json?` field with "A value is required but not set" — the field must be omitted from the `data` dict entirely to store SQL NULL. Build the create payload conditionally in both `create_memory` and the PUT-create branch of `upsert_memory`. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * feat(ui): add Memory page to view/manage /v1/memory entries Adds a new "Memory" sidebar item under Tools so users can see what their agents have stored. Lists all memories visible to the caller (scoped by the backend), with a key-search filter, preview column, scope tags, and view/edit/delete actions. Create modal accepts optional JSON metadata. - networking.tsx: fetchMemoryList / createMemory / updateMemory / deleteMemory wired to the /v1/memory CRUD endpoints. - MemoryView + MemoryEditModal: new antd-based components (per CLAUDE.md: use antd for new UI, not tremor). - page.tsx + leftnav.tsx: wire the "memory" route + sidebar entry. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * feat(memory): add key_prefix filter + promote Memory to AI GATEWAY nav Backend: - GET /v1/memory now accepts `key_prefix` for Redis-style namespace scans (e.g. `?key_prefix=user:`). When both `key` and `key_prefix` are passed, `key_prefix` wins. - Prefix filter sits under the visibility filter in the Prisma where clause, so it can never leak rows across user/team scopes. - New tests: prefix match, and cross-scope isolation (another user's `user:*` rows must not appear in the caller's results). UI: - Memory moved from a Tools submenu to a top-level AI GATEWAY item (alongside Agents, MCP Servers, Skills) — it's an API primitive, not a tool-management surface. - Search box now drives prefix search, matching the Redis mental model ("type the namespace, see everything under it"). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix(memory): enforce unique key per scope by using NULLS NOT DISTINCT The unique constraint `(key, user_id, team_id)` on LiteLLM_MemoryTable silently allowed duplicates when user_id or team_id was NULL, because Postgres treats every NULL as distinct by default (ANSI semantics). A caller with no team_id could POST the same key three times and get three rows. Migration: 1. Dedupe existing rows, keeping the most recent per (key, user_id, team_id), using `IS NOT DISTINCT FROM` so NULL == NULL. 2. Drop the old unique index. 3. Recreate it with `NULLS NOT DISTINCT` (Postgres 15+). No code change: POST already returns 409 on unique-violation error messages — it just wasn't firing before because the constraint didn't catch the NULL-team case. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix(memory): make key globally unique, 409 on any duplicate Switches from the compound unique `(key, user_id, team_id)` to a simple `key @unique`. The compound form silently allowed duplicates when user_id or team_id was NULL (Postgres treats each NULL as distinct), so callers could POST the same key repeatedly. Globally-unique key means one row per key, period — any duplicate create → 409. - schema.prisma (×3): `key String @unique`, drop `@@unique(...)`. - initial add_memory_table migration: unique index on (key) only. - Remove the now-unused follow-up NULLS NOT DISTINCT migration. - Endpoint error message simplified ("already exists" — no "for this scope"). - Test fake's create() now enforces global key uniqueness. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix(ui/memory): full-width layout + user/teams-style columns - Add `w-full` to the MemoryView outer div so the page fills the flex-flex-1 container (was collapsing to intrinsic width). - Replace the combined "Scope" column with separate User ID / Team ID columns, matching the layout of the Users / Teams pages: ID, Name, Preview, User ID, Team ID, Updated, Actions. - IDs render with a truncated mono label + copy-to-clipboard button, same pattern as view_users. - Detail drawer now shows Memory ID / User ID / Team ID as separate fields instead of stacked color tags. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix(ui/memory): use clean MCP-style ID pill, drop copy icons The ID / User ID / Team ID columns showed a mono text blob with a copy-to-clipboard icon next to each value — too busy compared to the MCP Servers page. Swap the renderer for MCP's pill style: - Truncated mono ID inside a blue Tailwind pill (`font-mono text-blue-600 bg-blue-50 ... rounded-md border`). - No copy icon. Full ID surfaces via tooltip. - ID column is a button that opens the detail drawer on click; user/team ID pills are static (not clickable). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix(memory): address greptile review feedback Addresses 5 greptile findings (3/5 → higher confidence target): 1. Identity-less orphan rows (P1): non-admin callers with no user_id AND no team_id could create rows that the visibility filter would never match again. Now rejected up front with 400 — caller must authenticate with a scoped key or act as PROXY_ADMIN. 2. Upsert race returning 500 (P1): PUT's check-then-create isn't atomic; a concurrent writer could slip a row in between the 404-check and the create call. Now catch unique-violation on create, re-read, and fall through to update — PUT stays idempotent. If the conflicting row belongs to a different scope, surface a 409 instead of 500. 3. PUT-create scope inconsistency (P2): PUT's create branch always used the caller's own user_id/team_id, so admins couldn't bootstrap rows scoped elsewhere via PUT (only POST). Now PUT-create calls the shared `_resolve_scope()` helper, matching POST semantics. 4. Stale schema comment (P2): schema said "Keyed by (key, user_id, team_id)" but `key` is globally unique. Updated all three schema copies to reflect the actual design. 5. UI silently truncated at 200 (P2): MemoryView fetched pageSize=200 with no load-more. Swapped to real server-side pagination driven by `data.total`; page size is now 50 and the pager is a real AntD control. Also extracts a shared `_resolve_scope()` helper and `_is_unique_violation()` from create_memory so POST and PUT don't drift on the scope/error logic. Tests: +3 new (identity-less 400, PUT admin bootstrap, PUT race → update), 18/18 pass. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix(memory): typed Prisma error + explicit-null metadata on PUT Two more greptile threads from the last review: - Unique-violation detection was string-matching "Unique"/"UniqueViolation" in the exception message, fragile across Prisma/driver versions. Now check the typed error `code == "P2002"` first, with string fallback. - PUT could not distinguish "metadata omitted" from "metadata: null" — both parsed as `None`, so callers had no way to clear stored metadata. Switch to Pydantic v2's `model_fields_set` to tell which fields the caller actually sent; explicit null now clears the column. New tests: - explicit null clears metadata - omitted metadata preserves existing value Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix(ui/memory): send explicit null when user clears metadata Addresses the remaining P1 from the last greptile review: When the edit modal's metadata textarea was cleared and saved, `metadataParsed` stayed `undefined`, `JSON.stringify` dropped the key entirely, and the backend's `model_fields_set` guard therefore left the stored metadata untouched — UI showed success but nothing changed. Now: empty textarea on edit → send explicit `null` so the backend sees `metadata` in `model_fields_set` and clears the column. Empty textarea on create still maps to `undefined` (field omitted) to avoid Prisma's `Json? = None` quirk on insert. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix(ui/memory): preserve slashes in key path encoding The backend route `/v1/memory/{key:path}` supports keys with slashes, but `encodeURIComponent` encoded `/` as `%2F`. Some proxies (nginx default, CloudFlare, AWS ALB) reject or re-decode `%2F` mid-flight, so UI update/delete calls on slash-containing keys could fail or silently misroute. New helper `encodeMemoryKeyForPath` splits by `/`, URL-encodes each segment, then rejoins with literal `/`. Every other unsafe char (spaces, `?`, `#`, `%`) stays encoded per-segment; slashes stay as path delimiters, matching what the `:path` converter expects. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix(ui/memory): drop misleading client-side column sorters With server-side pagination, client sorters on `key` and `updated_at` only reorder the current page while pretending to sort the full dataset — users would see "sorted by name" but only the visible 50 rows would actually be sorted. Remove the sorters. The backend already returns rows in `updated_at DESC` order (sensible default for a memory view), and users can narrow the result with the key-prefix filter. Greptile also flagged missing `@@map` on the new model as a "consistency" issue, but only 1 of 59 tables in this repo uses `@@map` — the dominant pattern is to rely on Prisma's default (model name == table name). Skipping that finding as a false-positive on convention. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix(memory): compose visibility + key filters via explicit AND Greptile P1 (filter-fragility): `where.update(vis)` was semantically correct today, but dict-merging by key meant any future visibility filter that grew a new top-level "OR" would silently clobber the existing key filter. Compose explicitly instead: where = {"AND": [key_filter, vis]} Applied to both `list_memory` and `_find_memory_for_caller`. When either side is empty (admin has no visibility filter; list has no key filter), skip the wrapper and use the non-empty side directly to keep the generated SQL clean. Test fake's `_matches` now understands top-level `AND` too. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * refactor(ui/memory): wrap write helpers with react-query useMutation Previously the Memory view read via `useQuery` but called the raw create/update/delete fetch helpers directly in handlers, tracking loading state with a local `submitting` flag and invalidating state via `refetch()`. That mixes two concerns: - it skips react-query's mutation state (isPending / isError / isSuccess) - `refetch()` only retouches the currently-mounted query instance, not other cached pages, so navigating back to an older page could show stale rows Switch the three write paths to `useMutation`: - `createMutation`, `updateMutation`, `deleteMutation` — each owns the mutation fn, success toast, and error toast. - Success handlers invalidate the whole `["memoryList", ...]` prefix via `queryClient.invalidateQueries`, so every cached page refetches (pagination + filter-aware). - Refresh button now invalidates instead of `refetch()`, keeping all behavior consistent. - handleSave/handleDelete become thin adapters that call `.mutateAsync`; their errors are swallowed locally since the mutation's onError has already surfaced the toast. Also tightened the edit modal's key-field tooltip to reflect the actual global-unique semantics (was "Unique per user/team scope"). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix(memory): close cross-user write gap + sanitize 500 errors (Veria) Addresses two Veria findings: **High — cross-user memory tampering via team membership.** The visibility filter uses an OR (`user_id == caller OR team_id == caller`) so team members can SEE each other's team-scoped rows. That's intentional for list/get. But because PUT/DELETE used the same filter to find the target row, any team member could overwrite or delete a teammate's *personal* row whenever both `user_id` and `team_id` were stamped on it — broader visibility was being silently treated as broader authority. New `_assert_write_access(row, caller)` enforces ownership for mutations. Non-admin rules: - The row's `user_id` must match the caller (personal ownership), OR - The row has no `user_id` and its `team_id` matches the caller's team (a "pure team row" intended for shared writes). Admins bypass the check. The same gate runs in PUT (both regular and post-race-recovery branches) and DELETE. **Medium — DB internals leaked through 500 detail.** Every `except` block was raising `HTTPException(500, detail=str(e))`, which surfaces Prisma error strings (table/column names, host:port, error class names) to API callers. New `_internal_error()` helper logs the real exception server-side and returns a generic, caller-safe `detail`. Applied to create, list, upsert (general fallthrough), and delete. Also tightened the race-recovery 409 message to drop the "in a different scope" wording — the caller never needs to know whose scope it lives in. Tests (+5): - teammate cannot overwrite personal row → 403 - teammate cannot delete personal row → 403 - teammate CAN modify pure team row (no user_id stamped) → 200 - admin bypasses write-auth → 200 - 500 response never echoes Prisma internals (table/host/class names) 25/25 unit tests pass. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix(memory): require team admin to modify pure team rows Tightens the write-authorization rule for "pure team rows" (rows with no user_id stamped, only team_id) to match the pattern used by team-management endpoints (`_is_user_team_admin` + `_is_user_org_admin_for_team`): - Plain team members can READ team rows via the OR visibility filter (intentional, unchanged). - Only PROXY_ADMIN, team admins of the row's team_id, or org admins for the team's organization may MODIFY them. Plain members get 403. `_assert_write_access` is now async and takes the prisma_client so it can fetch the team and run the existing `_is_user_team_admin` / `_is_user_org_admin_for_team` helpers from `litellm.proxy.management_endpoints.common_utils`. The org-admin path is best-effort: it calls `get_user_object`, which depends on the proxy_server module being initialized, so any exception there is treated as "not an org admin" rather than crashing the request. Tests: - team admin can modify pure team row → 200 - plain team member cannot modify pure team row → 403 - plain team member cannot delete pure team row → 403 Updates the test fake to add a tiny `litellm_teamtable.find_unique` implementation and a `_make_team(team_id, admin_user_ids=[...])` helper. 27/27 unit tests pass. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix: mypy + UI page-metadata sync for memory page Two CI failures: 1. mypy: `_find_memory_for_caller` had `key_filter` inferred as `dict[str, str]` (literal type) and the conditional `{"AND": [key_filter, vis]}` returned `dict[str, list[...]]`, so the join site failed `dict-item` typing. Annotate both intermediates as `dict` so mypy widens the value type. 2. UI test (`page_utils.test.ts > should have descriptions for all pages`): every leftnav entry must have a description in `page_metadata.ts`, and `memory` was missing. Added a one-line description, matching the style of neighboring entries. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * [Feat] Day-0 support for GPT-5.5 and GPT-5.5 Pro (#26449) * feat(openai): day-0 support for GPT-5.5 and GPT-5.5 Pro Add pricing + capability entries for the new GPT-5.5 family launched by OpenAI on 2026-04-24: - gpt-5.5 / gpt-5.5-2026-04-23 (chat): $5/$30/$0.50 per 1M input/output/cached input - gpt-5.5-pro / gpt-5.5-pro-2026-04-23 (responses-only): $60/$360/$6 per 1M input/output/cached input Other fees (long-context >272k, flex, batches, priority, cache discounts) follow the same ratios as GPT-5.4, with context window retained at 1.05M input / 128K output. No transformation / classifier code changes are required: OpenAIGPT5Config.is_model_gpt_5_4_plus_model() already matches 5.5+ via numeric version parsing, and model registration is driven from the JSON. The existing responses-API bridge for tools + reasoning_effort (litellm/main.py:970) already covers gpt-5.5-pro. Tests: - GPT5_MODELS regression list now covers gpt-5.5-pro and dated variants - New test_generic_cost_per_token_gpt55_pro cost-calc test - Updated test_generic_cost_per_token_gpt55 for long-context fields * fix(openai): mirror reasoning_effort flags onto gpt-5.5 dated variants gpt-5.5-2026-04-23 and gpt-5.5-pro-2026-04-23 were missing the supports_none_reasoning_effort, supports_xhigh_reasoning_effort, and supports_minimal_reasoning_effort flags that their non-dated counterparts define. Reasoning-effort routing in OpenAIGPT5Config is fully capability-driven from these JSON flags — since an absent flag is treated as False for opt-in levels (xhigh), users pinning to a dated snapshot would silently lose xhigh support and diverge from the base alias on logprobs + flexible temperature handling. Copy the flags onto both dated variants so every dated snapshot inherits the base model's reasoning-effort capability profile. Adds a parametrized regression test that asserts supports_{none,minimal,xhigh}_reasoning_effort parity between each dated variant and its non-dated counterpart, preventing future drift when new snapshots are added. * fix(schema): close LiteLLM_MemoryTable model brace dropped during merge The rebase against `litellm_internal_staging` (which added `LiteLLM_AdaptiveRouterState` / `LiteLLM_AdaptiveRouterSession`) left the closing brace of `LiteLLM_MemoryTable` missing in all three schema copies — the next model declaration ended up parsed as a field of the memory table, surfacing as the CI prisma error: error: This line is not a valid field or attribute definition. --> schema.prisma:1250 | 1249 | // Per-(router, request_type, model) Beta posterior for the adaptive router. 1250 | model LiteLLM_AdaptiveRouterState { Add the missing `}` (and the standard blank line) after the memory table's `@@index([team_id])` in `schema.prisma`, `litellm/proxy/schema.prisma`, and `litellm-proxy-extras/litellm_proxy_extras/schema.prisma`. `prisma generate --schema litellm/proxy/schema.prisma` now runs clean; 27/27 memory unit tests pass. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com> Co-authored-by: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
2026-04-25 09:38:07 +08:00
// User/team-scoped memory store with a GLOBAL unique key.
// `value` is a string (typically markdown/text meant for LLM context);
// `metadata` is an optional JSON envelope for structured tags without schema changes.
// Note: `key` is globally unique across all users/teams — callers should
// namespace their keys (e.g. `user:123:notes`) if they need per-user isolation.
// `user_id` / `team_id` stamp ownership for visibility filtering, but do NOT
// participate in the unique constraint.
model LiteLLM_MemoryTable {
memory_id String @id @default(uuid())
key String @unique
value String
metadata Json?
user_id String?
team_id String?
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
@@index([user_id])
@@index([team_id])
}
// Per-(router, request_type, model) Beta posterior for the adaptive router.
model LiteLLM_AdaptiveRouterState {
router_name String
request_type String
model_name String
alpha Float
beta Float
total_samples Int @default(0)
last_updated_at DateTime @default(now()) @updatedAt
@@id([router_name, request_type, model_name])
}
// Per-(session, router, model) signal counters for the adaptive router.
model LiteLLM_AdaptiveRouterSession {
session_id String
router_name String
model_name String
classified_type String
misalignment_count Int @default(0)
stagnation_count Int @default(0)
disengagement_count Int @default(0)
satisfaction_count Int @default(0)
failure_count Int @default(0)
loop_count Int @default(0)
exhaustion_count Int @default(0)
last_user_content String?
last_assistant_content String?
tool_call_history Json @default("[]")
pending_tool_calls Json @default("{}")
turn_count Int @default(0)
last_processed_turn Int @default(-1)
clean_credit_awarded Boolean @default(false)
terminal_status Int?
last_activity_at DateTime @default(now()) @updatedAt
@@id([session_id, router_name, model_name])
@@index([last_activity_at], map: "idx_adaptive_router_session_activity")
}
// ---------------------------------------------------------------------------
// Workflow Run Tracking
//
// Generic durable state tracking for any agent or automated workflow.
// Design: three tables — run (header + materialized status), event (append-only
// source of truth for state transitions), message (conversation inbox/outbox).
//
// Usage:
// - Set `workflow_type` to identify the owning system (e.g. "shin-builder").
// - Store domain-specific fields in `metadata` (worktree_path, pr_url, etc.).
// - `session_id` on WorkflowRun matches `x-litellm-session-id` header sent to
// the proxy — all spend logs for this run are automatically tagged.
// ---------------------------------------------------------------------------
// One instance of work being done. `status` is a materialized cache of the
// latest event; the event log is the authoritative source of truth.
model LiteLLM_WorkflowRun {
run_id String @id @default(uuid())
session_id String @unique @default(uuid())
workflow_type String
status String @default("pending")
created_by String? // user_id of the key that created this run; null = created by master key
created_at DateTime @default(now())
updated_at DateTime @updatedAt
input Json?
output Json?
metadata Json?
events LiteLLM_WorkflowEvent[]
messages LiteLLM_WorkflowMessage[]
@@index([workflow_type, status])
@@index([session_id])
@@index([created_at])
@@index([created_by])
}
// Append-only log of state transitions. Never mutate rows here.
// `step_name` and `event_type` are caller-defined strings — no hardcoded enums.
// Status auto-update rules (applied by the append endpoint):
// step.started → run.status = running
// step.failed → run.status = failed
// hook.waiting → run.status = paused
// hook.received → run.status = running
model LiteLLM_WorkflowEvent {
event_id String @id @default(uuid())
run_id String
event_type String
step_name String
sequence_number Int
data Json?
created_at DateTime @default(now())
run LiteLLM_WorkflowRun @relation(fields: [run_id], references: [run_id])
@@unique([run_id, sequence_number])
@@index([run_id])
}
// Conversation inbox/outbox — full message content, separate from the durable
// event log. Spend logs truncate messages; this table stores them in full.
// `session_id` here is the Claude --resume session ID (or similar).
model LiteLLM_WorkflowMessage {
message_id String @id @default(uuid())
run_id String
role String
content String
sequence_number Int
session_id String?
created_at DateTime @default(now())
run LiteLLM_WorkflowRun @relation(fields: [run_id], references: [run_id])
@@unique([run_id, sequence_number])
@@index([run_id])
}