diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index ba605e316d..79b1b121e9 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -1,689 +1,468 @@ +--- +id: index +title: Getting Started +sidebar_label: Quickstart +--- + import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; +import NavigationCards from '@site/src/components/NavigationCards'; +import Image from '@theme/IdealImage'; -# LiteLLM - Getting Started + -https://github.com/BerriAI/litellm +**LiteLLM** is an open-source library that gives you a single, unified interface to call 100+ LLMs — OpenAI, Anthropic, Vertex AI, Bedrock, and more — using the OpenAI format. -## **Call 100+ LLMs using the OpenAI Input/Output Format** +- Call any provider using the same `completion()` interface — no re-learning the API for each one +- Consistent output format regardless of which provider or model you use +- Built-in retry / fallback logic across multiple deployments via the [Router](./routing.md) +- Self-hosted [LLM Gateway (Proxy)](./simple_proxy) with virtual keys, cost tracking, and an admin UI -- Translate inputs to provider's endpoints (`/chat/completions`, `/responses`, `/embeddings`, `/images`, `/audio`, `/batches`, and more) -- [Consistent output](https://docs.litellm.ai/docs/supported_endpoints) - same response format regardless of which provider you use -- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing) -- Track spend & set budgets per project [LiteLLM Proxy Server](https://docs.litellm.ai/docs/simple_proxy) +[![PyPI](https://img.shields.io/pypi/v/litellm.svg)](https://pypi.org/project/litellm/) +[![GitHub Stars](https://img.shields.io/github/stars/BerriAI/litellm?style=social)](https://github.com/BerriAI/litellm) -## How to use LiteLLM +--- -You can use LiteLLM through either the Proxy Server or Python SDK. Both gives you a unified interface to access multiple LLMs (100+ LLMs). Choose the option that best fits your needs: - - - - - - - - - - - - - - - - - - - - - - - - - - -
LiteLLM Proxy ServerLiteLLM Python SDK
Use CaseCentral service (LLM Gateway) to access multiple LLMsUse LiteLLM directly in your Python code
Who Uses It?Gen AI Enablement / ML Platform TeamsDevelopers building LLM projects
Key Features• Centralized API gateway with authentication & authorization
• Multi-tenant cost tracking and spend management per project/user
• Per-project customization (logging, guardrails, caching)
• Virtual keys for secure access control
• Admin dashboard UI for monitoring and management		• Direct Python library integration in your codebase
• Router with retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - Router
• Application-level load balancing and cost tracking
• Exception handling with OpenAI-compatible errors
• Observability callbacks (Lunary, MLflow, Langfuse, etc.)
- - -## **LiteLLM Python SDK** - -### Basic usage - - - Open In Colab - +## Installation ```shell pip install litellm ``` - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["OPENAI_API_KEY"] = "your-api-key" - -response = completion( - model="openai/gpt-4o", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["ANTHROPIC_API_KEY"] = "your-api-key" - -response = completion( - model="anthropic/claude-3-sonnet-20240229", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["XAI_API_KEY"] = "your-api-key" - -response = completion( - model="xai/grok-2-latest", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - -```python -from litellm import completion -import os - -# auth: run 'gcloud auth application-default' -os.environ["VERTEXAI_PROJECT"] = "hardy-device-386718" -os.environ["VERTEXAI_LOCATION"] = "us-central1" - -response = completion( - model="vertex_ai/gemini-1.5-pro", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["NVIDIA_NIM_API_KEY"] = "nvidia_api_key" -os.environ["NVIDIA_NIM_API_BASE"] = "nvidia_nim_endpoint_url" - -response = completion( - model="nvidia_nim/", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - -```python -from litellm import completion -import os - -os.environ["HUGGINGFACE_API_KEY"] = "huggingface_api_key" - -# e.g. Call 'WizardLM/WizardCoder-Python-34B-V1.0' hosted on HF Inference endpoints -response = completion( - model="huggingface/WizardLM/WizardCoder-Python-34B-V1.0", - messages=[{ "content": "Hello, how are you?","role": "user"}], - api_base="https://my-endpoint.huggingface.cloud" -) - -print(response) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["AZURE_API_KEY"] = "" -os.environ["AZURE_API_BASE"] = "" -os.environ["AZURE_API_VERSION"] = "" - -# azure call -response = completion( - "azure/", - messages = [{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - -```python -from litellm import completion - -response = completion( - model="ollama/llama2", - messages = [{ "content": "Hello, how are you?","role": "user"}], - api_base="http://localhost:11434" -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["OPENROUTER_API_KEY"] = "openrouter_api_key" - -response = completion( - model="openrouter/google/palm-2-chat-bison", - messages = [{ "content": "Hello, how are you?","role": "user"}], -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables. Visit https://novita.ai/settings/key-management to get your API key -os.environ["NOVITA_API_KEY"] = "novita-api-key" - -response = completion( - model="novita/deepseek/deepseek-r1", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for insturctions on obtaining a key -os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key" - -response = completion( - model="vercel_ai_gateway/openai/gpt-4o", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - -### Response Format (OpenAI Chat Completions Format) - -```json -{ - "id": "chatcmpl-565d891b-a42e-4c39-8d14-82a1f5208885", - "created": 1734366691, - "model": "gpt-4o-2024-08-06", - "object": "chat.completion", - "system_fingerprint": null, - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "Hello! As an AI language model, I don't have feelings, but I'm operating properly and ready to assist you with any questions or tasks you may have. How can I help you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null - } - } - ], - "usage": { - "completion_tokens": 43, - "prompt_tokens": 13, - "total_tokens": 56, - "completion_tokens_details": null, - "prompt_tokens_details": { - "audio_tokens": null, - "cached_tokens": 0 - }, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } -} -``` - -### Streaming -Set `stream=True` in the `completion` args. - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["OPENAI_API_KEY"] = "your-api-key" - -response = completion( - model="openai/gpt-4o", - messages=[{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["ANTHROPIC_API_KEY"] = "your-api-key" - -response = completion( - model="anthropic/claude-3-sonnet-20240229", - messages=[{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["XAI_API_KEY"] = "your-api-key" - -response = completion( - model="xai/grok-2-latest", - messages=[{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - -```python -from litellm import completion -import os - -# auth: run 'gcloud auth application-default' -os.environ["VERTEX_PROJECT"] = "hardy-device-386718" -os.environ["VERTEX_LOCATION"] = "us-central1" - -response = completion( - model="vertex_ai/gemini-1.5-pro", - messages=[{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["NVIDIA_NIM_API_KEY"] = "nvidia_api_key" -os.environ["NVIDIA_NIM_API_BASE"] = "nvidia_nim_endpoint_url" - -response = completion( - model="nvidia_nim/", - messages=[{ "content": "Hello, how are you?","role": "user"}] - stream=True, -) -``` - - - - -```python -from litellm import completion -import os - -os.environ["HUGGINGFACE_API_KEY"] = "huggingface_api_key" - -# e.g. Call 'WizardLM/WizardCoder-Python-34B-V1.0' hosted on HF Inference endpoints -response = completion( - model="huggingface/WizardLM/WizardCoder-Python-34B-V1.0", - messages=[{ "content": "Hello, how are you?","role": "user"}], - api_base="https://my-endpoint.huggingface.cloud", - stream=True, -) - -print(response) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["AZURE_API_KEY"] = "" -os.environ["AZURE_API_BASE"] = "" -os.environ["AZURE_API_VERSION"] = "" - -# azure call -response = completion( - "azure/", - messages = [{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - - -```python -from litellm import completion - -response = completion( - model="ollama/llama2", - messages = [{ "content": "Hello, how are you?","role": "user"}], - api_base="http://localhost:11434", - stream=True, -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["OPENROUTER_API_KEY"] = "openrouter_api_key" - -response = completion( - model="openrouter/google/palm-2-chat-bison", - messages = [{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables. Visit https://novita.ai/settings/key-management to get your API key -os.environ["NOVITA_API_KEY"] = "novita_api_key" - -response = completion( - model="novita/deepseek/deepseek-r1", - messages = [{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for insturctions on obtaining a key -os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key" - -response = completion( - model="vercel_ai_gateway/openai/gpt-4o", - messages = [{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - - -### Streaming Response Format (OpenAI Format) - -```json -{ - "id": "chatcmpl-2be06597-eb60-4c70-9ec5-8cd2ab1b4697", - "created": 1734366925, - "model": "claude-3-sonnet-20240229", - "object": "chat.completion.chunk", - "system_fingerprint": null, - "choices": [ - { - "finish_reason": null, - "index": 0, - "delta": { - "content": "Hello", - "role": "assistant", - "function_call": null, - "tool_calls": null, - "audio": null - }, - "logprobs": null - } - ] -} -``` - -### Exception handling - -LiteLLM maps exceptions across all supported providers to the OpenAI exceptions. All our exceptions inherit from OpenAI's exception types, so any error-handling you have for that, should work out of the box with LiteLLM. - -```python -import litellm -from litellm import completion -import os - -os.environ["ANTHROPIC_API_KEY"] = "bad-key" -try: - completion(model="anthropic/claude-instant-1", messages=[{"role": "user", "content": "Hey, how's it going?"}]) -except litellm.AuthenticationError as e: - # Thrown when the API key is invalid - print(f"Authentication failed: {e}") -except litellm.RateLimitError as e: - # Thrown when you've exceeded your rate limit - print(f"Rate limited: {e}") -except litellm.APIError as e: - # Thrown for general API errors - print(f"API error: {e}") -``` -### See How LiteLLM Transforms Your Requests - -Want to understand how LiteLLM parses and normalizes your LLM API requests? Use the `/utils/transform_request` endpoint to see exactly how your request is transformed internally. - -You can try it out now directly on our Demo App! -Go to the [LiteLLM API docs for transform_request](https://litellm-api.up.railway.app/#/llm%20utils/transform_request_utils_transform_request_post) - -LiteLLM will show you the normalized, provider-agnostic version of your request. This is useful for debugging, learning, and understanding how LiteLLM handles different providers and options. - - -### Logging Observability - Log LLM Input/Output ([Docs](https://docs.litellm.ai/docs/observability/callbacks)) -LiteLLM exposes pre defined callbacks to send data to Lunary, MLflow, Langfuse, Helicone, Promptlayer, Traceloop, Slack - -```python -from litellm import completion - -## set env variables for logging tools (API key set up is not required when using MLflow) -os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" # get your public key at https://app.lunary.ai/settings -os.environ["HELICONE_API_KEY"] = "your-helicone-key" -os.environ["LANGFUSE_PUBLIC_KEY"] = "" -os.environ["LANGFUSE_SECRET_KEY"] = "" - -os.environ["OPENAI_API_KEY"] - -# set callbacks -litellm.success_callback = ["lunary", "mlflow", "langfuse", "helicone"] # log input/output to lunary, mlflow, langfuse, helicone - -#openai call -response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}]) -``` - -### Track Costs, Usage, Latency for streaming -Use a callback function for this - more info on custom callbacks: https://docs.litellm.ai/docs/observability/custom_callback - -```python -import litellm - -# track_cost_callback -def track_cost_callback( - kwargs, # kwargs to completion - completion_response, # response from completion - start_time, end_time # start/end time -): - try: - response_cost = kwargs.get("response_cost", 0) - print("streaming response_cost", response_cost) - except: - pass -# set callback -litellm.success_callback = [track_cost_callback] # set custom callback function - -# litellm.completion() call -response = completion( - model="gpt-3.5-turbo", - messages=[ - { - "role": "user", - "content": "Hi 👋 - i'm openai" - } - ], - stream=True -) -``` - -## **LiteLLM Proxy Server (LLM Gateway)** - -Track spend across multiple projects/people - -![ui_3](https://github.com/BerriAI/litellm/assets/29436595/47c97d5e-b9be-4839-b28c-43d7f4f10033) - -The proxy provides: - -1. [Hooks for auth](https://docs.litellm.ai/docs/proxy/virtual_keys#custom-auth) -2. [Hooks for logging](https://docs.litellm.ai/docs/proxy/logging#step-1---create-your-custom-litellm-callback-class) -3. [Cost tracking](https://docs.litellm.ai/docs/proxy/virtual_keys#tracking-spend) -4. [Rate Limiting](https://docs.litellm.ai/docs/proxy/users#set-rate-limits) - -### 📖 Proxy Endpoints - [Swagger Docs](https://litellm-api.up.railway.app/) - -Go here for a complete tutorial with keys + rate limits - [**here**](./proxy/docker_quick_start.md) - -### Quick Start Proxy - CLI +To run the full Proxy Server (LLM Gateway): ```shell pip install 'litellm[proxy]' ``` -#### Step 1: Start litellm proxy +--- + +## Quick Start + +Make your first LLM call using the provider of your choice: + - +```python +from litellm import completion +import os -```shell -$ litellm --model huggingface/bigcode/starcoder +os.environ["OPENAI_API_KEY"] = "your-api-key" -#INFO: Proxy running on http://0.0.0.0:4000 +response = completion( + model="openai/gpt-4o", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) +print(response.choices[0].message.content) ``` + - +```python +from litellm import completion +import os +os.environ["ANTHROPIC_API_KEY"] = "your-api-key" -Step 1. CREATE config.yaml +response = completion( + model="anthropic/claude-3-5-sonnet-20241022", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) +print(response.choices[0].message.content) +``` -Example `litellm_config.yaml` + + -```yaml +```python +from litellm import completion +import os + +# auth: run 'gcloud auth application-default login' +os.environ["VERTEXAI_PROJECT"] = "your-project-id" +os.environ["VERTEXAI_LOCATION"] = "us-central1" + +response = completion( + model="vertex_ai/gemini-1.5-pro", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) +print(response.choices[0].message.content) +``` + + + + +```python +from litellm import completion +import os + +os.environ["AWS_ACCESS_KEY_ID"] = "your-key" +os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret" +os.environ["AWS_REGION_NAME"] = "us-east-1" + +response = completion( + model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) +print(response.choices[0].message.content) +``` + + + + +```python +from litellm import completion + +response = completion( + model="ollama/llama3", + messages=[{"role": "user", "content": "Hello, how are you?"}], + api_base="http://localhost:11434" +) +print(response.choices[0].message.content) +``` + + + + +```python +from litellm import completion +import os + +os.environ["AZURE_API_KEY"] = "your-key" +os.environ["AZURE_API_BASE"] = "https://your-resource.openai.azure.com" +os.environ["AZURE_API_VERSION"] = "2024-02-01" + +response = completion( + model="azure/your-deployment-name", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) +print(response.choices[0].message.content) +``` + + + + +Every response follows the OpenAI Chat Completions format, regardless of provider. ✅ + +### Response Format + +Non-streaming responses return a `ModelResponse` object: + +```json +{ + "id": "chatcmpl-abc123", + "object": "chat.completion", + "created": 1677858242, + "model": "gpt-4o", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! I'm doing well, thanks for asking." + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 13, + "completion_tokens": 12, + "total_tokens": 25 + } +} +``` + +Streaming responses (`stream=True`) yield `ModelResponseStream` chunks: + +```json +{ + "id": "chatcmpl-abc123", + "object": "chat.completion.chunk", + "created": 1677858242, + "model": "gpt-4o", + "choices": [ + { + "index": 0, + "delta": { + "role": "assistant", + "content": "Hello" + }, + "finish_reason": null + } + ] +} +``` + +📖 [Full output format reference →](./completion/output) + +:::tip Open in Colab + +Open In Colab + +::: + +--- + +## New to LiteLLM? + +**Want to get started fast?** Head to [Tutorials](/docs/tutorials) for step-by-step walkthroughs — AI coding tools, agent SDKs, proxy setup, and more. + +**Need to understand a specific feature?** Check [Guides](/docs/guides) for streaming, function calling, prompt caching, and other how-tos. + +--- + +## Choose Your Path + + + +--- + +## LiteLLM Python SDK + +### Streaming + +Add `stream=True` to receive chunks as they are generated: + +```python +from litellm import completion +import os + +os.environ["OPENAI_API_KEY"] = "your-api-key" + +for chunk in completion( + model="openai/gpt-4o", + messages=[{"role": "user", "content": "Write a short poem"}], + stream=True, +): + print(chunk.choices[0].delta.content or "", end="") +``` + +### Exception Handling + +LiteLLM maps every provider's errors to the OpenAI exception types — your existing error handling works out of the box: + +```python +import litellm + +try: + litellm.completion( + model="anthropic/claude-instant-1", + messages=[{"role": "user", "content": "Hey!"}] + ) +except litellm.AuthenticationError as e: + print(f"Bad API key: {e}") +except litellm.RateLimitError as e: + print(f"Rate limited: {e}") +except litellm.APIError as e: + print(f"API error: {e}") +``` + +### Logging & Observability + +Send input/output to Langfuse, MLflow, Helicone, Lunary, and more with a single line: + +```python +import litellm + +litellm.success_callback = ["langfuse", "mlflow", "helicone"] + +response = litellm.completion( + model="gpt-4o", + messages=[{"role": "user", "content": "Hi!"}] +) +``` + +📖 [See all observability integrations →](/docs/observability/agentops_integration) + +### Track Costs & Usage + +Use a callback to capture cost per response: + +```python +import litellm + +def track_cost(kwargs, completion_response, start_time, end_time): + print("Cost:", kwargs.get("response_cost", 0)) + +litellm.success_callback = [track_cost] + +litellm.completion( + model="gpt-4o", + messages=[{"role": "user", "content": "Hello!"}], + stream=True +) +``` + +📖 [Custom callback docs →](./observability/custom_callback) + +--- + +## LiteLLM Proxy Server (LLM Gateway) + +The proxy is a self-hosted OpenAI-compatible gateway. Any client that works with OpenAI works with the proxy — no code changes needed. + +![LiteLLM Proxy Dashboard](https://github.com/BerriAI/litellm/assets/29436595/47c97d5e-b9be-4839-b28c-43d7f4f10033) + +#### Step 1 — Start the proxy + + + + +```shell +litellm --model huggingface/bigcode/starcoder +# Proxy running on http://0.0.0.0:4000 +``` + + + + +```yaml title="litellm_config.yaml" model_list: - model_name: gpt-3.5-turbo litellm_params: - model: azure/ - api_base: os.environ/AZURE_API_BASE # runs os.getenv("AZURE_API_BASE") - api_key: os.environ/AZURE_API_KEY # runs os.getenv("AZURE_API_KEY") + model: azure/your-deployment + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY api_version: "2023-07-01-preview" ``` -Step 2. RUN Docker Image - ```shell docker run \ - -v $(pwd)/litellm_config.yaml:/app/config.yaml \ - -e AZURE_API_KEY=d6*********** \ - -e AZURE_API_BASE=https://openai-***********/ \ - -p 4000:4000 \ - docker.litellm.ai/berriai/litellm:main-latest \ - --config /app/config.yaml --detailed_debug + -v $(pwd)/litellm_config.yaml:/app/config.yaml \ + -e AZURE_API_KEY=your-key \ + -e AZURE_API_BASE=https://your-resource.openai.azure.com/ \ + -p 4000:4000 \ + docker.litellm.ai/berriai/litellm:main-latest \ + --config /app/config.yaml --detailed_debug ``` - -#### Step 2: Make ChatCompletions Request to Proxy +#### Step 2 — Call it with the OpenAI client ```python -import openai # openai v1.0.0+ -client = openai.OpenAI(api_key="anything",base_url="http://0.0.0.0:4000") # set proxy to base_url -# request sent to model set on litellm proxy, `litellm --model` -response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [ - { - "role": "user", - "content": "this is a test request, write a short poem" - } -]) +import openai -print(response) +client = openai.OpenAI(api_key="anything", base_url="http://0.0.0.0:4000") + +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Write a short poem"}] +) +print(response.choices[0].message.content) ``` -## More details +👉 [Full proxy quickstart with Docker →](./proxy/docker_quick_start) -- [exception mapping](./exception_mapping.md) -- [retries + model fallbacks for completion()](./completion/reliable_completions.md) -- [proxy virtual keys & spend management](./proxy/virtual_keys.md) -- [E2E Tutorial for LiteLLM Proxy Server](./proxy/docker_quick_start.md) +:::tip Debugging tool +Use [**`/utils/transform_request`**](./utils/transform_request) to inspect exactly what LiteLLM sends to any provider — useful for debugging prompt formatting, header issues, and provider-specific parameters. +::: + +🔗 [Interactive API explorer (Swagger) →](https://litellm-api.up.railway.app/) + +--- + +## Agent & MCP Gateway + +LiteLLM is a unified gateway for **LLMs, agents, and MCP** — you don't need a separate agent or MCP gateway. One endpoint for 100+ models, A2A agents, and MCP tools. + + + +--- + +## What to Explore Next + + diff --git a/docs/my-website/docusaurus.config.js b/docs/my-website/docusaurus.config.js index 32d5d800b7..1b46ed107d 100644 --- a/docs/my-website/docusaurus.config.js +++ b/docs/my-website/docusaurus.config.js @@ -2,9 +2,9 @@ // Note: type annotations allow type checking and IDEs autocompletion // @ts-ignore -const lightCodeTheme = require('prism-react-renderer/themes/github'); +const lightCodeTheme = require('prism-react-renderer/themes/vsLight'); // @ts-ignore -const darkCodeTheme = require('prism-react-renderer/themes/dracula'); +const darkCodeTheme = require('prism-react-renderer/themes/nightOwl'); const inkeepConfig = { baseSettings: { @@ -87,18 +87,83 @@ const config = { }, ], [ - '@docusaurus/plugin-content-blog', + '@docusaurus/plugin-content-docs', { - id: 'release_notes', + id: 'release-notes', path: './release_notes', routeBasePath: 'release_notes', - blogTitle: 'Release Notes', - blogSidebarTitle: 'Releases', - blogSidebarCount: 'ALL', - postsPerPage: 'ALL', - showReadingTime: false, - sortPosts: 'descending', - include: ['**/*.{md,mdx}'], + sidebarPath: require.resolve('./sidebars-release-notes.js'), + async sidebarItemsGenerator({defaultSidebarItemsGenerator, docs, ...args}) { + const items = await defaultSidebarItemsGenerator({docs, ...args}); + + // Build map of doc id -> year from frontmatter date + const docYearMap = {}; + for (const doc of docs) { + const date = doc.frontMatter && doc.frontMatter.date; + if (date) { + const year = new Date(date).getFullYear(); + docYearMap[doc.id] = year; + } + } + + function parseVersion(str) { + const match = (str || '').match(/v?(\d+)\.(\d+)\.(\d+)/); + if (!match) return [0, 0, 0]; + return [parseInt(match[1]), parseInt(match[2]), parseInt(match[3])]; + } + function compareVersionsDesc(a, b) { + const [aMaj, aMin, aPatch] = parseVersion(a.label || a.id || ''); + const [bMaj, bMin, bPatch] = parseVersion(b.label || b.id || ''); + if (bMaj !== aMaj) return bMaj - aMaj; + if (bMin !== aMin) return bMin - aMin; + return bPatch - aPatch; + } + + // Flatten and transform doc items (filter index, shorten labels) + function flattenDocs(list) { + const result = []; + for (const item of list) { + if (item.type === 'doc' && item.id === 'index') continue; + if (item.type === 'doc') { + const label = item.id.replace(/\/index$/, ''); + result.push({...item, label}); + } else if (item.type === 'category') { + if (item.link && item.link.type === 'doc' && item.link.id !== 'index') { + const id = item.link.id; + const label = id.replace(/\/index$/, ''); + result.push({type: 'doc', id, label}); + } else { + result.push(...flattenDocs(item.items)); + } + } + } + return result; + } + + const docItems = flattenDocs(items); + + // Group by year + const byYear = {}; + for (const item of docItems) { + const year = docYearMap[item.id] || 'Other'; + if (!byYear[year]) byYear[year] = []; + byYear[year].push(item); + } + + // Sort each year's items by version descending + for (const year of Object.keys(byYear)) { + byYear[year].sort(compareVersionsDesc); + } + + // Build categories sorted by year descending + const years = Object.keys(byYear).sort((a, b) => b - a); + return years.map(year => ({ + type: 'category', + label: String(year), + collapsed: year !== String(years[0]), + items: byYear[year], + })); + }, }, ], [ @@ -181,33 +246,29 @@ const config = { label: 'Docs', }, { + type: 'docSidebar', sidebarId: 'integrationsSidebar', position: 'left', label: 'Integrations', - to: "docs/integrations" }, { - sidebarId: 'tutorialSidebar', position: 'left', label: 'Enterprise', to: "docs/enterprise" }, { to: '/release_notes', label: 'Release Notes', position: 'left' }, { to: '/blog', label: 'Blog', position: 'left' }, - { - href: 'https://models.litellm.ai/', - label: '💸 LLM Model Cost Map', - position: 'right', - }, { href: 'https://github.com/BerriAI/litellm', - label: 'GitHub', position: 'right', + className: 'header-github-link', + 'aria-label': 'GitHub repository', }, { href: 'https://www.litellm.ai/support', - label: 'Slack/Discord', position: 'right', + className: 'header-discord-link', + 'aria-label': 'Discord / Slack community', } ], }, diff --git a/docs/my-website/img/hero.png b/docs/my-website/img/hero.png new file mode 100644 index 0000000000..9f77a28d71 Binary files /dev/null and b/docs/my-website/img/hero.png differ diff --git a/docs/my-website/release_notes/index.md b/docs/my-website/release_notes/index.md new file mode 100644 index 0000000000..44d6d39ba0 --- /dev/null +++ b/docs/my-website/release_notes/index.md @@ -0,0 +1,51 @@ +--- +title: Release Notes +sidebar_label: Overview +slug: / +--- + +# Release Notes + +LiteLLM ships new releases regularly with new provider support, performance improvements, and enterprise features. Use the sidebar to browse all releases. + +## Latest Release + +### [v1.82.0 — Realtime Guardrails, Projects Management, and 10+ Performance Optimizations](/release_notes/v1-82-0) + +_February 28, 2026_ + +Real-time guardrail enforcement, a new Projects Management UI, and over 10 backend performance optimizations. + +--- + +## Recent Releases + +| Version | Date | Highlights | +| ----------------------------------- | ------------ | ---------------------------------------------------------- | +| [v1.81.14](/release_notes/v1-81-14) | Feb 21, 2026 | New Gateway Level Guardrails & Compliance Playground | +| [v1.81.12](/release_notes/v1-81-12) | Feb 14, 2026 | Guardrail Policy Templates & Action Builder | +| [v1.81.9](/release_notes/v1-81-9) | Feb 7, 2026 | Control which MCP Servers are exposed on the Internet | +| [v1.81.6](/release_notes/v1-81-6) | Jan 31, 2026 | Logs v2 with Tool Call Tracing | +| [v1.81.3](/release_notes/v1-81-3) | Jan 26, 2026 | Performance — 25% CPU Usage Reduction | +| [v1.81.0](/release_notes/v1-81-0) | Jan 18, 2026 | Claude Code — Web Search Across All Providers | +| [v1.80.15](/release_notes/v1-80-15) | Jan 10, 2026 | Manus API Support | +| [v1.80.8](/release_notes/v1-80-8) | Dec 6, 2025 | Introducing A2A Agent Gateway | +| [v1.80.5](/release_notes/v1-80-5) | Nov 22, 2025 | Gemini 3.0 Support | +| [v1.80.0](/release_notes/v1-80-0) | Nov 15, 2025 | Introducing Agent Hub: Register, Publish, and Share Agents | +| [v1.79.3](/release_notes/v1-79-3) | Nov 8, 2025 | Built-in Guardrails on AI Gateway | +| [v1.79.0](/release_notes/v1-79-0) | Oct 26, 2025 | Search APIs | +| [v1.78.5](/release_notes/v1-78-5) | Oct 18, 2025 | Native OCR Support | +| [v1.78.0](/release_notes/v1-78-0) | Oct 11, 2025 | MCP Gateway: Control Tool Access by Team, Key | +| [v1.77.7](/release_notes/v1-77-7) | Oct 4, 2025 | 2.9x Lower Median Latency | +| [v1.77.5](/release_notes/v1-77-5) | Sep 29, 2025 | MCP OAuth 2.0 Support | +| [v1.77.3](/release_notes/v1-77-3) | Sep 21, 2025 | Priority Based Rate Limiting | + +--- + +## Stay Updated + +- **GitHub**: Watch the [BerriAI/litellm](https://github.com/BerriAI/litellm) repository for release notifications +- **Discord**: Join our [community](https://discord.com/invite/wuPM9dRgDw) for announcements +- **Twitter**: Follow [@LiteLLM](https://twitter.com/LiteLLM) + +Use the sidebar to browse the full release history. diff --git a/docs/my-website/sidebars-release-notes.js b/docs/my-website/sidebars-release-notes.js new file mode 100644 index 0000000000..6ed29003ce --- /dev/null +++ b/docs/my-website/sidebars-release-notes.js @@ -0,0 +1,14 @@ +// @ts-check + +/** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */ +const sidebars = { + releaseNotesSidebar: [ + { type: 'doc', id: 'index', label: 'Release Notes' }, + { + type: 'autogenerated', + dirName: '.', + }, + ], +}; + +module.exports = sidebars; diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 1362745a91..93f692d88f 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -212,7 +212,20 @@ const sidebars = { ], // But you can create a sidebar manually tutorialSidebar: [ - { type: "doc", id: "index", label: "Getting Started" }, + // ════════════════════════════════════════════════════════════ + // GET STARTED + // ════════════════════════════════════════════════════════════ + { + type: "category", + label: "Get Started", + collapsible: false, + collapsed: false, + items: [ + { type: "doc", id: "index", label: "Quickstart" }, + { type: "link", label: "Models & Pricing", href: "https://models.litellm.ai" }, + { type: "link", label: "Changelog", href: "/release_notes" }, + ], + }, { type: "category", @@ -1181,22 +1194,6 @@ const sidebars = { "troubleshoot", ], }, - { - type: "category", - label: "Blog", - items: [ - { - type: "link", - label: "Day 0 Support: Claude Sonnet 4.6", - href: "/blog/claude_sonnet_4_6", - }, - { - type: "link", - label: "Incident: Broken Model Cost Map", - href: "/blog/model-cost-map-incident", - }, - ], - }, ], }; diff --git a/docs/my-website/src/components/NavigationCards/index.js b/docs/my-website/src/components/NavigationCards/index.js new file mode 100644 index 0000000000..1d5d3e577a --- /dev/null +++ b/docs/my-website/src/components/NavigationCards/index.js @@ -0,0 +1,43 @@ +import React from 'react'; +import styles from './styles.module.css'; + +export default function NavigationCards({ items, columns = 2 }) { + return ( +
+ {items.map((item, i) => { + const isExternal = + item.to && (item.to.startsWith('http://') || item.to.startsWith('https://')); + return ( + + {item.icon && ( +
{item.icon}
+ )} +
{item.title}
+ {item.description && ( +
{item.description}
+ )} + {item.listDescription && ( +
    + {item.listDescription.map((line, j) => ( +
  • {line}
    • + ))} +
+ )} + {isExternal && ( + ↗ + )} + + ); + })} +
+ ); +} diff --git a/docs/my-website/src/components/NavigationCards/styles.module.css b/docs/my-website/src/components/NavigationCards/styles.module.css new file mode 100644 index 0000000000..64f5a42374 --- /dev/null +++ b/docs/my-website/src/components/NavigationCards/styles.module.css @@ -0,0 +1,82 @@ +.grid { + display: grid; + grid-template-columns: repeat(var(--nav-columns, 2), 1fr); + gap: 0.75rem; + margin: 1.25rem 0; +} + +@media (max-width: 768px) { + .grid { + grid-template-columns: 1fr; + } +} + +.card { + position: relative; + display: flex; + flex-direction: column; + padding: 1rem 1.1rem; + border: 1px solid var(--ifm-color-emphasis-200); + border-radius: 6px; + text-decoration: none !important; + color: inherit !important; + background: var(--ifm-background-surface-color); + transition: border-color 0.15s ease, box-shadow 0.15s ease; +} + +.card:hover { + border-color: var(--ifm-color-primary); + box-shadow: 0 0 0 1px var(--ifm-color-primary); + text-decoration: none !important; +} + +[data-theme='dark'] .card { + background: var(--ifm-background-surface-color); + border-color: #2d3748; +} + +[data-theme='dark'] .card:hover { + border-color: var(--ifm-color-primary); + box-shadow: 0 0 0 1px var(--ifm-color-primary); +} + +.icon { + font-size: 1.4rem; + margin-bottom: 0.5rem; + line-height: 1; +} + +.title { + font-size: 14px; + font-weight: 600; + margin-bottom: 0.35rem; + color: var(--ifm-heading-color); +} + +.description { + font-size: 13px; + line-height: 1.5; + color: var(--ifm-color-emphasis-700); + margin-bottom: 0.5rem; +} + +.list { + margin: 0.35rem 0 0 0; + padding-left: 1.1rem; + list-style: disc; +} + +.list li { + font-size: 12.5px; + color: var(--ifm-color-emphasis-700); + line-height: 1.6; + margin-bottom: 0; +} + +.externalIcon { + position: absolute; + top: 0.75rem; + right: 0.75rem; + font-size: 12px; + color: var(--ifm-color-emphasis-500); +} diff --git a/docs/my-website/src/css/custom.css b/docs/my-website/src/css/custom.css index 9fa4443afc..d0702fcc57 100644 --- a/docs/my-website/src/css/custom.css +++ b/docs/my-website/src/css/custom.css @@ -1,10 +1,16 @@ /** - * Any CSS included here will be global. The classic template - * bundles Infima by default. Infima is a CSS framework designed to - * work well for content-centric websites. + * Global CSS overrides for LiteLLM docs. + * Infima (Docusaurus CSS framework) variables + custom styling. */ -/* You can override the default Infima variables here. */ +/* ========================================= + FONTS + ========================================= */ +@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&display=swap'); + +/* ========================================= + ROOT — Light Mode Variables + ========================================= */ :root { --ifm-color-primary: #2e8555; --ifm-color-primary-dark: #29784c; @@ -13,11 +19,22 @@ --ifm-color-primary-light: #33925d; --ifm-color-primary-lighter: #359962; --ifm-color-primary-lightest: #3cad6e; - --ifm-code-font-size: 95%; - --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.1); + --ifm-code-font-size: 85%; + --ifm-menu-color: #6b7280; + --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.08); + --ifm-font-family-base: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + --ifm-heading-font-weight: 600; + --ifm-font-size-base: 15px; + --ifm-line-height-base: 1.65; + --ifm-border-radius: 6px; + /* Wider reading column — reduces excessive whitespace on large monitors */ + --ifm-container-width: 1380px; + --ifm-container-width-xl: 1560px; } -/* For readability concerns, you should choose a lighter palette in dark mode. */ +/* ========================================= + DARK MODE Variables + ========================================= */ [data-theme='dark'] { --ifm-color-primary: #25c2a0; --ifm-color-primary-dark: #21af90; @@ -26,10 +43,710 @@ --ifm-color-primary-light: #29d5b0; --ifm-color-primary-lighter: #32d8b4; --ifm-color-primary-lightest: #4fddbf; - --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.3); + --ifm-background-color: #0d1117; + --ifm-background-surface-color: #161b22; + --docusaurus-highlighted-code-line-bg: rgba(255, 255, 255, 0.07); } -/* Levo logo sizing and theme switching */ +/* ========================================= + TYPOGRAPHY + ========================================= */ +.theme-doc-markdown h1 { + font-size: 2.2rem; + letter-spacing: -0.02em; + line-height: 1.2; +} + +.theme-doc-markdown h2 { + font-size: 1.6rem; + letter-spacing: -0.01em; + line-height: 1.3; +} + +.theme-doc-markdown h3 { + font-size: 1.25rem; + line-height: 1.4; +} + +.theme-doc-markdown p, +.theme-doc-markdown ul, +.theme-doc-markdown ol { + font-size: 0.9rem; +} + +.theme-doc-markdown table { + font-size: 0.875rem; +} + +.theme-doc-markdown td { + font-size: 0.85rem; +} + +/* ========================================= + NAVBAR + ========================================= */ +[data-theme='light'] .navbar { + background-color: #ffffff; + box-shadow: 0 1px 0 0 #e5e7eb; +} + +[data-theme='dark'] .navbar { + background-color: var(--ifm-background-color); + border-bottom: 1px solid #21262d; + box-shadow: none; +} + +.navbar__link { + font-weight: 400 !important; + font-size: 14px !important; + border-bottom: 2px solid transparent !important; + padding-bottom: 2px; +} + +.navbar__link--active { + font-weight: 500 !important; + border-bottom: 2px solid var(--ifm-color-primary) !important; +} + +@media (max-width: 1330px) { + .navbar__link { + font-size: 13px !important; + } +} + +/* Three-column navbar: logo | center nav | right icons */ +@media (min-width: 997px) { + .navbar__inner { + display: flex !important; + align-items: center; + justify-content: space-between; + } + + .navbar__brand-col { + display: flex; + align-items: center; + flex: 0 0 auto; + margin-left: 1rem; + } + + .navbar__brand-col .navbar__brand { + font-size: 1.25rem; + } + + .navbar__brand-col .navbar__logo { + height: 2rem; + width: auto; + } + + .navbar__center-col { + display: flex; + align-items: center; + justify-content: center; + flex: 1; + } + + .navbar__right-col { + display: flex; + align-items: center; + flex: 0 0 auto; + gap: 0.25rem; + margin-right: 2rem; + } +} + +/* ========================================= + ALERTS / ADMONITIONS + ========================================= */ +.alert { + padding: 0.75rem 1rem; + font-size: 14px; + border-radius: var(--ifm-border-radius); + border-left-width: 3px; +} + +/* Light mode */ +.alert--info { + --ifm-alert-background-color: #f0f7ff !important; + --ifm-alert-border-color: #2264ab !important; +} + +.alert--success { + --ifm-alert-background-color: #f0fff8 !important; + --ifm-alert-border-color: #09bda8 !important; +} + +.alert--secondary { + --ifm-alert-background-color: #f8fafc !important; + --ifm-alert-border-color: #64748b !important; +} + +.alert--danger { + --ifm-alert-background-color: #fff0f5 !important; + --ifm-alert-border-color: #e11d48 !important; +} + +.alert--warning { + --ifm-alert-background-color: #fffbeb !important; + --ifm-alert-border-color: #d97706 !important; +} + +/* Dark mode */ +[data-theme='dark'] .alert--info { + --ifm-alert-background-color: #0c1e30 !important; + --ifm-alert-border-color: #3b82f6 !important; + color: #bfdbfe !important; +} + +[data-theme='dark'] .alert--success { + --ifm-alert-background-color: #022c22 !important; + --ifm-alert-border-color: #10b981 !important; +} + +[data-theme='dark'] .alert--secondary { + --ifm-alert-background-color: #0f172a !important; + --ifm-alert-border-color: #475569 !important; +} + +[data-theme='dark'] .alert--danger { + --ifm-alert-background-color: #2d0a14 !important; + --ifm-alert-border-color: #f43f5e !important; +} + +[data-theme='dark'] .alert--warning { + --ifm-alert-background-color: #1c1200 !important; + --ifm-alert-border-color: #f59e0b !important; +} + +/* ========================================= + COLLAPSIBLE / DETAILS + ========================================= */ +details { + color: #1d232e; + background-color: #ffffff; + border: 1px solid #e9eef2 !important; + border-radius: var(--ifm-border-radius) !important; + padding: 0.75rem !important; + margin-bottom: 1rem !important; + margin-top: 1.5rem !important; + box-shadow: 0 1px 4px 0 rgba(0, 0, 0, 0.05) !important; +} + +details summary { + font-weight: 600 !important; +} + +details [class*='collapsibleContent'] { + border-top: 1px solid #e2e8f0 !important; +} + +details [class*='collapsibleContent'] p, +details [class*='collapsibleContent'] ul { + font-size: 13px !important; + line-height: 1.75; +} + +[data-theme='dark'] details { + background-color: #1c2130 !important; + color: #e5e7eb !important; + border: 1px solid #2d3748 !important; + box-shadow: 0 1px 6px 0 rgba(0, 0, 0, 0.3) !important; +} + +[data-theme='dark'] details summary { + color: #f3f4f6 !important; +} + +[data-theme='dark'] details [class*='collapsibleContent'] { + border-top: 1px solid #2d3748 !important; +} + +/* ========================================= + TABS + ========================================= */ +.tabs-container > div { + padding: 1rem; + border: 1px solid #e2e8f0; + border-radius: var(--ifm-border-radius); +} + +/* Remove styling from nested tabs containers */ +.tabs-container .tabs-container > div { + padding: 0; + border: none; + border-radius: 0; +} + +[data-theme='dark'] .tabs-container > div { + border-color: #2d3748; +} + +ul.tabs { + border-bottom: 1px solid #e2e8f0 !important; + column-gap: 0.5rem !important; +} + +[data-theme='dark'] ul.tabs { + border-bottom-color: #2d3748 !important; +} + +li.tabs__item { + padding: 0.5rem !important; + font-weight: 500 !important; + font-size: 14px !important; + border-bottom: 2px solid transparent !important; +} + +li.tabs__item--active { + border-bottom: 2px solid var(--ifm-color-primary) !important; +} + +/* ========================================= + CODE BLOCKS + ========================================= */ +.prism-code { + border-radius: var(--ifm-border-radius); + font-size: 12.5px !important; + line-height: 1.6; +} + +[data-theme='dark'] .prism-code { + border: 1px solid #21262d; +} + +[class*='codeLineNumber']::before { + font-size: 12px !important; + display: flex !important; + align-items: center !important; + justify-content: center !important; +} + +[class*='codeBlockTitle'] { + font-size: 13px !important; + font-weight: 500 !important; + padding: 0.5rem 1rem !important; + border-bottom: 1px solid #334155 !important; +} + +.theme-code-block-highlighted-line { + background-color: rgba(0, 0, 0, 0.1) !important; +} + +[data-theme='dark'] .theme-code-block-highlighted-line { + background-color: rgba(255, 255, 255, 0.06) !important; +} + +.theme-code-block-highlighted-line > span { + background-color: transparent !important; +} + +/* ========================================= + SIDEBAR / MENU + ========================================= */ +.menu { + font-weight: 400; + padding: 0.5rem 0.25rem !important; + background-image: radial-gradient(rgba(0, 0, 0, 0.07) 1px, transparent 1px); + background-size: 24px 24px; +} + +[data-theme='dark'] .menu { + background-image: radial-gradient(rgba(255, 255, 255, 0.04) 1px, transparent 1px); + background-size: 24px 24px; +} + +.menu__link { + font-size: 14px; + padding: 0.22rem 0.75rem !important; + border-radius: 4px; +} + +.menu__link--active { + font-weight: 600 !important; + background-color: rgba(46, 133, 85, 0.08) !important; +} + +[data-theme='dark'] .menu__link--active { + background-color: rgba(37, 194, 160, 0.1) !important; +} + +/* ─── Sidebar collapse arrows — uniform size & alignment ────── */ + +/* 1. Categories WITHOUT a link prop: + The button itself holds the text + ::after arrow. + Make it flex so the arrow never wraps to a new line. */ +.menu__link--sublist-caret { + display: flex !important; + align-items: center !important; + justify-content: space-between !important; + gap: 0.5rem; + padding-right: 0.625rem !important; +} + +.menu__link--sublist-caret::after { + content: '' !important; + display: block !important; + flex-shrink: 0 !important; + width: 1.25rem !important; + height: 1.25rem !important; + min-width: 1.25rem !important; + background: var(--ifm-menu-link-sublist-icon) center / 1.25rem 1.25rem no-repeat !important; + margin: 0 !important; +} + +/* 2. Categories WITH a link prop: + A separate +