diff --git a/docs/my-website/docs/providers/crusoe.md b/docs/my-website/docs/providers/crusoe.md new file mode 100644 index 0000000000..696da3fa1b --- /dev/null +++ b/docs/my-website/docs/providers/crusoe.md @@ -0,0 +1,186 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Crusoe + +## Overview + +| Property | Details | +|-------|-------| +| Description | Crusoe Cloud provides GPU-accelerated inference for open-source large language models, optimized for performance and cost efficiency. | +| Provider Route on LiteLLM | `crusoe/` | +| Link to Provider Doc | [Crusoe Managed Inference Documentation ↗](https://docs.crusoecloud.com/managed-inference/overview/index.html) | +| Base URL | `https://managed-inference-api-proxy.crusoecloud.com/v1/` | +| Supported Operations | [`/chat/completions`](#sample-usage) | + +
+
+ +**We support ALL Crusoe models, just set `crusoe/` as a prefix when sending completion requests** + +## Available Models + +| Model | Description | Context Window | +|-------|-------------|----------------| +| `crusoe/deepseek-ai/DeepSeek-R1-0528` | DeepSeek R1 reasoning model (May 2025) | 163,840 tokens | +| `crusoe/deepseek-ai/DeepSeek-V3-0324` | DeepSeek V3 chat model (March 2025) | 163,840 tokens | +| `crusoe/google/gemma-3-12b-it` | Google Gemma 3 12B instruction-tuned | 131,072 tokens | +| `crusoe/meta-llama/Llama-3.3-70B-Instruct` | Llama 3.3 70B instruction-tuned | 131,072 tokens | +| `crusoe/moonshotai/Kimi-K2-Thinking` | Kimi K2 extended thinking model | 262,144 tokens | +| `crusoe/openai/gpt-oss-120b` | OpenAI 120B open-source model | 131,072 tokens | +| `crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507` | Qwen3 235B MoE instruction-tuned | 262,144 tokens | + +## Required Variables + +```python showLineNumbers title="Environment Variables" +os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key +``` + +## Usage - LiteLLM Python SDK + +### Non-streaming + +```python showLineNumbers title="Crusoe Non-streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key + +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# Crusoe call +response = completion( + model="crusoe/meta-llama/Llama-3.3-70B-Instruct", + messages=messages +) + +print(response) +``` + +### Streaming + +```python showLineNumbers title="Crusoe Streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key + +messages = [{"content": "Write a short story about AI", "role": "user"}] + +# Crusoe call with streaming +response = completion( + model="crusoe/meta-llama/Llama-3.1-70B-Instruct", + messages=messages, + stream=True +) + +for chunk in response: + print(chunk) +``` + +### Function Calling + +```python showLineNumbers title="Crusoe Function Calling" +import os +import litellm +from litellm import completion + +os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key + +tools = [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + } + }, + "required": ["location"] + } + } +}] + +messages = [{"role": "user", "content": "What's the weather in Boston?"}] + +response = completion( + model="crusoe/meta-llama/Llama-3.3-70B-Instruct", + messages=messages, + tools=tools, + tool_choice="auto" +) + +print(response) +``` + +## Usage - LiteLLM Proxy Server + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: llama-3.3-70b + litellm_params: + model: crusoe/meta-llama/Llama-3.3-70B-Instruct + api_key: os.environ/CRUSOE_API_KEY + - model_name: deepseek-r1 + litellm_params: + model: crusoe/deepseek-ai/DeepSeek-R1-0528 + api_key: os.environ/CRUSOE_API_KEY + - model_name: deepseek-v3 + litellm_params: + model: crusoe/deepseek-ai/DeepSeek-V3-0324 + api_key: os.environ/CRUSOE_API_KEY + - model_name: qwen3-235b + litellm_params: + model: crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507 + api_key: os.environ/CRUSOE_API_KEY + - model_name: kimi-k2 + litellm_params: + model: crusoe/moonshotai/Kimi-K2-Thinking + api_key: os.environ/CRUSOE_API_KEY +``` + +## Custom API Base + +```python showLineNumbers title="Custom API Base" +import os +import litellm +from litellm import completion + +# Using environment variable +os.environ["CRUSOE_API_BASE"] = "https://custom.crusoecloud.com/v1/" +os.environ["CRUSOE_API_KEY"] = "" # your API key + +# Or pass directly +response = completion( + model="crusoe/meta-llama/Llama-3.3-70B-Instruct", + messages=[{"content": "Hello!", "role": "user"}], + api_base="https://custom.crusoecloud.com/v1/", + api_key="your-api-key" +) +``` + +## Supported OpenAI Parameters + +- `temperature` +- `max_tokens` +- `max_completion_tokens` +- `top_p` +- `frequency_penalty` +- `presence_penalty` +- `stop` +- `n` +- `stream` +- `tools` +- `tool_choice` +- `response_format` +- `seed` +- `user` +- `logit_bias` +- `logprobs` +- `top_logprobs` diff --git a/litellm/__init__.py b/litellm/__init__.py index 77fa48625d..014a8d75f1 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -600,6 +600,7 @@ publicai_models: Set = set() v0_models: Set = set() morph_models: Set = set() lambda_ai_models: Set = set() +crusoe_models: Set = set() hyperbolic_models: Set = set() black_forest_labs_models: Set = set() recraft_models: Set = set() @@ -847,6 +848,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None): morph_models.add(key) elif value.get("litellm_provider") == "lambda_ai": lambda_ai_models.add(key) + elif value.get("litellm_provider") == "crusoe": + crusoe_models.add(key) elif value.get("litellm_provider") == "hyperbolic": hyperbolic_models.add(key) elif value.get("litellm_provider") == "black_forest_labs": @@ -1082,6 +1085,7 @@ models_by_provider: dict = { "v0": v0_models, "morph": morph_models, "lambda_ai": lambda_ai_models, + "crusoe": crusoe_models, "hyperbolic": hyperbolic_models, "black_forest_labs": black_forest_labs_models, "recraft": recraft_models, diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 119e62a5b3..c0a8666eb7 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -1140,6 +1140,7 @@ _LLM_CONFIGS_IMPORT_MAP = { "MorphChatConfig": (".llms.morph.chat.transformation", "MorphChatConfig"), "RAGFlowConfig": (".llms.ragflow.chat.transformation", "RAGFlowConfig"), "LambdaAIChatConfig": (".llms.lambda_ai.chat.transformation", "LambdaAIChatConfig"), + "CrusoeChatConfig": (".llms.crusoe.chat.transformation", "CrusoeChatConfig"), "HyperbolicChatConfig": ( ".llms.hyperbolic.chat.transformation", "HyperbolicChatConfig", diff --git a/litellm/constants.py b/litellm/constants.py index 6c889a317b..5e75c0feab 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -610,6 +610,7 @@ LITELLM_CHAT_PROVIDERS = [ "oci", "morph", "lambda_ai", + "crusoe", "vercel_ai_gateway", "wandb", "ovhcloud", @@ -768,6 +769,7 @@ openai_compatible_endpoints: List = [ "https://api.v0.dev/v1", "https://api.morphllm.com/v1", "https://api.lambda.ai/v1", + "https://managed-inference-api-proxy.crusoecloud.com/v1/", "https://api.hyperbolic.xyz/v1", "https://ai-gateway.helicone.ai/", "https://ai-gateway.vercel.sh/v1", @@ -823,6 +825,7 @@ openai_compatible_providers: List = [ "helicone", "morph", "lambda_ai", + "crusoe", "hyperbolic", "vercel_ai_gateway", "aiml", @@ -851,6 +854,7 @@ openai_text_completion_compatible_providers: List = ( "chutes", "v0", "lambda_ai", + "crusoe", "hyperbolic", "wandb", ] diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index c0ca6835ee..54450bccba 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -353,6 +353,9 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "https://api.lambda.ai/v1": custom_llm_provider = "lambda_ai" dynamic_api_key = get_secret_str("LAMBDA_API_KEY") + elif endpoint == "https://managed-inference-api-proxy.crusoecloud.com/v1/": + custom_llm_provider = "crusoe" + dynamic_api_key = get_secret_str("CRUSOE_API_KEY") elif endpoint == "https://api.hyperbolic.xyz/v1": custom_llm_provider = "hyperbolic" dynamic_api_key = get_secret_str("HYPERBOLIC_API_KEY") @@ -919,6 +922,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.LambdaAIChatConfig()._get_openai_compatible_provider_info( api_base, api_key ) + elif custom_llm_provider == "crusoe": + ( + api_base, + dynamic_api_key, + ) = litellm.CrusoeChatConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) elif custom_llm_provider == "hyperbolic": ( api_base, diff --git a/litellm/llms/crusoe/__init__.py b/litellm/llms/crusoe/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/litellm/llms/crusoe/chat/__init__.py b/litellm/llms/crusoe/chat/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/litellm/llms/crusoe/chat/transformation.py b/litellm/llms/crusoe/chat/transformation.py new file mode 100644 index 0000000000..dab9a9f833 --- /dev/null +++ b/litellm/llms/crusoe/chat/transformation.py @@ -0,0 +1,42 @@ +""" +Translate from OpenAI's `/v1/chat/completions` to Crusoe's `/v1/chat/completions` +""" + +from typing import Optional, Tuple + +from litellm.secret_managers.main import get_secret_str + +from ...openai_like.chat.transformation import OpenAILikeChatConfig + + +class CrusoeChatConfig(OpenAILikeChatConfig): + """ + Crusoe is OpenAI-compatible with standard endpoints. + + Docs: https://docs.crusoecloud.com/managed-inference/overview/index.html + """ + + @property + def custom_llm_provider(self) -> Optional[str]: + return "crusoe" + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + api_base = ( + api_base + or get_secret_str("CRUSOE_API_BASE") + or "https://managed-inference-api-proxy.crusoecloud.com/v1/" + ) # type: ignore + dynamic_api_key = api_key or get_secret_str("CRUSOE_API_KEY") + return api_base, dynamic_api_key + + def get_supported_openai_params(self, model: str) -> list: + return [ + "messages", + "model", + "temperature", + "top_p", + "frequency_penalty", + "presence_penalty", + ] diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index bbe13442d6..8a5a455e6b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -22109,6 +22109,95 @@ "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, + "crusoe/deepseek-ai/DeepSeek-R1-0528": { + "input_cost_per_token": 3e-06, + "litellm_provider": "crusoe", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 7e-06, + "supports_function_calling": false, + "supports_system_messages": true, + "supports_tool_choice": false + }, + "crusoe/deepseek-ai/DeepSeek-V3-0324": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "crusoe", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "crusoe/google/gemma-3-12b-it": { + "input_cost_per_token": 1e-07, + "litellm_provider": "crusoe", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1e-07, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "crusoe/meta-llama/Llama-3.3-70B-Instruct": { + "input_cost_per_token": 2e-07, + "litellm_provider": "crusoe", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2e-07, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "crusoe/moonshotai/Kimi-K2-Thinking": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "crusoe", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "supports_function_calling": false, + "supports_system_messages": true, + "supports_tool_choice": false + }, + "crusoe/openai/gpt-oss-120b": { + "input_cost_per_token": 8e-07, + "litellm_provider": "crusoe", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 8e-07, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507": { + "input_cost_per_token": 3e-06, + "litellm_provider": "crusoe", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "lambda_ai/deepseek-llama3.3-70b": { "input_cost_per_token": 2e-07, "litellm_provider": "lambda_ai", diff --git a/tests/llm_translation/test_crusoe.py b/tests/llm_translation/test_crusoe.py new file mode 100644 index 0000000000..254d6530a4 --- /dev/null +++ b/tests/llm_translation/test_crusoe.py @@ -0,0 +1,164 @@ +""" +Tests for Crusoe provider integration +""" +import os +from unittest import mock + +import pytest + +import litellm +from litellm import completion +from litellm.llms.crusoe.chat.transformation import CrusoeChatConfig + +CRUSOE_API_BASE = "https://managed-inference-api-proxy.crusoecloud.com/v1/" + + +def test_crusoe_config_initialization(): + """Test CrusoeChatConfig initializes correctly""" + config = CrusoeChatConfig() + assert config.custom_llm_provider == "crusoe" + + +def test_crusoe_get_openai_compatible_provider_info(): + """Test Crusoe provider info retrieval""" + config = CrusoeChatConfig() + + # Test with default values (no env vars set) + with mock.patch.dict(os.environ, {}, clear=True): + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == CRUSOE_API_BASE + assert api_key is None + + # Test with environment variables + with mock.patch.dict( + os.environ, + { + "CRUSOE_API_KEY": "test-key", + "CRUSOE_API_BASE": "https://custom.crusoecloud.com/v1/", + }, + ): + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == "https://custom.crusoecloud.com/v1/" + assert api_key == "test-key" + + # Test with explicit parameters (should override env vars) + with mock.patch.dict( + os.environ, + { + "CRUSOE_API_KEY": "env-key", + "CRUSOE_API_BASE": "https://env.crusoecloud.com/v1/", + }, + ): + api_base, api_key = config._get_openai_compatible_provider_info( + "https://param.crusoecloud.com/v1/", "param-key" + ) + assert api_base == "https://param.crusoecloud.com/v1/" + assert api_key == "param-key" + + +def test_get_llm_provider_crusoe(): + """Test that get_llm_provider correctly identifies Crusoe""" + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + # Test with crusoe/model-name format + model, provider, api_key, api_base = get_llm_provider( + "crusoe/meta-llama/Llama-3.3-70B-Instruct" + ) + assert model == "meta-llama/Llama-3.3-70B-Instruct" + assert provider == "crusoe" + + # Test with api_base containing Crusoe endpoint + model, provider, api_key, api_base = get_llm_provider( + "meta-llama/Llama-3.3-70B-Instruct", + api_base=CRUSOE_API_BASE, + ) + assert model == "meta-llama/Llama-3.3-70B-Instruct" + assert provider == "crusoe" + assert api_base == CRUSOE_API_BASE + + +def test_crusoe_in_provider_lists(): + """Test that Crusoe is registered in all necessary provider lists""" + assert "crusoe" in litellm.openai_compatible_providers + assert "crusoe" in litellm.provider_list + assert CRUSOE_API_BASE in litellm.openai_compatible_endpoints + + +def test_crusoe_models_configuration(): + """Test that Crusoe models are configured correctly""" + from litellm import get_model_info + + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + litellm.crusoe_models = set() + litellm.add_known_models() + + crusoe_models = [ + "crusoe/meta-llama/Llama-3.3-70B-Instruct", + "crusoe/deepseek-ai/DeepSeek-R1-0528", + "crusoe/deepseek-ai/DeepSeek-V3-0324", + "crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507", + "crusoe/moonshotai/Kimi-K2-Thinking", + "crusoe/openai/gpt-oss-120b", + "crusoe/google/gemma-3-12b-it", + ] + + for model in crusoe_models: + model_info = get_model_info(model) + assert model_info is not None, f"Model info not found for {model}" + assert model_info.get("litellm_provider") == "crusoe", ( + f"{model} should have crusoe as provider" + ) + assert model_info.get("mode") == "chat", f"{model} should be in chat mode" + + +def test_crusoe_model_list_populated(): + """Test that crusoe_models list is populated correctly""" + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + litellm.crusoe_models = set() + litellm.add_known_models() + + assert len(litellm.crusoe_models) > 0, "crusoe_models list should not be empty" + + for model in litellm.crusoe_models: + assert model.startswith("crusoe/"), ( + f"Model {model} should start with 'crusoe/'" + ) + + expected_models = [ + "crusoe/meta-llama/Llama-3.3-70B-Instruct", + "crusoe/deepseek-ai/DeepSeek-R1-0528", + "crusoe/deepseek-ai/DeepSeek-V3-0324", + "crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507", + "crusoe/moonshotai/Kimi-K2-Thinking", + "crusoe/openai/gpt-oss-120b", + "crusoe/google/gemma-3-12b-it", + ] + + for model in expected_models: + assert model in litellm.crusoe_models, ( + f"{model} should be in crusoe_models list" + ) + + +@pytest.mark.asyncio +async def test_crusoe_completion_call(): + """Test completion call with Crusoe provider (requires CRUSOE_API_KEY)""" + if not os.getenv("CRUSOE_API_KEY"): + pytest.skip("CRUSOE_API_KEY not set") + + try: + response = await litellm.acompletion( + model="crusoe/meta-llama/Llama-3.3-70B-Instruct", + messages=[{"role": "user", "content": "Hello, this is a test"}], + max_tokens=10, + ) + assert response.choices[0].message.content + assert response.model + assert response.usage + except Exception as e: + if "crusoe" not in str(e) and "provider" not in str(e).lower(): + raise