diff --git a/.gitignore b/.gitignore index 76cf6fdba2..38bf9554b5 100644 --- a/.gitignore +++ b/.gitignore @@ -72,8 +72,7 @@ tests/local_testing/log.txt .codegpt litellm/proxy/_new_new_secret_config.yaml litellm/proxy/custom_guardrail.py -.mypy_cache/* -.mypy_cache/* +**/.mypy_cache/ litellm/proxy/application.log tests/llm_translation/vertex_test_account.json tests/llm_translation/test_vertex_key.json diff --git a/tests/batches_tests/bedrock_batch_completions.jsonl b/tests/batches_tests/bedrock_batch_completions.jsonl index adef9ac2dd..2cd0438fcf 100644 --- a/tests/batches_tests/bedrock_batch_completions.jsonl +++ b/tests/batches_tests/bedrock_batch_completions.jsonl @@ -1,128 +1,128 @@ -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} diff --git a/tests/batches_tests/test_bedrock_files_and_batches.py b/tests/batches_tests/test_bedrock_files_and_batches.py index 6ae373995d..5e216015b5 100644 --- a/tests/batches_tests/test_bedrock_files_and_batches.py +++ b/tests/batches_tests/test_bedrock_files_and_batches.py @@ -70,7 +70,7 @@ async def test_async_file_and_batch(): ######################################################### # bedrock specific params ######################################################### - model="us.anthropic.claude-3-5-sonnet-20240620-v1:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", aws_batch_role_arn="arn:aws:iam::888602223428:role/service-role/AmazonBedrockExecutionRoleForAgents_BB9HNW6V4CV" ) print("CREATED BATCH RESPONSE=", create_batch_response) @@ -79,7 +79,7 @@ async def test_async_file_and_batch(): retrieve_batch_response = await litellm.aretrieve_batch( batch_id=create_batch_response.id, custom_llm_provider="bedrock", - model="us.anthropic.claude-3-5-sonnet-20240620-v1:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", ) print("RETRIEVED BATCH RESPONSE=", retrieve_batch_response) @@ -144,7 +144,7 @@ async def test_bedrock_retrieve_batch(): mock_bedrock_response = { "jobArn": "arn:aws:bedrock:us-west-2:123456789012:model-invocation-job/test-job-123", "jobName": "test-job-123", - "modelId": "us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "modelId": "us.anthropic.claude-haiku-4-5-20251001-v1:0", "roleArn": "arn:aws:iam::123456789012:role/service-role/AmazonBedrockExecutionRoleForAgents_TEST", "status": "InProgress", "message": "Job is in progress", @@ -178,7 +178,7 @@ async def test_bedrock_retrieve_batch(): batch_response = await litellm.aretrieve_batch( batch_id="arn:aws:bedrock:us-west-2:123456789012:model-invocation-job/test-job-123", custom_llm_provider="bedrock", - model="us.anthropic.claude-3-5-sonnet-20240620-v1:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", ) print("MOCKED BATCH RESPONSE=", batch_response) @@ -226,7 +226,7 @@ def test_bedrock_batch_with_encryption_key_in_post_request(): endpoint="/v1/chat/completions", input_file_id="s3://test-bucket/input/test.jsonl", custom_llm_provider="bedrock", - model="us.anthropic.claude-3-5-sonnet-20240620-v1:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", s3_encryption_key_id=test_kms_key_id, aws_batch_role_arn="arn:aws:iam::123456789012:role/test-role" ) diff --git a/tests/litellm_utils_tests/test_health_check.py b/tests/litellm_utils_tests/test_health_check.py index d014b3f72a..c774ea4f1e 100644 --- a/tests/litellm_utils_tests/test_health_check.py +++ b/tests/litellm_utils_tests/test_health_check.py @@ -323,11 +323,11 @@ def test_update_litellm_params_for_health_check(): # Test with Bedrock cross-region inference profile - should preserve the inference profile prefix # AWS requires inference profile IDs like "us.anthropic.claude..." for cross-region routing litellm_params = { - "model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "api_key": "fake_key", } updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) - assert updated_params["model"] == "us.anthropic.claude-3-5-sonnet-20240620-v1:0" + assert updated_params["model"] == "us.anthropic.claude-haiku-4-5-20251001-v1:0" # Test with Bedrock model without region routing - should just strip bedrock/ prefix litellm_params = { @@ -398,13 +398,13 @@ def test_update_litellm_params_for_health_check(): # Test route specifications - routes should be preserved litellm_params = { - "model": "bedrock/converse/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0", "api_key": "fake_key", } updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) assert ( updated_params["model"] - == "converse/us.anthropic.claude-3-5-sonnet-20240620-v1:0" + == "converse/us.anthropic.claude-haiku-4-5-20251001-v1:0" ) litellm_params = { diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index c7c179fd49..8a85102d3a 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -2653,13 +2653,13 @@ def test_bedrock_top_k_param(model, expected_params): def test_bedrock_invoke_provider(): assert ( litellm.AmazonInvokeConfig().get_bedrock_invoke_provider( - "bedrock/invoke/us.anthropic.claude-3-5-sonnet-20240620-v1:0" + "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0" ) == "anthropic" ) assert ( litellm.AmazonInvokeConfig().get_bedrock_invoke_provider( - "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0" + "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" ) == "anthropic" ) @@ -3075,16 +3075,16 @@ async def test_bedrock_passthrough(sync_mode: bool): if sync_mode: response = litellm.llm_passthrough_route( - model="bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", method="POST", - endpoint="/model/us.anthropic.claude-3-5-sonnet-20240620-v1:0/invoke", + endpoint="/model/us.anthropic.claude-haiku-4-5-20251001-v1:0/invoke", data=data, ) else: response = await litellm.allm_passthrough_route( - model="bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", method="POST", - endpoint="/model/us.anthropic.claude-3-5-sonnet-20240620-v1:0/invoke", + endpoint="/model/us.anthropic.claude-haiku-4-5-20251001-v1:0/invoke", data=data, ) @@ -3111,7 +3111,7 @@ async def test_bedrock_passthrough_router(): { "model_name": "special-bedrock-model", "litellm_params": { - "model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", }, } ] @@ -3177,9 +3177,9 @@ async def test_bedrock_converse__streaming_passthrough(monkeypatch): } with patch.object(mock_custom_logger, "async_log_success_event") as mock_callback: response = await litellm.allm_passthrough_route( - model="bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", method="POST", - endpoint="/model/us.anthropic.claude-3-5-sonnet-20240620-v1:0/converse-stream", + endpoint="/model/us.anthropic.claude-haiku-4-5-20251001-v1:0/converse-stream", data=data, ) async for chunk in response: @@ -3230,9 +3230,9 @@ async def test_bedrock_streaming_passthrough_test2(monkeypatch): with patch.object(mock_custom_logger, "async_log_success_event") as mock_callback: response = await litellm.allm_passthrough_route( - model="bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", method="POST", - endpoint="/model/us.anthropic.claude-3-5-sonnet-20240620-v1:0/invoke-with-response-stream", + endpoint="/model/us.anthropic.claude-haiku-4-5-20251001-v1:0/invoke-with-response-stream", data=data, ) async for chunk in response: @@ -3282,9 +3282,9 @@ async def test_bedrock_streaming_passthrough_test1(monkeypatch): with patch.object(mock_custom_logger, "async_log_success_event") as mock_callback: response = await litellm.allm_passthrough_route( - model="bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", method="POST", - endpoint="/model/us.anthropic.claude-3-5-sonnet-20240620-v1:0/invoke-with-response-stream", + endpoint="/model/us.anthropic.claude-haiku-4-5-20251001-v1:0/invoke-with-response-stream", data=data, ) async for chunk in response: diff --git a/tests/llm_translation/test_optional_params.py b/tests/llm_translation/test_optional_params.py index 56f05580cb..b14b25f384 100644 --- a/tests/llm_translation/test_optional_params.py +++ b/tests/llm_translation/test_optional_params.py @@ -1220,7 +1220,7 @@ def test_anthropic_thinking_param(model, expected_thinking): def test_bedrock_invoke_anthropic_max_tokens(): passed_params = { - "model": "invoke/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0", "functions": None, "function_call": None, "temperature": 0.8, diff --git a/tests/local_testing/whitelisted_bedrock_models.txt b/tests/local_testing/whitelisted_bedrock_models.txt index c529fa0160..82e62f01f0 100644 --- a/tests/local_testing/whitelisted_bedrock_models.txt +++ b/tests/local_testing/whitelisted_bedrock_models.txt @@ -26,7 +26,7 @@ anthropic.claude-3-haiku-20240307-v1:0 anthropic.claude-3-5-haiku-20241022-v1:0 anthropic.claude-3-opus-20240229-v1:0 us.anthropic.claude-3-sonnet-20240229-v1:0 -us.anthropic.claude-3-5-sonnet-20240620-v1:0 +us.anthropic.claude-haiku-4-5-20251001-v1:0 us.anthropic.claude-3-7-sonnet-20250219-v1:0 us.anthropic.claude-3-5-sonnet-20241022-v2:0 us.anthropic.claude-3-haiku-20240307-v1:0 diff --git a/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py b/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py index 39b18577a2..cd56ab1f35 100644 --- a/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py +++ b/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py @@ -135,7 +135,7 @@ async def test_e2e_bedrock_knowledgebase_retrieval_with_llm_api_call(setup_vecto litellm._turn_on_debug() async_client = AsyncHTTPHandler() response = await litellm.acompletion( - model="bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "what is litellm?"}], vector_store_ids = [ "T37J8R4WTM" diff --git a/tests/openai_endpoints_tests/bedrock_batch_completions.jsonl b/tests/openai_endpoints_tests/bedrock_batch_completions.jsonl index cfcc5cb246..662866c62d 100644 --- a/tests/openai_endpoints_tests/bedrock_batch_completions.jsonl +++ b/tests/openai_endpoints_tests/bedrock_batch_completions.jsonl @@ -1,2 +1,2 @@ -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} \ No newline at end of file +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} \ No newline at end of file diff --git a/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py b/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py index afc68dc9d4..3d6ccae0e1 100644 --- a/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py +++ b/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py @@ -95,7 +95,7 @@ class TestAnthropicBedrockAPI(BaseAnthropicMessagesTest): @property def model_config(self) -> Dict[str, Any]: return { - "model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", } @property @@ -103,7 +103,7 @@ class TestAnthropicBedrockAPI(BaseAnthropicMessagesTest): """ This is the model name that is expected to be in the logging payload """ - return "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0" + return "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" class TestAnthropicOpenAIAPI(BaseAnthropicMessagesTest): @@ -756,7 +756,7 @@ async def test_anthropic_messages_bedrock_credentials_passthrough(): "type": "message", "role": "assistant", "content": [{"type": "text", "text": "This is a mock response"}], - "model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "stop_reason": "end_turn", "usage": {"input_tokens": 10, "output_tokens": 20}, } @@ -778,7 +778,7 @@ async def test_anthropic_messages_bedrock_credentials_passthrough(): # Call the function with AWS credentials await litellm.anthropic.messages.acreate( messages=[{"role": "user", "content": "Hello, test credentials"}], - model="bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", max_tokens=100, **aws_params, ) @@ -807,7 +807,7 @@ async def test_anthropic_messages_bedrock_dynamic_region(): "type": "message", "role": "assistant", "content": [{"type": "text", "text": "This is a mock response"}], - "model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "stop_reason": "end_turn", "usage": {"input_tokens": 10, "output_tokens": 20}, } @@ -836,7 +836,7 @@ async def test_anthropic_messages_bedrock_dynamic_region(): # Call anthropic.messages.acreate with aws_region_name response = await litellm.anthropic.messages.acreate( messages=[{"role": "user", "content": "Hello, test region"}], - model="bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", max_tokens=100, aws_region_name=test_region, client=mock_client, diff --git a/tests/pass_through_unit_tests/test_bedrock_tool_use_beta_header.py b/tests/pass_through_unit_tests/test_bedrock_tool_use_beta_header.py index e36b2ce9a5..141651dfcd 100644 --- a/tests/pass_through_unit_tests/test_bedrock_tool_use_beta_header.py +++ b/tests/pass_through_unit_tests/test_bedrock_tool_use_beta_header.py @@ -50,7 +50,7 @@ async def test_bedrock_sonnet_4_5_with_advanced_tool_use_beta_header(): # """ # response = await litellm.anthropic.messages.acreate( -# model="bedrock/invoke/us.anthropic.claude-3-5-sonnet-20240620-v1:0", +# model="bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0", # messages=[{"role": "user", "content": "What is 2+2?"}], # max_tokens=100, # provider_specific_header={ diff --git a/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml b/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml index eea8ad6ec1..e137b7ca9d 100644 --- a/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml +++ b/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml @@ -6,7 +6,7 @@ model_list: - model_name: bedrock-claude-sonnet-3.5 litellm_params: - model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0" + model: "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" aws_region_name: "us-east-1" - model_name: bedrock-claude-sonnet-4 diff --git a/tests/test_litellm/llms/bedrock/files/input_batch_completions.jsonl b/tests/test_litellm/llms/bedrock/files/input_batch_completions.jsonl index 41559dfd6f..f0bb4ed81d 100644 --- a/tests/test_litellm/llms/bedrock/files/input_batch_completions.jsonl +++ b/tests/test_litellm/llms/bedrock/files/input_batch_completions.jsonl @@ -1,2 +1,2 @@ -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} diff --git a/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py index 40a17c1211..1f405dbfbf 100644 --- a/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py @@ -16,76 +16,86 @@ class TestBedrockFilesTransformation: def test_transform_openai_jsonl_content_to_bedrock_jsonl_content(self): """ Test transformation of OpenAI JSONL format to Bedrock batch format. - - Validates that the transformation correctly converts OpenAI batch completion + + Validates that the transformation correctly converts OpenAI batch completion format to Bedrock's expected batch format with proper recordId and modelInput structure. """ # Initialize the transformation class transformation = BedrockJsonlFilesTransformation() - + # Load input JSONL file input_file_path = os.path.join( - os.path.dirname(__file__), - "input_batch_completions.jsonl" + os.path.dirname(__file__), "input_batch_completions.jsonl" ) - + # Read and parse the JSONL content openai_jsonl_content = [] - with open(input_file_path, 'r') as f: + with open(input_file_path, "r") as f: for line in f: if line.strip(): openai_jsonl_content.append(json.loads(line)) - + # Transform the content - bedrock_jsonl_content = transformation._transform_openai_jsonl_content_to_bedrock_jsonl_content( - openai_jsonl_content=openai_jsonl_content + bedrock_jsonl_content = ( + transformation._transform_openai_jsonl_content_to_bedrock_jsonl_content( + openai_jsonl_content=openai_jsonl_content + ) ) - + # Print the transformation results for validation print("\n=== INPUT (OpenAI format) ===") for i, content in enumerate(openai_jsonl_content): print(f"Record {i+1}:") print(json.dumps(content, indent=2)) print() - + print("\n=== OUTPUT (Bedrock format) ===") for i, content in enumerate(bedrock_jsonl_content): print(f"Record {i+1}:") print(json.dumps(content, indent=2)) print() - + # Basic validation - assert len(bedrock_jsonl_content) == len(openai_jsonl_content), "Should have same number of records" - + assert len(bedrock_jsonl_content) == len( + openai_jsonl_content + ), "Should have same number of records" + # Check structure of transformed records for i, record in enumerate(bedrock_jsonl_content): assert "recordId" in record, f"Record {i+1} should have recordId" assert "modelInput" in record, f"Record {i+1} should have modelInput" - + # Check recordId matches custom_id from input expected_custom_id = openai_jsonl_content[i].get("custom_id") - assert record["recordId"] == expected_custom_id, f"Record {i+1} recordId should match custom_id" - + assert ( + record["recordId"] == expected_custom_id + ), f"Record {i+1} recordId should match custom_id" + # Check modelInput has expected structure model_input = record["modelInput"] - assert isinstance(model_input, dict), f"Record {i+1} modelInput should be a dictionary" - + assert isinstance( + model_input, dict + ), f"Record {i+1} modelInput should be a dictionary" + # For Anthropic models, should have anthropic_version and messages if "anthropic.claude" in openai_jsonl_content[i]["body"]["model"]: - assert "anthropic_version" in model_input, f"Record {i+1} should have anthropic_version" + assert ( + "anthropic_version" in model_input + ), f"Record {i+1} should have anthropic_version" assert "messages" in model_input, f"Record {i+1} should have messages" - assert "max_tokens" in model_input, f"Record {i+1} should have max_tokens" - + assert ( + "max_tokens" in model_input + ), f"Record {i+1} should have max_tokens" + # Write expected output to file for reference expected_output_path = os.path.join( - os.path.dirname(__file__), - "expected_bedrock_batch_completions.jsonl" + os.path.dirname(__file__), "expected_bedrock_batch_completions.jsonl" ) - - with open(expected_output_path, 'w') as f: + + with open(expected_output_path, "w") as f: for record in bedrock_jsonl_content: - f.write(json.dumps(record) + '\n') - + f.write(json.dumps(record) + "\n") + print(f"\n=== Expected output written to: {expected_output_path} ===") def test_nova_text_only_uses_converse_format(self): @@ -128,17 +138,17 @@ class TestBedrockFilesTransformation: model_input = record["modelInput"] # Must have inferenceConfig with maxTokens, NOT top-level max_tokens - assert "inferenceConfig" in model_input, ( - "Nova modelInput must contain inferenceConfig" - ) + assert ( + "inferenceConfig" in model_input + ), "Nova modelInput must contain inferenceConfig" assert model_input["inferenceConfig"]["maxTokens"] == 50 assert model_input["inferenceConfig"]["temperature"] == 0.7 - assert "max_tokens" not in model_input, ( - "max_tokens must NOT be at the top level for Nova" - ) - assert "temperature" not in model_input, ( - "temperature must NOT be at the top level for Nova" - ) + assert ( + "max_tokens" not in model_input + ), "max_tokens must NOT be at the top level for Nova" + assert ( + "temperature" not in model_input + ), "temperature must NOT be at the top level for Nova" # Must have messages assert "messages" in model_input @@ -215,22 +225,18 @@ class TestBedrockFilesTransformation: if "image" in block: has_image = True # Verify Converse image format - assert "format" in block["image"], ( - "Image block must have format field" - ) - assert "source" in block["image"], ( - "Image block must have source field" - ) - assert "bytes" in block["image"]["source"], ( - "Image source must have bytes field" - ) + assert "format" in block["image"], "Image block must have format field" + assert "source" in block["image"], "Image block must have source field" + assert ( + "bytes" in block["image"]["source"] + ), "Image source must have bytes field" # Must NOT have OpenAI-style image_url - assert "image_url" not in block, ( - "image_url must not appear in Converse format" - ) - assert block.get("type") != "image_url", ( - "type=image_url must not appear in Converse format" - ) + assert ( + "image_url" not in block + ), "image_url must not appear in Converse format" + assert ( + block.get("type") != "image_url" + ), "type=image_url must not appear in Converse format" assert has_text, "Should have a text content block" assert has_image, "Should have an image content block" @@ -250,7 +256,7 @@ class TestBedrockFilesTransformation: "method": "POST", "url": "/v1/chat/completions", "body": { - "model": "us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [ {"role": "system", "content": "You are helpful."}, {"role": "user", "content": "Hello!"}, @@ -314,12 +320,10 @@ class TestBedrockFilesTransformation: data=create_file_data, ) - assert "us-gov-west-1" in url, ( - f"Expected us-gov-west-1 in URL but got: {url}" - ) - assert "us-west-2" not in url, ( - f"us-west-2 must not appear when s3_region_name is set, got: {url}" - ) + assert "us-gov-west-1" in url, f"Expected us-gov-west-1 in URL but got: {url}" + assert ( + "us-west-2" not in url + ), f"us-west-2 must not appear when s3_region_name is set, got: {url}" assert "litellm-batch-352026" in url def test_transform_create_file_request_injects_s3_region_for_signing(self): @@ -370,9 +374,9 @@ class TestBedrockFilesTransformation: litellm_params=litellm_params, ) - assert captured_optional_params.get("aws_region_name") == "us-gov-west-1", ( - "s3_region_name must be forwarded as aws_region_name for SigV4 signing" - ) + assert ( + captured_optional_params.get("aws_region_name") == "us-gov-west-1" + ), "s3_region_name must be forwarded as aws_region_name for SigV4 signing" def test_s3_region_name_wins_over_aws_region_name_for_signing(self): """ @@ -426,9 +430,9 @@ class TestBedrockFilesTransformation: litellm_params=litellm_params, ) - assert captured_optional_params.get("aws_region_name") == "us-gov-west-1", ( - "s3_region_name must override aws_region_name for SigV4 signing" - ) + assert ( + captured_optional_params.get("aws_region_name") == "us-gov-west-1" + ), "s3_region_name must override aws_region_name for SigV4 signing" def test_openai_passthrough_still_works(self): """ @@ -465,4 +469,3 @@ class TestBedrockFilesTransformation: assert "messages" in model_input assert "max_tokens" in model_input assert model_input["max_tokens"] == 10 - diff --git a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py index 76fe0d7568..dfe240979e 100644 --- a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py +++ b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py @@ -14,19 +14,23 @@ def test_bedrock_passthrough_get_complete_url_default_endpoint(): config = BedrockPassthroughConfig() # Mock the methods following the pattern from test_base_aws_llm.py - with patch.object(config, '_get_aws_region_name', return_value="us-east-1"), \ - patch.object(config, 'get_runtime_endpoint', return_value=( - "https://bedrock-runtime.us-east-1.amazonaws.com", - "https://bedrock-runtime.us-east-1.amazonaws.com" - )) as mock_get_runtime: - + with patch.object( + config, "_get_aws_region_name", return_value="us-east-1" + ), patch.object( + config, + "get_runtime_endpoint", + return_value=( + "https://bedrock-runtime.us-east-1.amazonaws.com", + "https://bedrock-runtime.us-east-1.amazonaws.com", + ), + ) as mock_get_runtime: url, api_base = config.get_complete_url( api_base=None, api_key=None, model="anthropic.claude-3-sonnet", endpoint="/model/anthropic.claude-3-sonnet/invoke", request_query_params=None, - litellm_params={} + litellm_params={}, ) # Verify get_runtime_endpoint was called with correct parameters @@ -34,11 +38,14 @@ def test_bedrock_passthrough_get_complete_url_default_endpoint(): api_base=None, aws_bedrock_runtime_endpoint=None, aws_region_name="us-east-1", - endpoint_type="runtime" + endpoint_type="runtime", ) - + # Verify URL construction - assert str(url) == "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-sonnet/invoke" + assert ( + str(url) + == "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-sonnet/invoke" + ) assert api_base == "https://bedrock-runtime.us-east-1.amazonaws.com" @@ -46,19 +53,20 @@ def test_bedrock_passthrough_get_complete_url_custom_endpoint_no_path(): """Test get_complete_url with custom endpoint (no base path)""" config = BedrockPassthroughConfig() - with patch.object(config, '_get_aws_region_name', return_value="us-west-2"), \ - patch.object(config, 'get_runtime_endpoint', return_value=( - "http://proxy.com", - "http://proxy.com" - )) as mock_get_runtime: - + with patch.object( + config, "_get_aws_region_name", return_value="us-west-2" + ), patch.object( + config, + "get_runtime_endpoint", + return_value=("http://proxy.com", "http://proxy.com"), + ) as mock_get_runtime: url, api_base = config.get_complete_url( api_base="http://proxy.com", api_key=None, model="anthropic.claude-3-sonnet", endpoint="/model/anthropic.claude-3-sonnet/invoke", request_query_params=None, - litellm_params={} + litellm_params={}, ) # Verify get_runtime_endpoint was called with the api_base @@ -66,9 +74,9 @@ def test_bedrock_passthrough_get_complete_url_custom_endpoint_no_path(): api_base="http://proxy.com", aws_bedrock_runtime_endpoint=None, aws_region_name="us-west-2", - endpoint_type="runtime" + endpoint_type="runtime", ) - + # Verify URL construction assert str(url) == "http://proxy.com/model/anthropic.claude-3-sonnet/invoke" assert api_base == "http://proxy.com" @@ -78,12 +86,13 @@ def test_bedrock_passthrough_get_complete_url_custom_endpoint_with_path(): """Test get_complete_url with custom endpoint that has a base path""" config = BedrockPassthroughConfig() - with patch.object(config, '_get_aws_region_name', return_value="us-west-2"), \ - patch.object(config, 'get_runtime_endpoint', return_value=( - "http://proxy.com/bedrockproxy", - "http://proxy.com/bedrockproxy" - )) as mock_get_runtime: - + with patch.object( + config, "_get_aws_region_name", return_value="us-west-2" + ), patch.object( + config, + "get_runtime_endpoint", + return_value=("http://proxy.com/bedrockproxy", "http://proxy.com/bedrockproxy"), + ) as mock_get_runtime: url, api_base = config.get_complete_url( api_base="http://proxy.com/bedrockproxy", api_key=None, @@ -92,7 +101,7 @@ def test_bedrock_passthrough_get_complete_url_custom_endpoint_with_path(): request_query_params=None, litellm_params={ "aws_bedrock_runtime_endpoint": "http://proxy.com/bedrockproxy" - } + }, ) # Verify get_runtime_endpoint was called with correct parameters @@ -100,37 +109,40 @@ def test_bedrock_passthrough_get_complete_url_custom_endpoint_with_path(): api_base="http://proxy.com/bedrockproxy", aws_bedrock_runtime_endpoint="http://proxy.com/bedrockproxy", aws_region_name="us-west-2", - endpoint_type="runtime" + endpoint_type="runtime", ) - + # Verify URL construction preserves the proxy path - assert str(url) == "http://proxy.com/bedrockproxy/model/anthropic.claude-3-sonnet/invoke" + assert ( + str(url) + == "http://proxy.com/bedrockproxy/model/anthropic.claude-3-sonnet/invoke" + ) assert api_base == "http://proxy.com/bedrockproxy" def test_format_url_simple_joining(): """Test format_url with simple URL joining""" config = BedrockPassthroughConfig() - + result = config.format_url( endpoint="model/test/invoke", base_target_url="https://api.example.com", - request_query_params={} + request_query_params={}, ) - + assert str(result) == "https://api.example.com/model/test/invoke" def test_format_url_preserves_proxy_paths(): """Test format_url preserves proxy paths in base URL""" config = BedrockPassthroughConfig() - + result = config.format_url( endpoint="model/test/invoke", base_target_url="http://proxy.com/bedrockproxy", - request_query_params={} + request_query_params={}, ) - + # This is the key test - proxy path should be preserved assert str(result) == "http://proxy.com/bedrockproxy/model/test/invoke" @@ -138,13 +150,13 @@ def test_format_url_preserves_proxy_paths(): def test_format_url_with_query_parameters(): """Test format_url properly handles query parameters""" config = BedrockPassthroughConfig() - + result = config.format_url( endpoint="model/test/invoke", base_target_url="http://proxy.com/bedrockproxy", - request_query_params={"param1": "value1", "param2": "value2"} + request_query_params={"param1": "value1", "param2": "value2"}, ) - + # Should preserve proxy path and add query params result_str = str(result) assert "http://proxy.com/bedrockproxy/model/test/invoke" in result_str @@ -155,21 +167,21 @@ def test_format_url_with_query_parameters(): def test_format_url_handles_trailing_slash_normalization(): """Test format_url properly handles base URLs with and without trailing slashes""" config = BedrockPassthroughConfig() - + # Test with trailing slash result_with_slash = config.format_url( endpoint="model/test/invoke", base_target_url="http://proxy.com/bedrockproxy/", - request_query_params={} + request_query_params={}, ) - + # Test without trailing slash result_without_slash = config.format_url( endpoint="model/test/invoke", base_target_url="http://proxy.com/bedrockproxy", - request_query_params={} + request_query_params={}, ) - + # Both should produce the same result assert str(result_with_slash) == str(result_without_slash) assert str(result_with_slash) == "http://proxy.com/bedrockproxy/model/test/invoke" @@ -178,39 +190,49 @@ def test_format_url_handles_trailing_slash_normalization(): def test_bedrock_passthrough_with_application_inference_profile(): """ Test get_complete_url with Application Inference Profile ARN as model_id. - + This test verifies the fix for GitHub issue #18761 where Bedrock passthrough was not working with Application Inference Profiles. The model_id (ARN) should replace the translated model name in the endpoint URL and be properly encoded. """ config = BedrockPassthroughConfig() - + model = "anthropic.claude-sonnet-4-20250514-v1:0" - model_id = "arn:aws:bedrock:eu-west-1:123456789:application-inference-profile/abcdefgh1234" + model_id = ( + "arn:aws:bedrock:eu-west-1:123456789:application-inference-profile/abcdefgh1234" + ) endpoint = f"model/{model}/invoke" - - with patch.object(config, '_get_aws_region_name', return_value="eu-west-1"), \ - patch.object(config, 'get_runtime_endpoint', return_value=( - "https://bedrock-runtime.eu-west-1.amazonaws.com", - "https://bedrock-runtime.eu-west-1.amazonaws.com" - )): - + + with patch.object( + config, "_get_aws_region_name", return_value="eu-west-1" + ), patch.object( + config, + "get_runtime_endpoint", + return_value=( + "https://bedrock-runtime.eu-west-1.amazonaws.com", + "https://bedrock-runtime.eu-west-1.amazonaws.com", + ), + ): url, api_base = config.get_complete_url( api_base=None, api_key=None, model=model, endpoint=endpoint, request_query_params=None, - litellm_params={"model_id": model_id, "aws_region_name": "eu-west-1"} + litellm_params={"model_id": model_id, "aws_region_name": "eu-west-1"}, ) - + # Verify that the URL contains the encoded model_id (ARN) instead of the model name url_str = str(url) # The ARN slash should be encoded as %2F - assert "application-inference-profile%2F" in url_str, f"Expected encoded ARN in URL, but got: {url_str}" - assert model not in url_str, f"Model name should be replaced by model_id, but got: {url_str}" + assert ( + "application-inference-profile%2F" in url_str + ), f"Expected encoded ARN in URL, but got: {url_str}" + assert ( + model not in url_str + ), f"Model name should be replaced by model_id, but got: {url_str}" assert "/invoke" in url_str, "Expected /invoke action in URL" - + # Verify the complete URL structure with encoded ARN encoded_model_id = "arn:aws:bedrock:eu-west-1:123456789:application-inference-profile%2Fabcdefgh1234" expected_url = f"https://bedrock-runtime.eu-west-1.amazonaws.com/model/{encoded_model_id}/invoke" @@ -220,26 +242,32 @@ def test_bedrock_passthrough_with_application_inference_profile(): def test_bedrock_passthrough_with_inference_profile_converse_endpoint(): """Test Application Inference Profile with converse endpoint and proper ARN encoding""" config = BedrockPassthroughConfig() - + model = "anthropic.claude-sonnet-4-20250514-v1:0" - model_id = "arn:aws:bedrock:us-east-1:123456789:application-inference-profile/xyz123" + model_id = ( + "arn:aws:bedrock:us-east-1:123456789:application-inference-profile/xyz123" + ) endpoint = f"model/{model}/converse" - - with patch.object(config, '_get_aws_region_name', return_value="us-east-1"), \ - patch.object(config, 'get_runtime_endpoint', return_value=( - "https://bedrock-runtime.us-east-1.amazonaws.com", - "https://bedrock-runtime.us-east-1.amazonaws.com" - )): - + + with patch.object( + config, "_get_aws_region_name", return_value="us-east-1" + ), patch.object( + config, + "get_runtime_endpoint", + return_value=( + "https://bedrock-runtime.us-east-1.amazonaws.com", + "https://bedrock-runtime.us-east-1.amazonaws.com", + ), + ): url, api_base = config.get_complete_url( api_base=None, api_key=None, model=model, endpoint=endpoint, request_query_params=None, - litellm_params={"model_id": model_id} + litellm_params={"model_id": model_id}, ) - + url_str = str(url) # The ARN should be encoded with %2F assert "application-inference-profile%2F" in url_str @@ -250,108 +278,131 @@ def test_bedrock_passthrough_with_inference_profile_converse_endpoint(): def test_bedrock_passthrough_without_model_id_backward_compatibility(): """ Test that passthrough still works without model_id (backward compatibility). - + When model_id is not provided, the system should use the model name as before. """ config = BedrockPassthroughConfig() - + model = "anthropic.claude-3-sonnet" endpoint = f"model/{model}/invoke" - - with patch.object(config, '_get_aws_region_name', return_value="us-east-1"), \ - patch.object(config, 'get_runtime_endpoint', return_value=( - "https://bedrock-runtime.us-east-1.amazonaws.com", - "https://bedrock-runtime.us-east-1.amazonaws.com" - )): - + + with patch.object( + config, "_get_aws_region_name", return_value="us-east-1" + ), patch.object( + config, + "get_runtime_endpoint", + return_value=( + "https://bedrock-runtime.us-east-1.amazonaws.com", + "https://bedrock-runtime.us-east-1.amazonaws.com", + ), + ): url, api_base = config.get_complete_url( api_base=None, api_key=None, model=model, endpoint=endpoint, request_query_params=None, - litellm_params={} # No model_id provided + litellm_params={}, # No model_id provided ) - + # Verify that the URL contains the model name (not replaced) url_str = str(url) - assert model in url_str, f"Expected model name in URL when model_id not provided, but got: {url_str}" - expected_url = f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model}/invoke" + assert ( + model in url_str + ), f"Expected model name in URL when model_id not provided, but got: {url_str}" + expected_url = ( + f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model}/invoke" + ) assert url_str == expected_url def test_bedrock_passthrough_region_extraction_from_inference_profile_arn(): """Test that AWS region is correctly extracted from Application Inference Profile ARN""" config = BedrockPassthroughConfig() - + model = "anthropic.claude-sonnet-4-20250514-v1:0" # ARN contains us-west-2 region - model_id = "arn:aws:bedrock:us-west-2:123456789:application-inference-profile/test123" + model_id = ( + "arn:aws:bedrock:us-west-2:123456789:application-inference-profile/test123" + ) endpoint = f"model/{model}/invoke" - + # Don't provide aws_region_name in litellm_params to test ARN extraction - with patch.object(config, 'get_runtime_endpoint', return_value=( - "https://bedrock-runtime.us-west-2.amazonaws.com", - "https://bedrock-runtime.us-west-2.amazonaws.com" - )): - + with patch.object( + config, + "get_runtime_endpoint", + return_value=( + "https://bedrock-runtime.us-west-2.amazonaws.com", + "https://bedrock-runtime.us-west-2.amazonaws.com", + ), + ): url, api_base = config.get_complete_url( api_base=None, api_key=None, model=model, endpoint=endpoint, request_query_params=None, - litellm_params={"model_id": model_id} # Region should be extracted from ARN + litellm_params={ + "model_id": model_id + }, # Region should be extracted from ARN ) - + # Verify that the region from ARN is used in the base URL - assert "us-west-2" in api_base, f"Expected region 'us-west-2' from ARN in base URL, but got: {api_base}" + assert ( + "us-west-2" in api_base + ), f"Expected region 'us-west-2' from ARN in base URL, but got: {api_base}" def test_bedrock_passthrough_model_id_arn_encoding(): """ Test that model_id ARNs are properly URL-encoded when used in endpoints. - + This is the critical fix for the issue where ARNs with slashes need to be encoded so they're treated as a single path component rather than multiple path segments. - + For example: arn:aws:bedrock:us-east-1:590183661440:application-inference-profile/b943q2qbl3m7 should become: arn:aws:bedrock:us-east-1:590183661440:application-inference-profile%2Fb943q2qbl3m7 """ config = BedrockPassthroughConfig() - + model = "bedrock-claude-4-5-sonnet" # ARN with a slash that needs encoding model_id = "arn:aws:bedrock:us-east-1:590183661440:application-inference-profile/b943q2qbl3m7" endpoint = f"/model/{model}/converse" - - with patch.object(config, '_get_aws_region_name', return_value="us-east-1"), \ - patch.object(config, 'get_runtime_endpoint', return_value=( - "https://bedrock-runtime.us-east-1.amazonaws.com", - "https://bedrock-runtime.us-east-1.amazonaws.com" - )): - + + with patch.object( + config, "_get_aws_region_name", return_value="us-east-1" + ), patch.object( + config, + "get_runtime_endpoint", + return_value=( + "https://bedrock-runtime.us-east-1.amazonaws.com", + "https://bedrock-runtime.us-east-1.amazonaws.com", + ), + ): url, api_base = config.get_complete_url( api_base=None, api_key=None, model=model, endpoint=endpoint, request_query_params=None, - litellm_params={"model_id": model_id} + litellm_params={"model_id": model_id}, ) - + url_str = str(url) - + # The slash in the ARN after application-inference-profile should be encoded as %2F - assert "application-inference-profile%2F" in url_str, \ - f"Expected encoded ARN with %2F in URL, but got: {url_str}" - + assert ( + "application-inference-profile%2F" in url_str + ), f"Expected encoded ARN with %2F in URL, but got: {url_str}" + # The unencoded version should NOT be in the URL - assert "application-inference-profile/" not in url_str, \ - f"ARN slash should be encoded, but found unencoded version in: {url_str}" - + assert ( + "application-inference-profile/" not in url_str + ), f"ARN slash should be encoded, but found unencoded version in: {url_str}" + # Verify the complete expected URL structure expected_encoded_model_id = "arn:aws:bedrock:us-east-1:590183661440:application-inference-profile%2Fb943q2qbl3m7" expected_url = f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{expected_encoded_model_id}/converse" @@ -363,33 +414,41 @@ def test_bedrock_passthrough_model_id_arn_encoding_invoke_endpoint(): Test ARN encoding with /invoke endpoint (not just /converse). """ config = BedrockPassthroughConfig() - + model = "anthropic.claude-sonnet-4-5-20250929-v1:0" - model_id = "arn:aws:bedrock:us-east-1:123456789:application-inference-profile/xyz789" + model_id = ( + "arn:aws:bedrock:us-east-1:123456789:application-inference-profile/xyz789" + ) endpoint = f"/model/{model}/invoke" - - with patch.object(config, '_get_aws_region_name', return_value="us-east-1"), \ - patch.object(config, 'get_runtime_endpoint', return_value=( - "https://bedrock-runtime.us-east-1.amazonaws.com", - "https://bedrock-runtime.us-east-1.amazonaws.com" - )): - + + with patch.object( + config, "_get_aws_region_name", return_value="us-east-1" + ), patch.object( + config, + "get_runtime_endpoint", + return_value=( + "https://bedrock-runtime.us-east-1.amazonaws.com", + "https://bedrock-runtime.us-east-1.amazonaws.com", + ), + ): url, api_base = config.get_complete_url( api_base=None, api_key=None, model=model, endpoint=endpoint, request_query_params=None, - litellm_params={"model_id": model_id} + litellm_params={"model_id": model_id}, ) - + url_str = str(url) - + # Verify encoding assert "application-inference-profile%2F" in url_str assert "/invoke" in url_str - - expected_encoded_model_id = "arn:aws:bedrock:us-east-1:123456789:application-inference-profile%2Fxyz789" + + expected_encoded_model_id = ( + "arn:aws:bedrock:us-east-1:123456789:application-inference-profile%2Fxyz789" + ) expected_url = f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{expected_encoded_model_id}/invoke" assert url_str == expected_url @@ -399,33 +458,38 @@ def test_bedrock_passthrough_model_id_without_arn(): Test that non-ARN model_ids (regular model IDs) are not affected by encoding logic. """ config = BedrockPassthroughConfig() - + model = "my-model" # Regular model ID (not an ARN) - model_id = "us.anthropic.claude-3-5-sonnet-20240620-v1:0" + model_id = "us.anthropic.claude-haiku-4-5-20251001-v1:0" endpoint = f"/model/{model}/converse" - - with patch.object(config, '_get_aws_region_name', return_value="us-east-1"), \ - patch.object(config, 'get_runtime_endpoint', return_value=( - "https://bedrock-runtime.us-east-1.amazonaws.com", - "https://bedrock-runtime.us-east-1.amazonaws.com" - )): - + + with patch.object( + config, "_get_aws_region_name", return_value="us-east-1" + ), patch.object( + config, + "get_runtime_endpoint", + return_value=( + "https://bedrock-runtime.us-east-1.amazonaws.com", + "https://bedrock-runtime.us-east-1.amazonaws.com", + ), + ): url, api_base = config.get_complete_url( api_base=None, api_key=None, model=model, endpoint=endpoint, request_query_params=None, - litellm_params={"model_id": model_id} + litellm_params={"model_id": model_id}, ) - + url_str = str(url) - + # Regular model ID should be used as-is (no encoding needed) assert model_id in url_str assert "%2F" not in url_str, "Non-ARN model IDs should not be encoded" - - expected_url = f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model_id}/converse" - assert url_str == expected_url + expected_url = ( + f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model_id}/converse" + ) + assert url_str == expected_url diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index a02464e946..73bd200f8d 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1992,7 +1992,7 @@ def test_add_deployment_model_to_endpoint_for_llm_passthrough_route(): { "model_name": "special-bedrock-model", "litellm_params": { - "model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", }, } ], @@ -2006,12 +2006,12 @@ def test_add_deployment_model_to_endpoint_for_llm_passthrough_route(): result = router._add_deployment_model_to_endpoint_for_llm_passthrough_route( kwargs=kwargs, model="special-bedrock-model", - model_name="bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + model_name="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", ) assert ( result["endpoint"] - == "/model/us.anthropic.claude-3-5-sonnet-20240620-v1:0/invoke" - ), f"Expected '/model/us.anthropic.claude-3-5-sonnet-20240620-v1:0/invoke', got '{result['endpoint']}'" + == "/model/us.anthropic.claude-haiku-4-5-20251001-v1:0/invoke" + ), f"Expected '/model/us.anthropic.claude-haiku-4-5-20251001-v1:0/invoke', got '{result['endpoint']}'" # Test Case 2: Bedrock invoke-with-response-stream endpoint kwargs = { @@ -2021,11 +2021,11 @@ def test_add_deployment_model_to_endpoint_for_llm_passthrough_route(): result = router._add_deployment_model_to_endpoint_for_llm_passthrough_route( kwargs=kwargs, model="special-bedrock-model", - model_name="bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + model_name="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", ) assert ( result["endpoint"] - == "/model/us.anthropic.claude-3-5-sonnet-20240620-v1:0/invoke-with-response-stream" + == "/model/us.anthropic.claude-haiku-4-5-20251001-v1:0/invoke-with-response-stream" ), f"Expected streaming endpoint with stripped prefix, got '{result['endpoint']}'" # Test Case 3: Bedrock converse endpoint @@ -2139,7 +2139,7 @@ def test_get_deployment_credentials_with_provider_aws_bedrock_runtime_endpoint() { "model_name": "bedrock-claude-model", "litellm_params": { - "model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "aws_access_key_id": "test-access-key", "aws_secret_access_key": "test-secret-key", "aws_region_name": "us-east-1",