From be8bdb580a27f69698db8570a7eb455289c83c50 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 30 Nov 2023 13:17:11 -0800 Subject: [PATCH] (test) proxy + router: add bursty load test --- .../tests/bursty_load_test_completion.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 litellm/proxy/tests/bursty_load_test_completion.py diff --git a/litellm/proxy/tests/bursty_load_test_completion.py b/litellm/proxy/tests/bursty_load_test_completion.py new file mode 100644 index 0000000000..11766c6abe --- /dev/null +++ b/litellm/proxy/tests/bursty_load_test_completion.py @@ -0,0 +1,52 @@ +import time, asyncio +from openai import AsyncOpenAI +import uuid +import traceback + + +litellm_client = AsyncOpenAI( + api_key="test", + base_url="http://0.0.0.0:8000" +) + + +async def litellm_completion(): + # Your existing code for litellm_completion goes here + try: + response = await litellm_client.chat.completions.create( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": f"This is a test: {uuid.uuid4()}"*180}], # this is about 4k tokens per request + ) + print(response) + return response + + except Exception as e: + # If there's an exception, log the error message + with open("error_log.txt", "a") as error_log: + error_log.write(f"Error during completion: {str(e)}\n") + pass + + + +async def main(): + start = time.time() + n = 60 # Send 60 concurrent requests, each with 4k tokens = 240k Tokens + tasks = [litellm_completion() for _ in range(n)] + + chat_completions = await asyncio.gather(*tasks) + + successful_completions = [c for c in chat_completions if c is not None] + + # Write errors to error_log.txt + with open("error_log.txt", "a") as error_log: + for completion in chat_completions: + if isinstance(completion, str): + error_log.write(completion + "\n") + + print(n, time.time() - start, len(successful_completions)) + +if __name__ == "__main__": + # Blank out contents of error_log.txt + open("error_log.txt", "w").close() + + asyncio.run(main())