From b002328c8da39f5e9dba88fcefa54ab526470b8f Mon Sep 17 00:00:00 2001 From: Andres Barbaro Date: Thu, 23 May 2024 22:42:42 -0500 Subject: [PATCH 1/5] Fix issue with delta being None when Deferred / Async Content Filter is enabled on Azure OpenAI --- litellm/utils.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 33dfb261eb..b7237041d1 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10637,7 +10637,8 @@ class CustomStreamWrapper: data_json = json.loads(chunk[5:]) # chunk.startswith("data:"): try: if len(data_json["choices"]) > 0: - text = data_json["choices"][0]["delta"].get("content", "") + delta = data_json["choices"][0]["delta"] + text = "" if delta is None else delta.get("content", "") if data_json["choices"][0].get("finish_reason", None): is_finished = True finish_reason = data_json["choices"][0]["finish_reason"] @@ -11405,12 +11406,14 @@ class CustomStreamWrapper: model_response.id = original_chunk.id self.response_id = original_chunk.id if len(original_chunk.choices) > 0: + delta = original_chunk.choices[0].delta if ( - original_chunk.choices[0].delta.function_call is not None - or original_chunk.choices[0].delta.tool_calls is not None + delta is not None and ( + delta.function_call is not None + or delta.tool_calls is not None + ) ): try: - delta = original_chunk.choices[0].delta model_response.system_fingerprint = ( original_chunk.system_fingerprint ) @@ -11469,7 +11472,7 @@ class CustomStreamWrapper: model_response.choices[0].delta = Delta() else: try: - delta = dict(original_chunk.choices[0].delta) + delta = dict() if original_chunk.choices[0].delta is None else dict(original_chunk.choices[0].delta) print_verbose(f"original delta: {delta}") model_response.choices[0].delta = Delta(**delta) print_verbose( From 565a7f1ce2df607019a6c0acb94b866d2b61e8ba Mon Sep 17 00:00:00 2001 From: Andres Barbaro Date: Fri, 24 May 2024 10:57:35 -0500 Subject: [PATCH 2/5] Add test to validate chunks with no delta are processed correctly --- litellm/tests/test_streaming.py | 238 ++++++++++++++++++++++++++++++++ 1 file changed, 238 insertions(+) diff --git a/litellm/tests/test_streaming.py b/litellm/tests/test_streaming.py index 237d3895d2..f3082c93f4 100644 --- a/litellm/tests/test_streaming.py +++ b/litellm/tests/test_streaming.py @@ -6,6 +6,7 @@ import traceback import time, pytest from pydantic import BaseModel from typing import Tuple +from unittest.mock import patch, MagicMock sys.path.insert( 0, os.path.abspath("../..") @@ -235,6 +236,243 @@ def test_completion_azure_stream_special_char(): assert len(response_str) > 0 +def test_completion_azure_stream_content_filter_no_delta(): + """ + Tests streaming from Azure when the chunks have no delta because they represent the filtered content + """ + try: + chunks = [ + { + "id": "chatcmpl-9SQxdH5hODqkWyJopWlaVOOUnFwlj", + "choices": [ + { + "delta": { + "content": "", + "role": "assistant" + }, + "finish_reason": None, + "index": 0 + } + ], + "created": 1716563849, + "model": "gpt-4o-2024-05-13", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_5f4bad809a" + }, + { + "id": "chatcmpl-9SQxdH5hODqkWyJopWlaVOOUnFwlj", + "choices": [ + { + "delta": { + "content": "This" + }, + "finish_reason": None, + "index": 0 + } + ], + "created": 1716563849, + "model": "gpt-4o-2024-05-13", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_5f4bad809a" + }, + { + "id": "chatcmpl-9SQxdH5hODqkWyJopWlaVOOUnFwlj", + "choices": [ + { + "delta": { + "content": " is" + }, + "finish_reason": None, + "index": 0 + } + ], + "created": 1716563849, + "model": "gpt-4o-2024-05-13", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_5f4bad809a" + }, + { + "id": "chatcmpl-9SQxdH5hODqkWyJopWlaVOOUnFwlj", + "choices": [ + { + "delta": { + "content": " a" + }, + "finish_reason": None, + "index": 0 + } + ], + "created": 1716563849, + "model": "gpt-4o-2024-05-13", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_5f4bad809a" + }, + { + "id": "chatcmpl-9SQxdH5hODqkWyJopWlaVOOUnFwlj", + "choices": [ + { + "delta": { + "content": " dummy" + }, + "finish_reason": None, + "index": 0 + } + ], + "created": 1716563849, + "model": "gpt-4o-2024-05-13", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_5f4bad809a" + }, + { + "id": "chatcmpl-9SQxdH5hODqkWyJopWlaVOOUnFwlj", + "choices": [ + { + "delta": { + "content": " response" + }, + "finish_reason": None, + "index": 0 + } + ], + "created": 1716563849, + "model": "gpt-4o-2024-05-13", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_5f4bad809a" + }, + { + "id": "", + "choices": [ + { + "finish_reason": None, + "index": 0, + "content_filter_offsets": { + "check_offset": 35159, + "start_offset": 35159, + "end_offset": 36150 + }, + "content_filter_results": { + "hate": { + "filtered": False, + "severity": "safe" + }, + "self_harm": { + "filtered": False, + "severity": "safe" + }, + "sexual": { + "filtered": False, + "severity": "safe" + }, + "violence": { + "filtered": False, + "severity": "safe" + } + } + } + ], + "created": 0, + "model": "", + "object": "" + }, + { + "id": "chatcmpl-9SQxdH5hODqkWyJopWlaVOOUnFwlj", + "choices": [ + { + "delta": { + "content": "." + }, + "finish_reason": None, + "index": 0 + } + ], + "created": 1716563849, + "model": "gpt-4o-2024-05-13", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_5f4bad809a" + }, + { + "id": "chatcmpl-9SQxdH5hODqkWyJopWlaVOOUnFwlj", + "choices": [ + { + "delta": {}, + "finish_reason": "stop", + "index": 0 + } + ], + "created": 1716563849, + "model": "gpt-4o-2024-05-13", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_5f4bad809a" + }, + { + "id": "", + "choices": [ + { + "finish_reason": None, + "index": 0, + "content_filter_offsets": { + "check_offset": 36150, + "start_offset": 36060, + "end_offset": 37029 + }, + "content_filter_results": { + "hate": { + "filtered": False, + "severity": "safe" + }, + "self_harm": { + "filtered": False, + "severity": "safe" + }, + "sexual": { + "filtered": False, + "severity": "safe" + }, + "violence": { + "filtered": False, + "severity": "safe" + } + } + } + ], + "created": 0, + "model": "", + "object": "" + } + ] + + stream_iterator = iter(chunks) + + litellm.set_verbose = True + + logging_obj = litellm.Logging( + model="berri-benchmarking-Llama-2-70b-chat-hf-4", + messages=messages, + stream=True, + litellm_call_id="1234", + function_id="function_id", + call_type="acompletion", + start_time=time.time(), + ) + response = litellm.CustomStreamWrapper( + completion_stream=stream_iterator, + model="azure/gpt-4o", + custom_llm_provider="azure", + logging_obj=logging_obj, + ) + complete_response = "" + for idx, chunk in enumerate(response): + # print + chunk, finished = streaming_format_tests(idx, chunk) + complete_response += chunk + if finished: + break + assert len(complete_response) > 0 + + except Exception as e: + pytest.fail(f"An exception occurred - {str(e)}") + + def test_completion_cohere_stream_bad_key(): try: litellm.cache = None From 638984b764f3c19925118ff2a030f5a1c1592bf8 Mon Sep 17 00:00:00 2001 From: Andres Barbaro Date: Fri, 24 May 2024 11:00:29 -0500 Subject: [PATCH 3/5] Remove unnecessary import --- litellm/tests/test_streaming.py | 1 - 1 file changed, 1 deletion(-) diff --git a/litellm/tests/test_streaming.py b/litellm/tests/test_streaming.py index f3082c93f4..f69dce9a3d 100644 --- a/litellm/tests/test_streaming.py +++ b/litellm/tests/test_streaming.py @@ -6,7 +6,6 @@ import traceback import time, pytest from pydantic import BaseModel from typing import Tuple -from unittest.mock import patch, MagicMock sys.path.insert( 0, os.path.abspath("../..") From 2e5278990d7752238c4dfa9b03b78fd2a8fe5e43 Mon Sep 17 00:00:00 2001 From: Andres Barbaro Date: Fri, 24 May 2024 11:11:43 -0500 Subject: [PATCH 4/5] Improve test --- litellm/tests/test_streaming.py | 54 ++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/litellm/tests/test_streaming.py b/litellm/tests/test_streaming.py index f69dce9a3d..b39e1fc0ab 100644 --- a/litellm/tests/test_streaming.py +++ b/litellm/tests/test_streaming.py @@ -440,33 +440,45 @@ def test_completion_azure_stream_content_filter_no_delta(): } ] - stream_iterator = iter(chunks) + chunk_list = [] + for chunk in chunks: + new_chunk = litellm.ModelResponse(stream=True, id=chunk["id"]) + if "choices" in chunk and isinstance(chunk["choices"], list): + new_choices = [] + for choice in chunk["choices"]: + if isinstance(choice, litellm.utils.StreamingChoices): + _new_choice = choice + elif isinstance(choice, dict): + _new_choice = litellm.utils.StreamingChoices(**choice) + new_choices.append(_new_choice) + new_chunk.choices = new_choices + chunk_list.append(new_chunk) + + completion_stream = ModelResponseListIterator(model_responses=chunk_list) litellm.set_verbose = True - logging_obj = litellm.Logging( - model="berri-benchmarking-Llama-2-70b-chat-hf-4", - messages=messages, - stream=True, - litellm_call_id="1234", - function_id="function_id", - call_type="acompletion", - start_time=time.time(), - ) response = litellm.CustomStreamWrapper( - completion_stream=stream_iterator, + completion_stream=completion_stream, model="azure/gpt-4o", - custom_llm_provider="azure", - logging_obj=logging_obj, - ) - complete_response = "" + custom_llm_provider="cached_response", + messages=[{"role": "user", "content": "Hey"}], + stream=True, + call_type="completion", + start_time=time.time(), + litellm_call_id="12345", + function_id="1245", + ), + for idx, chunk in enumerate(response): - # print - chunk, finished = streaming_format_tests(idx, chunk) - complete_response += chunk - if finished: - break - assert len(complete_response) > 0 + complete_response = "" + for idx, chunk in enumerate(response): + # print + chunk, finished = streaming_format_tests(idx, chunk) + complete_response += chunk + if finished: + break + assert len(complete_response) > 0 except Exception as e: pytest.fail(f"An exception occurred - {str(e)}") From 8165dd6b5ab8deeadc65423a2c0495c42bb461e2 Mon Sep 17 00:00:00 2001 From: Andres Barbaro Date: Fri, 24 May 2024 11:48:35 -0500 Subject: [PATCH 5/5] Update test --- litellm/tests/test_streaming.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/litellm/tests/test_streaming.py b/litellm/tests/test_streaming.py index b39e1fc0ab..3b60896d48 100644 --- a/litellm/tests/test_streaming.py +++ b/litellm/tests/test_streaming.py @@ -460,23 +460,27 @@ def test_completion_azure_stream_content_filter_no_delta(): response = litellm.CustomStreamWrapper( completion_stream=completion_stream, - model="azure/gpt-4o", + model="gpt-4-0613", custom_llm_provider="cached_response", - messages=[{"role": "user", "content": "Hey"}], - stream=True, - call_type="completion", - start_time=time.time(), - litellm_call_id="12345", - function_id="1245", - ), + logging_obj=litellm.Logging( + model="gpt-4-0613", + messages=[{"role": "user", "content": "Hey"}], + stream=True, + call_type="completion", + start_time=time.time(), + litellm_call_id="12345", + function_id="1245", + ), + ) for idx, chunk in enumerate(response): complete_response = "" for idx, chunk in enumerate(response): # print - chunk, finished = streaming_format_tests(idx, chunk) - complete_response += chunk - if finished: + delta = chunk.choices[0].delta + content = delta.content if delta else None + complete_response += content or "" + if chunk.choices[0].finish_reason is not None: break assert len(complete_response) > 0