ai-workspace-services/litellm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(predibase.py): fix async streaming

Browse Source
This commit is contained in:
Krrish Dholakia 2024-05-09 22:18:16 -07:00
parent 76d4290591
commit 714370956f
1 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
litellm/llms/predibase.py
View File

@ -454,10 +454,10 @@ class PredibaseChatCompletion(BaseLLM):
logger_fn=None,
headers={},
) -> ModelResponse:
async_handler = AsyncHTTPHandler(
self.async_handler = AsyncHTTPHandler(
timeout=httpx.Timeout(timeout=600.0, connect=5.0)
)
response = await async_handler.post(
response = await self.async_handler.post(
api_base, headers=headers, data=json.dumps(data)
)
return self.process_response(
@ -490,12 +490,12 @@ class PredibaseChatCompletion(BaseLLM):
logger_fn=None,
headers={},
) -> CustomStreamWrapper:
async_handler = AsyncHTTPHandler(
self.async_handler = AsyncHTTPHandler(
timeout=httpx.Timeout(timeout=600.0, connect=5.0)
)
data["stream"] = True
response = await async_handler.post(
url="https://serving.app.predibase.com/c4768f95/deployments/v2/llms/llama-3-8b-instruct/generate_stream",
response = await self.async_handler.post(
url=api_base,
headers=headers,
data=json.dumps(data),
stream=True,