north bramble
How can I return a normal JSON response if it fails to stream a response?
This is my current code for streaming responses:
@litestar.post("/v1/chat/completions", guards=[user_checks])
async def chat(request: Request, data: NormalChatBody) -> dict | StreamingResponse:
origin = request.headers.get("
HTTP-Referer", "") or request.headers.get("Referer", "") or request.headers.get("Origin", "")
if (await moderation(data.messages, origin)):
raise HTTPException(detail={"error": {"message": "Profanity detected.", "type": "error", "param": None, "code": 403}}, status_code=403)
if data.stream and ("gpt" in data.model or data.model in ["claude-3-opus", "claude-3-sonnet"]):
try:
return StreamingResponse(content=litellm.streaming_litellm(data.model_dump()), media_type="text/event-stream", status_code=200)
except:
raise HTTPException(detail={"error": {"message": f"The provider for {data.model} sent an invalid response.", "type": "error", "param": None, "code": 500}}, status_code=500)
else:
content = await handle_response(data.model, data.messages, data.model_dump())
if not content:
content = await handle_fallback(data.messages)
if not content or content == "":
raise HTTPException(detail={"error": {"message": f"The provider for {data.model} sent an invalid response.", "type": "error", "param": None, "code": 500}}, status_code=500)
prompt_tokens, completion_tokens = await helpers.tokenize(''.join([str(message['content']) for message in data.messages])), await helpers.tokenize(content)
completion_id, completion_timestamp = helpers.generate_completion_id(), helpers.generate_timestamp()
if data.stream:
return StreamingResponse(content=streaming(content, completion_id, completion_timestamp, data.model), media_type="text/event-stream", status_code=200)
else:
return {
"id": f"chatcmpl-{completion_id}",
"object": "chat.completion",
"created": completion_timestamp,
"model": data.model,
"choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": prompt_tokens + completion_tokens}
}```
The "StreamingResponse" class is just the "Stream" class, I just put an alias to it.
fallow lance