fix vllm output in chatqna (#1038)

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>
This commit is contained in:
XinyaoWa
2024-11-01 09:26:57 +08:00
committed by GitHub
parent 9d124161e0
commit c65d7d40fb
3 changed files with 4 additions and 8 deletions

View File

@@ -166,7 +166,10 @@ def align_generator(self, gen, **kwargs):
try:
# sometimes yield empty chunk, do a fallback here
json_data = json.loads(json_str)
if json_data["choices"][0]["finish_reason"] != "eos_token":
if (
json_data["choices"][0]["finish_reason"] != "eos_token"
and "content" in json_data["choices"][0]["delta"]
):
yield f"data: {repr(json_data['choices'][0]['delta']['content'].encode('utf-8'))}\n\n"
except Exception as e:
yield f"data: {repr(json_str.encode('utf-8'))}\n\n"