jandres/GenAIExamples
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

move examples gateway (#992)

Browse Source 
Co-authored-by: root <root@idc708073.jf.intel.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Sihan Chen <39623753+Spycsh@users.noreply.github.com>
This commit is contained in:
lkk
2024-12-06 14:40:25 +08:00
committed by GitHub
parent f5c08d4fbb
commit bde285dfce
17 changed files with 1236 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
EdgeCraftRAG/chatqna.py
View File

@@ -5,7 +5,6 @@ import os
from comps import MicroService, ServiceOrchestrator, ServiceType
MEGA_SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "127.0.0.1")
MEGA_SERVICE_PORT = int(os.getenv("MEGA_SERVICE_PORT", 16011))
PIPELINE_SERVICE_HOST_IP = os.getenv("PIPELINE_SERVICE_HOST_IP", "127.0.0.1")
PIPELINE_SERVICE_PORT = int(os.getenv("PIPELINE_SERVICE_PORT", 16010))
@@ -23,11 +22,22 @@ from fastapi import Request
from fastapi.responses import StreamingResponse
class EdgeCraftRagGateway(Gateway):
def __init__(self, megaservice, host="0.0.0.0", port=16011):
super().__init__(
megaservice, host, port, str(MegaServiceEndpoint.CHAT_QNA), ChatCompletionRequest, ChatCompletionResponse
class EdgeCraftRagService(Gateway):
def __init__(self, host="0.0.0.0", port=16010):
self.host = host
self.port = port
self.megaservice = ServiceOrchestrator()
def add_remote_service(self):
edgecraftrag = MicroService(
name="pipeline",
host=PIPELINE_SERVICE_HOST_IP,
port=PIPELINE_SERVICE_PORT,
endpoint="/v1/chatqna",
use_remote_service=True,
service_type=ServiceType.LLM,
)
self.megaservice.add(edgecraftrag)
async def handle_request(self, request: Request):
input = await request.json()
@@ -61,26 +71,18 @@ class EdgeCraftRagGateway(Gateway):
)
return ChatCompletionResponse(model="edgecraftrag", choices=choices, usage=usage)
class EdgeCraftRagService:
def __init__(self, host="0.0.0.0", port=16010):
self.host = host
self.port = port
self.megaservice = ServiceOrchestrator()
def add_remote_service(self):
edgecraftrag = MicroService(
name="pipeline",
host=PIPELINE_SERVICE_HOST_IP,
port=PIPELINE_SERVICE_PORT,
endpoint="/v1/chatqna",
use_remote_service=True,
service_type=ServiceType.LLM,
def start(self):
super().__init__(
megaservice=self.megaservice,
host=self.host,
port=self.port,
endpoint=str(MegaServiceEndpoint.CHAT_QNA),
input_datatype=ChatCompletionRequest,
output_datatype=ChatCompletionResponse,
)
self.megaservice.add(edgecraftrag)
self.gateway = EdgeCraftRagGateway(megaservice=self.megaservice, host="0.0.0.0", port=self.port)
if __name__ == "__main__":
edgecraftrag = EdgeCraftRagService(host=MEGA_SERVICE_HOST_IP, port=MEGA_SERVICE_PORT)
edgecraftrag = EdgeCraftRagService(port=MEGA_SERVICE_PORT)
edgecraftrag.add_remote_service()
edgecraftrag.start()