# Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 import os from comps import ChatQnAGateway, MicroService, ServiceOrchestrator, ServiceType MEGA_SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0") MEGA_SERVICE_PORT = int(os.getenv("MEGA_SERVICE_PORT", 8888)) GUARDRAIL_SERVICE_HOST_IP = os.getenv("GUARDRAIL_SERVICE_HOST_IP", "0.0.0.0") GUARDRAIL_SERVICE_PORT = int(os.getenv("GUARDRAIL_SERVICE_PORT", 9090)) EMBEDDING_SERVICE_HOST_IP = os.getenv("EMBEDDING_SERVICE_HOST_IP", "0.0.0.0") EMBEDDING_SERVICE_PORT = int(os.getenv("EMBEDDING_SERVICE_PORT", 6000)) RETRIEVER_SERVICE_HOST_IP = os.getenv("RETRIEVER_SERVICE_HOST_IP", "0.0.0.0") RETRIEVER_SERVICE_PORT = int(os.getenv("RETRIEVER_SERVICE_PORT", 7000)) RERANK_SERVICE_HOST_IP = os.getenv("RERANK_SERVICE_HOST_IP", "0.0.0.0") RERANK_SERVICE_PORT = int(os.getenv("RERANK_SERVICE_PORT", 8000)) LLM_SERVICE_HOST_IP = os.getenv("LLM_SERVICE_HOST_IP", "0.0.0.0") LLM_SERVICE_PORT = int(os.getenv("LLM_SERVICE_PORT", 9000)) class ChatQnAService: def __init__(self, host="0.0.0.0", port=8000): self.host = host self.port = port self.megaservice = ServiceOrchestrator() def add_remote_service(self): guardrail_in = MicroService( name="guardrail_in", host=GUARDRAIL_SERVICE_HOST_IP, port=GUARDRAIL_SERVICE_PORT, endpoint="/v1/guardrails", use_remote_service=True, service_type=ServiceType.GUARDRAIL, ) embedding = MicroService( name="embedding", host=EMBEDDING_SERVICE_HOST_IP, port=EMBEDDING_SERVICE_PORT, endpoint="/v1/embeddings", use_remote_service=True, service_type=ServiceType.EMBEDDING, ) retriever = MicroService( name="retriever", host=RETRIEVER_SERVICE_HOST_IP, port=RETRIEVER_SERVICE_PORT, endpoint="/v1/retrieval", use_remote_service=True, service_type=ServiceType.RETRIEVER, ) rerank = MicroService( name="rerank", host=RERANK_SERVICE_HOST_IP, port=RERANK_SERVICE_PORT, endpoint="/v1/reranking", use_remote_service=True, service_type=ServiceType.RERANK, ) llm = MicroService( name="llm", host=LLM_SERVICE_HOST_IP, port=LLM_SERVICE_PORT, endpoint="/v1/chat/completions", use_remote_service=True, service_type=ServiceType.LLM, ) # guardrail_out = MicroService( # name="guardrail_out", # host=GUARDRAIL_SERVICE_HOST_IP, # port=GUARDRAIL_SERVICE_PORT, # endpoint="/v1/guardrails", # use_remote_service=True, # service_type=ServiceType.GUARDRAIL, # ) # self.megaservice.add(guardrail_in).add(embedding).add(retriever).add(rerank).add(llm).add(guardrail_out) self.megaservice.add(guardrail_in).add(embedding).add(retriever).add(rerank).add(llm) self.megaservice.flow_to(guardrail_in, embedding) self.megaservice.flow_to(embedding, retriever) self.megaservice.flow_to(retriever, rerank) self.megaservice.flow_to(rerank, llm) # self.megaservice.flow_to(llm, guardrail_out) self.gateway = ChatQnAGateway(megaservice=self.megaservice, host="0.0.0.0", port=self.port) if __name__ == "__main__": chatqna = ChatQnAService(host=MEGA_SERVICE_HOST_IP, port=MEGA_SERVICE_PORT) chatqna.add_remote_service()