Refactor example code (#183)

Signed-off-by: lvliang-intel <liang1.lv@intel.com>
Signed-off-by: Yue, Wenjiao <wenjiao.yue@intel.com>
Signed-off-by: chensuyue <suyue.chen@intel.com>

.github/workflows/ChatQnA_gaudi.yml

@@ -8,7 +8,7 @@ on:
     branches: [main]
     types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
     paths:
-      - ChatQnA/microservice/gaudi/**
+      - ChatQnA/docker-composer/gaudi/**
       - ChatQnA/tests/test_chatqna_on_gaudi.sh
       - "!**.md"
       - "!**/ui/**"
@@ -43,7 +43,7 @@ jobs:
       - name: Clean Up Container
         if: cancelled() || failure()
         run: |
-          cd ${{ github.workspace }}/ChatQnA/microservice/gaudi
+          cd ${{ github.workspace }}/ChatQnA/docker-composer/gaudi
           container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
           for container_name in $container_list; do
               cid=$(docker ps -aq --filter "name=$container_name")

.github/workflows/ChatQnA_xeon.yml

@@ -8,7 +8,7 @@ on:
     branches: [main]
     types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
     paths:
-      - ChatQnA/microservice/xeon/**
+      - ChatQnA/docker-composer/xeon/**
       - ChatQnA/tests/test_chatqna_on_xeon.sh
       - "!**.md"
       - "!**/ui/**"
@@ -43,7 +43,7 @@ jobs:
       - name: Clean Up Container
         if: cancelled() || failure()
         run: |
-          cd ${{ github.workspace }}/ChatQnA/microservice/xeon
+          cd ${{ github.workspace }}/ChatQnA/docker-composer/xeon
           container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
           for container_name in $container_list; do
               cid=$(docker ps -aq --filter "name=$container_name")

.github/workflows/CodeGen_gaudi.yml

@@ -8,7 +8,7 @@ on:
     branches: [main]
     types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
     paths:
-      - CodeGen/microservice/gaudi/**
+      - CodeGen/docker-composer/gaudi/**
       - CodeGen/tests/test_codegen_on_gaudi.sh
       - "!**.md"
       - "!**/ui/**"
@@ -43,7 +43,7 @@ jobs:
       - name: Clean Up Container
         if: cancelled() || failure()
         run: |
-          cd ${{ github.workspace }}/CodeGen/microservice/gaudi
+          cd ${{ github.workspace }}/CodeGen/docker-composer/gaudi
           container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
           for container_name in $container_list; do
               cid=$(docker ps -aq --filter "name=$container_name")

.github/workflows/CodeGen_xeon.yml

@@ -8,7 +8,7 @@ on:
     branches: [main]
     types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
     paths:
-      - CodeGen/microservice/xeon/**
+      - CodeGen/docker-composer/xeon/**
       - CodeGen/tests/test_codegen_on_xeon.sh
       - "!**.md"
       - "!**/ui/**"
@@ -43,7 +43,7 @@ jobs:
       - name: Clean Up Container
         if: cancelled() || failure()
         run: |
-          cd ${{ github.workspace }}/CodeGen/microservice/xeon
+          cd ${{ github.workspace }}/CodeGen/docker-composer/xeon
           container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
           for container_name in $container_list; do
               cid=$(docker ps -aq --filter "name=$container_name")

.github/workflows/CodeTrans_gaudi.yml

@@ -8,7 +8,7 @@ on:
     branches: [main]
     types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
     paths:
-      - CodeTrans/microservice/gaudi/**
+      - CodeTrans/docker-composer/gaudi/**
       - CodeTrans/tests/test_codetrans_on_gaudi.sh
       - "!**.md"
       - "!**/ui/**"
@@ -43,7 +43,7 @@ jobs:
       - name: Clean Up Container
         if: cancelled() || failure()
         run: |
-          cd ${{ github.workspace }}/CodeTrans/microservice/gaudi
+          cd ${{ github.workspace }}/CodeTrans/docker-composer/gaudi
           container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
           for container_name in $container_list; do
               cid=$(docker ps -aq --filter "name=$container_name")

.github/workflows/CodeTrans_xeon.yml

@@ -8,7 +8,7 @@ on:
     branches: [main]
     types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
     paths:
-      - CodeTrans/microservice/xeon/**
+      - CodeTrans/docker-composer/xeon/**
       - CodeTrans/tests/test_codetrans_on_xeon.sh
       - "!**.md"
       - "!**/ui/**"
@@ -43,7 +43,7 @@ jobs:
       - name: Clean Up Container
         if: cancelled() || failure()
         run: |
-          cd ${{ github.workspace }}/CodeTrans/microservice/xeon
+          cd ${{ github.workspace }}/CodeTrans/docker-composer/xeon
           container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
           for container_name in $container_list; do
               cid=$(docker ps -aq --filter "name=$container_name")

.github/workflows/DocSum_gaudi.yml

@@ -8,7 +8,7 @@ on:
     branches: [main]
     types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
     paths:
-      - DocSum/microservice/gaudi/**
+      - DocSum/docker-composer/gaudi/**
       - DocSum/tests/test_docsum_on_gaudi.sh
       - "!**.md"
       - "!**/ui/**"
@@ -43,7 +43,7 @@ jobs:
       - name: Clean Up Container
         if: cancelled() || failure()
         run: |
-          cd ${{ github.workspace }}/DocSum/microservice/gaudi
+          cd ${{ github.workspace }}/DocSum/docker-composer/gaudi
           container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
           for container_name in $container_list; do
               cid=$(docker ps -aq --filter "name=$container_name")

.github/workflows/DocSum_xeon.yml

@@ -8,7 +8,7 @@ on:
     branches: [main]
     types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
     paths:
-      - DocSum/microservice/xeon/**
+      - DocSum/docker-composer/xeon/**
       - DocSum/tests/test_docsum_on_xeon.sh
       - "!**.md"
       - "!**/ui/**"
@@ -43,7 +43,7 @@ jobs:
       - name: Clean Up Container
         if: cancelled() || failure()
         run: |
-          cd ${{ github.workspace }}/DocSum/microservice/xeon
+          cd ${{ github.workspace }}/DocSum/docker-composer/xeon
           container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
           for container_name in $container_list; do
               cid=$(docker ps -aq --filter "name=$container_name")

ChatQnA/Dockerfile

@@ -31,7 +31,7 @@ RUN cd /home/user/ && \
 RUN cd /home/user/GenAIComps && pip install --no-cache-dir --upgrade pip && \
     pip install -r /home/user/GenAIComps/requirements.txt
 
-COPY ../chatqna.py /home/user/chatqna.py
+COPY ./chatqna.py /home/user/chatqna.py
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps
 

ChatQnA/README.md

@@ -16,8 +16,8 @@ The ChatQnA service can be effortlessly deployed on either Intel Gaudi2 or Intel
 
 ## Deploy ChatQnA on Gaudi
 
-Refer to the [Gaudi Guide](./microservice/gaudi/README.md) for instructions on deploying ChatQnA on Gaudi.
+Refer to the [Gaudi Guide](./docker-composer/gaudi/README.md) for instructions on deploying ChatQnA on Gaudi.
 
 ## Deploy ChatQnA on Xeon
 
-Refer to the [Xeon Guide](./microservice/xeon/README.md) for instructions on deploying ChatQnA on Xeon.
+Refer to the [Xeon Guide](./docker-composer/xeon/README.md) for instructions on deploying ChatQnA on Xeon.

ChatQnA/chatqna.py

@@ -17,43 +17,53 @@ import os
 
 from comps import ChatQnAGateway, MicroService, ServiceOrchestrator, ServiceType
 
-SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0")
+MEGA_SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0")
+MEGA_SERVICE_PORT = os.getenv("MEGA_SERVICE_PORT", 8888)
+EMBEDDING_SERVICE_HOST_IP = os.getenv("EMBEDDING_SERVICE_HOST_IP", "0.0.0.0")
+EMBEDDING_SERVICE_PORT = os.getenv("EMBEDDING_SERVICE_PORT", 6000)
+RETRIEVER_SERVICE_HOST_IP = os.getenv("RETRIEVER_SERVICE_HOST_IP", "0.0.0.0")
+RETRIEVER_SERVICE_PORT = os.getenv("RETRIEVER_SERVICE_PORT", 7000)
+RERANK_SERVICE_HOST_IP = os.getenv("RERANK_SERVICE_HOST_IP", "0.0.0.0")
+RERANK_SERVICE_PORT = os.getenv("RERANK_SERVICE_PORT", 8000)
+LLM_SERVICE_HOST_IP = os.getenv("LLM_SERVICE_HOST_IP", "0.0.0.0")
+LLM_SERVICE_PORT = os.getenv("LLM_SERVICE_PORT", 9000)
 
 
 class ChatQnAService:
-    def __init__(self, port=8000):
+    def __init__(self, host="0.0.0.0", port=8000):
+        self.host = host
         self.port = port
         self.megaservice = ServiceOrchestrator()
 
     def add_remote_service(self):
         embedding = MicroService(
             name="embedding",
-            host=SERVICE_HOST_IP,
+            host=EMBEDDING_SERVICE_HOST_IP,
-            port=6000,
+            port=EMBEDDING_SERVICE_PORT,
             endpoint="/v1/embeddings",
             use_remote_service=True,
             service_type=ServiceType.EMBEDDING,
         )
         retriever = MicroService(
             name="retriever",
-            host=SERVICE_HOST_IP,
+            host=RETRIEVER_SERVICE_HOST_IP,
-            port=7000,
+            port=RETRIEVER_SERVICE_PORT,
             endpoint="/v1/retrieval",
             use_remote_service=True,
             service_type=ServiceType.RETRIEVER,
         )
         rerank = MicroService(
             name="rerank",
-            host=SERVICE_HOST_IP,
+            host=RERANK_SERVICE_HOST_IP,
-            port=8000,
+            port=RERANK_SERVICE_PORT,
             endpoint="/v1/reranking",
             use_remote_service=True,
             service_type=ServiceType.RERANK,
         )
         llm = MicroService(
             name="llm",
-            host=SERVICE_HOST_IP,
+            host=LLM_SERVICE_HOST_IP,
-            port=9000,
+            port=LLM_SERVICE_PORT,
             endpoint="/v1/chat/completions",
             use_remote_service=True,
             service_type=ServiceType.LLM,
@@ -71,6 +81,6 @@ class ChatQnAService:
 
 
 if __name__ == "__main__":
-    chatqna = ChatQnAService(port=8888)
+    chatqna = ChatQnAService(host=MEGA_SERVICE_HOST_IP, port=MEGA_SERVICE_PORT)
     chatqna.add_remote_service()
     asyncio.run(chatqna.schedule())

ChatQnA/chatqna.yaml

@@ -0,0 +1,92 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+opea_micro_services:
+  redis-vector-db:
+    host: ${REDIS_SERVICE_HOST_IP}
+    ports:
+      - "6379:6379"
+      - "8001:8001"
+    image: redis/redis-stack:7.2.0-v9
+  dataprep-redis-service:
+    image: opea/gen-ai-comps:dataprep-redis-server
+    host: ${DATAPREP_SERVICE_HOST_IP}
+    ports: ${DATAPREP_SERVICE_PORT}
+    environment:
+      REDIS_URL: ${REDIS_URL}
+      INDEX_NAME: ${INDEX_NAME}
+  tei-embedding-service:
+    host: ${TEI_EMBEDDING_SERVICE_IP}
+    ports: ${TEI_EMBEDDING_SERVICE_PORT}
+    image: opea/tei-gaudi
+    volumes:
+      - "./data:/data"
+    runtime: habana
+    cap_add:
+      - SYS_NICE
+    ipc: host
+    environment:
+      HABANA_VISIBLE_DEVICES: all
+      OMPI_MCA_btl_vader_single_copy_mechanism: none
+    model-id: ${EMBEDDING_MODEL_ID}
+  embedding:
+    host: ${EMBEDDING_SERVICE_HOST_IP}
+    ports: ${EMBEDDING_SERVICE_PORT}
+    endpoint: /v1/embeddings
+    image: opea/gen-ai-comps:embedding-tei-server
+  retrieval:
+    host: ${RETRIEVER_SERVICE_HOST_IP}
+    ports: ${RETRIEVER_SERVICE_PORT}
+    endpoint: /v1/retrieval
+    image: opea/gen-ai-comps:retriever-redis-server
+  reranking:
+    host: ${RERANK_SERVICE_HOST_IP}
+    ports: ${RERANK_SERVICE_PORT}
+    image: opea/gen-ai-comps:reranking-tei-server
+    endpoint: /v1/reranking
+  tgi_service:
+    host: ${TGI_SERVICE_IP}
+    ports: ${TGI_SERVICE_PORT}
+    image: ghcr.io/huggingface/tgi-gaudi:1.2.1
+    volumes:
+      - "./data:/data"
+    runtime: habana
+    cap_add:
+      - SYS_NICE
+    ipc: host
+    environment:
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      HABANA_VISIBLE_DEVICES: all
+      OMPI_MCA_btl_vader_single_copy_mechanism: none
+    model-id: ${LLM_MODEL_ID}
+  llm:
+    host: ${LLM_SERVICE_HOST_IP}
+    ports: ${LLM_SERVICE_PORT}
+    image: opea/gen-ai-comps:llm-tgi-gaudi-server
+    endpoint: /v1/chat/completions
+  ui:
+    host: ${UI_SERVICE_HOST_IP}
+    ports:
+      - "5173:5173"
+    environment:
+      - CHAT_BASE_URL=${BACKEND_SERVICE_ENDPOINT}
+    image: opea/gen-ai-comps:chatqna-redis-server
+
+opea_mega_service:
+  host: ${MEGA_SERVICE_HOST_IP}
+  ports: ${MEGA_SERVICE_PORT}
+  endpoint: /v1/chatqna
+  image: opea/gen-ai-comps:chatqna-megaservice-server
+  mega_flow:
+    - embedding >> retrieval >> reranking >> llm

ChatQnA/docker-composer/gaudi/README.md

@@ -34,7 +34,7 @@ docker build --no-cache -t opea/gen-ai-comps:reranking-tei-server --build-arg ht
 ### 5. Build LLM Image
 
 ```bash
-docker build --no-cache -t opea/gen-ai-comps:llm-tgi-gaudi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/langchain/docker/Dockerfile .
+docker build --no-cache -t opea/gen-ai-comps:llm-tgi-gaudi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
 ```
 
 ### 6. Build Dataprep Image
@@ -59,8 +59,8 @@ To construct the Mega Service, we utilize the [GenAIComps](https://github.com/op
 
 ```bash
 git clone https://github.com/opea-project/GenAIExamples.git
-cd GenAIExamples/ChatQnA/microservice/gaudi/
+cd GenAIExamples/ChatQnA
-docker build --no-cache -t opea/gen-ai-comps:chatqna-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
+docker build --no-cache -t opea/gen-ai-comps:chatqna-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
 ```
 
 ### 9. Build UI Docker Image
@@ -102,6 +102,10 @@ export REDIS_URL="redis://${host_ip}:6379"
 export INDEX_NAME="rag-redis"
 export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
 export MEGA_SERVICE_HOST_IP=${host_ip}
+export EMBEDDING_SERVICE_HOST_IP=${host_ip}
+export RETRIEVER_SERVICE_HOST_IP=${host_ip}
+export RERANK_SERVICE_HOST_IP=${host_ip}
+export LLM_SERVICE_HOST_IP=${host_ip}
 export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/chatqna"
 ```
 
@@ -110,6 +114,7 @@ Note: Please replace with `host_ip` with you external IP address, do not use loc
 ### Start all the services Docker Containers
 
 ```bash
+cd GenAIExamples/ChatQnA/docker-composer/gaudi/
 docker compose -f docker_compose.yaml up -d
 ```
 

ChatQnA/docker-composer/gaudi/docker_compose.yaml

@@ -146,6 +146,10 @@ services:
       - https_proxy=${https_proxy}
       - http_proxy=${http_proxy}
       - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP}
+      - EMBEDDING_SERVICE_HOST_IP=${EMBEDDING_SERVICE_HOST_IP}
+      - RETRIEVER_SERVICE_HOST_IP=${RETRIEVER_SERVICE_HOST_IP}
+      - RERANK_SERVICE_HOST_IP=${RERANK_SERVICE_HOST_IP}
+      - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP}
     ipc: host
     restart: always
   chaqna-gaudi-ui-server:

ChatQnA/docker-composer/xeon/README.md

@@ -22,25 +22,25 @@ cd GenAIComps
 ### 1. Build Embedding Image
 
 ```bash
-docker build -t opea/gen-ai-comps:embedding-tei-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/langchain/docker/Dockerfile .
+docker build --no-cache -t opea/gen-ai-comps:embedding-tei-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/langchain/docker/Dockerfile .
 ```
 
 ### 2. Build Retriever Image
 
 ```bash
-docker build -t opea/gen-ai-comps:retriever-redis-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/langchain/docker/Dockerfile .
+docker build --no-cache -t opea/gen-ai-comps:retriever-redis-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/langchain/docker/Dockerfile .
 ```
 
 ### 3. Build Rerank Image
 
 ```bash
-docker build -t opea/gen-ai-comps:reranking-tei-xeon-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/langchain/docker/Dockerfile .
+docker build --no-cache -t opea/gen-ai-comps:reranking-tei-xeon-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/langchain/docker/Dockerfile .
 ```
 
 ### 4. Build LLM Image
 
 ```bash
-docker build -t opea/gen-ai-comps:llm-tgi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/langchain/docker/Dockerfile .
+docker build --no-cache -t opea/gen-ai-comps:llm-tgi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
 ```
 
 ### 5. Build Dataprep Image
@@ -55,8 +55,8 @@ To construct the Mega Service, we utilize the [GenAIComps](https://github.com/op
 
 ```bash
 git clone https://github.com/opea-project/GenAIExamples.git
-cd GenAIExamples/ChatQnA/microservice/xeon/
+cd GenAIExamples/ChatQnA
-docker build -t opea/gen-ai-comps:chatqna-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
+docker build --no-cache -t opea/gen-ai-comps:chatqna-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
 ```
 
 ### 7. Build UI Docker Image
@@ -65,7 +65,7 @@ Build frontend Docker image via below command:
 
 ```bash
 cd GenAIExamples/ChatQnA/ui/
-docker build -t opea/gen-ai-comps:chatqna-ui-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile .
+docker build --no-cache -t opea/gen-ai-comps:chatqna-ui-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile .
 ```
 
 Then run the command `docker images`, you will have the following four Docker Images:
@@ -96,6 +96,10 @@ export REDIS_URL="redis://${host_ip}:6379"
 export INDEX_NAME="rag-redis"
 export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
 export MEGA_SERVICE_HOST_IP=${host_ip}
+export EMBEDDING_SERVICE_HOST_IP=${host_ip}
+export RETRIEVER_SERVICE_HOST_IP=${host_ip}
+export RERANK_SERVICE_HOST_IP=${host_ip}
+export LLM_SERVICE_HOST_IP=${host_ip}
 export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/chatqna"
 ```
 
@@ -104,6 +108,7 @@ Note: Please replace with `host_ip` with you external IP address, do not use loc
 ### Start all the services Docker Containers
 
 ```bash
+cd GenAIExamples/ChatQnA/docker-composer/xeon/
 docker compose -f docker_compose.yaml up -d
 ```
 

ChatQnA/docker-composer/xeon/docker_compose.yaml

@@ -143,6 +143,10 @@ services:
       - https_proxy=${https_proxy}
       - http_proxy=${http_proxy}
       - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP}
+      - EMBEDDING_SERVICE_HOST_IP=${EMBEDDING_SERVICE_HOST_IP}
+      - RETRIEVER_SERVICE_HOST_IP=${RETRIEVER_SERVICE_HOST_IP}
+      - RERANK_SERVICE_HOST_IP=${RERANK_SERVICE_HOST_IP}
+      - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP}
     ipc: host
     restart: always
   chaqna-xeon-ui-server:

ChatQnA/microservice/gaudi/chatqna.py

@@ -1,76 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import asyncio
-import os
-
-from comps import ChatQnAGateway, MicroService, ServiceOrchestrator, ServiceType
-
-SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0")
-
-
-class ChatQnAService:
-    def __init__(self, port=8000):
-        self.port = port
-        self.megaservice = ServiceOrchestrator()
-
-    def add_remote_service(self):
-        embedding = MicroService(
-            name="embedding",
-            host=SERVICE_HOST_IP,
-            port=6000,
-            endpoint="/v1/embeddings",
-            use_remote_service=True,
-            service_type=ServiceType.EMBEDDING,
-        )
-        retriever = MicroService(
-            name="retriever",
-            host=SERVICE_HOST_IP,
-            port=7000,
-            endpoint="/v1/retrieval",
-            use_remote_service=True,
-            service_type=ServiceType.RETRIEVER,
-        )
-        rerank = MicroService(
-            name="rerank",
-            host=SERVICE_HOST_IP,
-            port=8000,
-            endpoint="/v1/reranking",
-            use_remote_service=True,
-            service_type=ServiceType.RERANK,
-        )
-        llm = MicroService(
-            name="llm",
-            host=SERVICE_HOST_IP,
-            port=9000,
-            endpoint="/v1/chat/completions",
-            use_remote_service=True,
-            service_type=ServiceType.LLM,
-        )
-        self.megaservice.add(embedding).add(retriever).add(rerank).add(llm)
-        self.megaservice.flow_to(embedding, retriever)
-        self.megaservice.flow_to(retriever, rerank)
-        self.megaservice.flow_to(rerank, llm)
-        self.gateway = ChatQnAGateway(megaservice=self.megaservice, host="0.0.0.0", port=self.port)
-
-    async def schedule(self):
-        await self.megaservice.schedule(initial_inputs={"text": "What is the revenue of Nike in 2023?"})
-        result_dict = self.megaservice.result_dict
-        print(result_dict)
-
-
-if __name__ == "__main__":
-    chatqna = ChatQnAService(port=8888)
-    chatqna.add_remote_service()
-    asyncio.run(chatqna.schedule())

ChatQnA/microservice/gaudi/docker/Dockerfile

@@ -1,42 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-FROM python:3.11-slim
-
-RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
-    libgl1-mesa-glx \
-    libjemalloc-dev \
-    vim \
-    git
-
-RUN useradd -m -s /bin/bash user && \
-    mkdir -p /home/user && \
-    chown -R user /home/user/
-
-RUN cd /home/user/ && \
-    git clone https://github.com/opea-project/GenAIComps.git
-
-RUN cd /home/user/GenAIComps && pip install --no-cache-dir --upgrade pip && \
-    pip install -r /home/user/GenAIComps/requirements.txt
-
-COPY ../chatqna.py /home/user/chatqna.py
-
-ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps
-
-USER user
-
-WORKDIR /home/user
-
-ENTRYPOINT ["python", "chatqna.py"]

ChatQnA/tests/test_chatqna_on_gaudi.sh

@@ -16,7 +16,7 @@ function build_docker_images() {
     docker build -t opea/gen-ai-comps:embedding-tei-server -f comps/embeddings/langchain/docker/Dockerfile .
     docker build -t opea/gen-ai-comps:retriever-redis-server -f comps/retrievers/langchain/docker/Dockerfile .
     docker build -t opea/gen-ai-comps:reranking-tei-server -f comps/reranks/langchain/docker/Dockerfile .
-    docker build -t opea/gen-ai-comps:llm-tgi-gaudi-server -f comps/llms/langchain/docker/Dockerfile .
+    docker build -t opea/gen-ai-comps:llm-tgi-gaudi-server -f comps/llms/text-generation/tgi/Dockerfile .
     docker build -t opea/gen-ai-comps:dataprep-redis-server -f comps/dataprep/redis/docker/Dockerfile .
 
     cd ..
@@ -27,8 +27,8 @@ function build_docker_images() {
     docker pull ghcr.io/huggingface/tgi-gaudi:1.2.1
     docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.2
 
-    cd $WORKPATH/microservice/gaudi
+    cd $WORKPATH
-    docker build --no-cache -t opea/gen-ai-comps:chatqna-megaservice-server -f docker/Dockerfile .
+    docker build --no-cache -t opea/gen-ai-comps:chatqna-megaservice-server -f Dockerfile .
 
     cd $WORKPATH/ui
     docker build --no-cache -t opea/gen-ai-comps:chatqna-ui-server -f docker/Dockerfile .
@@ -37,7 +37,7 @@ function build_docker_images() {
 }
 
 function start_services() {
-    cd $WORKPATH/microservice/gaudi
+    cd $WORKPATH/docker-composer/gaudi
 
     export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
     export RERANK_MODEL_ID="BAAI/bge-reranker-large"
@@ -49,6 +49,10 @@ function start_services() {
     export INDEX_NAME="rag-redis"
     export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
     export MEGA_SERVICE_HOST_IP=${ip_address}
+    export EMBEDDING_SERVICE_HOST_IP=${ip_address}
+    export RETRIEVER_SERVICE_HOST_IP=${ip_address}
+    export RERANK_SERVICE_HOST_IP=${ip_address}
+    export LLM_SERVICE_HOST_IP=${ip_address}
     export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/chatqna"
 
     # Start Docker Containers
@@ -180,7 +184,7 @@ function validate_megaservice() {
 }
 
 function stop_docker() {
-    cd $WORKPATH/microservice/gaudi
+    cd $WORKPATH/docker-composer/gaudi
     container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
     for container_name in $container_list; do
         cid=$(docker ps -aq --filter "name=$container_name")

ChatQnA/tests/test_chatqna_on_xeon.sh

@@ -2,7 +2,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-set -xe
+set -x
 
 WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
@@ -17,11 +17,11 @@ function build_docker_images() {
     docker build -t opea/gen-ai-comps:embedding-tei-server -f comps/embeddings/langchain/docker/Dockerfile .
     docker build -t opea/gen-ai-comps:retriever-redis-server -f comps/retrievers/langchain/docker/Dockerfile .
     docker build -t opea/gen-ai-comps:reranking-tei-xeon-server -f comps/reranks/langchain/docker/Dockerfile .
-    docker build -t opea/gen-ai-comps:llm-tgi-server -f comps/llms/langchain/docker/Dockerfile .
+    docker build -t opea/gen-ai-comps:llm-tgi-server -f comps/llms/text-generation/tgi/Dockerfile .
     docker build -t opea/gen-ai-comps:dataprep-redis-server -f comps/dataprep/redis/docker/Dockerfile .
 
-    cd $WORKPATH/microservice/xeon
+    cd $WORKPATH
-    docker build --no-cache -t opea/gen-ai-comps:chatqna-megaservice-server -f docker/Dockerfile .
+    docker build --no-cache -t opea/gen-ai-comps:chatqna-megaservice-server -f Dockerfile .
 
     cd $WORKPATH/ui
     docker build --no-cache -t opea/gen-ai-comps:chatqna-ui-server -f docker/Dockerfile .
@@ -30,7 +30,7 @@ function build_docker_images() {
 }
 
 function start_services() {
-    cd $WORKPATH/microservice/xeon
+    cd $WORKPATH/docker-composer/xeon
 
     export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
     export RERANK_MODEL_ID="BAAI/bge-reranker-large"
@@ -42,13 +42,17 @@ function start_services() {
     export INDEX_NAME="rag-redis"
     export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
     export MEGA_SERVICE_HOST_IP=${ip_address}
+    export EMBEDDING_SERVICE_HOST_IP=${ip_address}
+    export RETRIEVER_SERVICE_HOST_IP=${ip_address}
+    export RERANK_SERVICE_HOST_IP=${ip_address}
+    export LLM_SERVICE_HOST_IP=${ip_address}
     export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/chatqna"
 
     # Start Docker Containers
     # TODO: Replace the container name with a test-specific name
     docker compose -f docker_compose.yaml up -d
 
-    sleep 1m # Waits 1 minutes
+    sleep 2m # Waits 1 minutes
 }
 
 function validate_microservices() {
@@ -171,7 +175,7 @@ function validate_megaservice() {
 }
 
 function stop_docker() {
-    cd $WORKPATH/microservice/xeon
+    cd $WORKPATH/docker-composer/xeon
     container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
     for container_name in $container_list; do
         cid=$(docker ps -aq --filter "name=$container_name")

CodeGen/Dockerfile

@@ -33,7 +33,7 @@ RUN cd /home/user/ && \
 RUN cd /home/user/GenAIComps && pip install --no-cache-dir --upgrade pip && \
     pip install -r /home/user/GenAIComps/requirements.txt
 
-COPY ../codegen.py /home/user/codegen.py
+COPY ./codegen.py /home/user/codegen.py
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps
 

CodeGen/README.md

@@ -24,8 +24,8 @@ The CodeGen service can be effortlessly deployed on either Intel Gaudi2 or Intel
 
 ## Deploy CodeGen on Gaudi
 
-Refer to the [Gaudi Guide](./microservice/gaudi/README.md) for instructions on deploying CodeGen on Gaudi.
+Refer to the [Gaudi Guide](./docker-composer/gaudi/README.md) for instructions on deploying CodeGen on Gaudi.
 
 ## Deploy CodeGen on Xeon
 
-Refer to the [Xeon Guide](./microservice/xeon/README.md) for instructions on deploying CodeGen on Xeon.
+Refer to the [Xeon Guide](./docker-composer/xeon/README.md) for instructions on deploying CodeGen on Xeon.

CodeGen/codegen.py

@@ -17,19 +17,23 @@ import os
 
 from comps import CodeGenGateway, MicroService, ServiceOrchestrator, ServiceType
 
-SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0")
+MEGA_SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0")
+MEGA_SERVICE_PORT = os.getenv("MEGA_SERVICE_PORT", 6666)
+LLM_SERVICE_HOST_IP = os.getenv("LLM_SERVICE_HOST_IP", "0.0.0.0")
+LLM_SERVICE_PORT = os.getenv("LLM_SERVICE_PORT", 9000)
 
 
-class ChatQnAService:
+class CodeGenService:
-    def __init__(self, port=8000):
+    def __init__(self, host="0.0.0.0", port=8000):
+        self.host = host
         self.port = port
         self.megaservice = ServiceOrchestrator()
 
     def add_remote_service(self):
         llm = MicroService(
             name="llm",
-            host=SERVICE_HOST_IP,
+            host=LLM_SERVICE_HOST_IP,
-            port=9000,
+            port=LLM_SERVICE_PORT,
             endpoint="/v1/chat/completions",
             use_remote_service=True,
             service_type=ServiceType.LLM,
@@ -46,6 +50,6 @@ class ChatQnAService:
 
 
 if __name__ == "__main__":
-    chatqna = ChatQnAService(port=6666)
+    chatqna = CodeGenService(host=MEGA_SERVICE_HOST_IP, port=MEGA_SERVICE_PORT)
     chatqna.add_remote_service()
     asyncio.run(chatqna.schedule())

CodeGen/codegen.yaml

@@ -0,0 +1,53 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+opea_micro_services:
+  tgi_service:
+    host: ${TGI_SERVICE_IP}
+    ports: ${TGI_SERVICE_PORT}
+    image: ghcr.io/huggingface/tgi-gaudi:1.2.1
+    volumes:
+      - "./data:/data"
+    runtime: habana
+    cap_add:
+      - SYS_NICE
+    ipc: host
+    environment:
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      HABANA_VISIBLE_DEVICES: all
+      OMPI_MCA_btl_vader_single_copy_mechanism: none
+    model-id: ${LLM_MODEL_ID}
+  llm:
+    host: ${LLM_SERVICE_HOST_IP}
+    ports: ${LLM_SERVICE_PORT}
+    image: opea/gen-ai-comps:llm-tgi-gaudi-server
+    endpoint: /v1/chat/completions
+    environment:
+      TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+  ui:
+    host: ${UI_SERVICE_HOST_IP}
+    ports:
+      - "5173:5173"
+    environment:
+      - CHAT_BASE_URL=${BACKEND_SERVICE_ENDPOINT}
+    image: opea/gen-ai-comps:codegen-ui-server
+
+opea_mega_service:
+  host: ${MEGA_SERVICE_HOST_IP}
+  ports: ${MEGA_SERVICE_PORT}
+  endpoint: /v1/codegen
+  image: opea/gen-ai-comps:codegen-megaservice-server
+  mega_flow:
+    - llm

CodeGen/docker-composer/gaudi/README.md

@@ -16,7 +16,7 @@ cd GenAIComps
 ### 2. Build LLM Image
 
 ```bash
-docker build -t opea/gen-ai-comps:llm-tgi-gaudi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/langchain/docker/Dockerfile .
+docker build -t opea/gen-ai-comps:llm-tgi-gaudi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
 ```
 
 ### 3. Build MegaService Docker Image
@@ -25,8 +25,8 @@ To construct the Mega Service, we utilize the [GenAIComps](https://github.com/op
 
 ```bash
 git clone https://github.com/opea-project/GenAIExamples
-cd GenAIExamples/CodeGen/microservice/gaudi/
+cd GenAIExamples/CodeGen
-docker build -t opea/gen-ai-comps:codegen-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
+docker build -t opea/gen-ai-comps:codegen-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
 ```
 
 ### 4. Build UI Docker Image
@@ -57,6 +57,7 @@ export LLM_MODEL_ID="ise-uiuc/Magicoder-S-DS-6.7B"
 export TGI_LLM_ENDPOINT="http://${host_ip}:8028"
 export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
 export MEGA_SERVICE_HOST_IP=${host_ip}
+export LLM_SERVICE_HOST_IP=${host_ip}
 export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:6666/v1/codegen"
 ```
 
@@ -65,6 +66,7 @@ Note: Please replace with `host_ip` with you external IP address, do not use loc
 ### Start all the services Docker Containers
 
 ```bash
+cd GenAIExamples/CodeGen/docker-composer/gaudi
 docker compose -f docker_compose.yaml up -d
 ```
 

CodeGen/docker-composer/gaudi/docker_compose.yaml

@@ -55,6 +55,7 @@ services:
       - https_proxy=${https_proxy}
       - http_proxy=${http_proxy}
       - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP}
+      - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP}
     ipc: host
     restart: always
   codegen-gaudi-ui-server:

CodeGen/docker-composer/xeon/README.md

@@ -22,7 +22,7 @@ cd GenAIComps
 ### 1. Build LLM Image
 
 ```bash
-docker build -t opea/gen-ai-comps:llm-tgi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/langchain/docker/Dockerfile .
+docker build -t opea/gen-ai-comps:llm-tgi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
 ```
 
 ### 2. Build MegaService Docker Image
@@ -31,8 +31,8 @@ To construct the Mega Service, we utilize the [GenAIComps](https://github.com/op
 
 ```bash
 git clone https://github.com/opea-project/GenAIExamples
-cd GenAIExamples/CodeGen/microservice/xeon/
+cd GenAIExamples/CodeGen
-docker build -t opea/gen-ai-comps:codegen-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
+docker build -t opea/gen-ai-comps:codegen-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
 ```
 
 ### 6. Build UI Docker Image
@@ -63,6 +63,7 @@ export LLM_MODEL_ID="ise-uiuc/Magicoder-S-DS-6.7B"
 export TGI_LLM_ENDPOINT="http://${host_ip}:8028"
 export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
 export MEGA_SERVICE_HOST_IP=${host_ip}
+export LLM_SERVICE_HOST_IP=${host_ip}
 export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:6666/v1/codegen"
 ```
 
@@ -71,6 +72,7 @@ Note: Please replace with `host_ip` with you external IP address, do not use loc
 ### Start all the services Docker Containers
 
 ```bash
+cd GenAIExamples/CodeGen/docker-composer/xeon
 docker compose -f docker_compose.yaml up -d
 ```
 

CodeGen/docker-composer/xeon/docker_compose.yaml

@@ -53,6 +53,7 @@ services:
       - https_proxy=${https_proxy}
       - http_proxy=${http_proxy}
       - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP}
+      - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP}
     ipc: host
     restart: always
   codegen-xeon-ui-server:

CodeGen/microservice/gaudi/docker/Dockerfile

@@ -1,44 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-FROM python:3.11-slim
-
-ENV LANG C.UTF-8
-
-RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
-    libgl1-mesa-glx \
-    libjemalloc-dev \
-    vim \
-    git
-
-RUN useradd -m -s /bin/bash user && \
-    mkdir -p /home/user && \
-    chown -R user /home/user/
-
-RUN cd /home/user/ && \
-    git clone https://github.com/opea-project/GenAIComps.git
-
-RUN cd /home/user/GenAIComps && pip install --no-cache-dir --upgrade pip && \
-    pip install -r /home/user/GenAIComps/requirements.txt
-
-COPY ../codegen.py /home/user/codegen.py
-
-ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps
-
-USER user
-
-WORKDIR /home/user
-
-ENTRYPOINT ["python", "codegen.py"]

CodeGen/microservice/xeon/codegen.py

@@ -1,51 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import asyncio
-import os
-
-from comps import CodeGenGateway, MicroService, ServiceOrchestrator, ServiceType
-
-SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0")
-
-
-class ChatQnAService:
-    def __init__(self, port=8000):
-        self.port = port
-        self.megaservice = ServiceOrchestrator()
-
-    def add_remote_service(self):
-        llm = MicroService(
-            name="llm",
-            host=SERVICE_HOST_IP,
-            port=9000,
-            endpoint="/v1/chat/completions",
-            use_remote_service=True,
-            service_type=ServiceType.LLM,
-        )
-        self.megaservice.add(llm)
-        self.gateway = CodeGenGateway(megaservice=self.megaservice, host="0.0.0.0", port=self.port)
-
-    async def schedule(self):
-        await self.megaservice.schedule(
-            initial_inputs={"text": "Write a function that checks if a year is a leap year in Python."}
-        )
-        result_dict = self.megaservice.result_dict
-        print(result_dict)
-
-
-if __name__ == "__main__":
-    chatqna = ChatQnAService(port=6666)
-    chatqna.add_remote_service()
-    asyncio.run(chatqna.schedule())

CodeGen/tests/test_codegen_on_gaudi.sh

@@ -24,12 +24,12 @@ function build_docker_images() {
     git clone https://github.com/opea-project/GenAIComps.git
     cd GenAIComps
 
-    docker build -t opea/gen-ai-comps:llm-tgi-gaudi-server -f comps/llms/langchain/docker/Dockerfile .
+    docker build -t opea/gen-ai-comps:llm-tgi-gaudi-server -f comps/llms/text-generation/tgi/Dockerfile .
 
     docker pull ghcr.io/huggingface/tgi-gaudi:1.2.1
 
-    cd $WORKPATH/microservice/gaudi
+    cd $WORKPATH
-    docker build --no-cache -t opea/gen-ai-comps:codegen-megaservice-server -f docker/Dockerfile .
+    docker build --no-cache -t opea/gen-ai-comps:codegen-megaservice-server -f Dockerfile .
 
     cd $WORKPATH/ui
     docker build --no-cache -t opea/gen-ai-comps:codegen-ui-server -f docker/Dockerfile .
@@ -38,12 +38,13 @@ function build_docker_images() {
 }
 
 function start_services() {
-    cd $WORKPATH/microservice/gaudi
+    cd $WORKPATH/docker-composer/gaudi
 
     export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
     export TGI_LLM_ENDPOINT="http://${ip_address}:8028"
     export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
     export MEGA_SERVICE_HOST_IP=${ip_address}
+    export LLM_SERVICE_HOST_IP=${ip_address}
     export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:6666/v1/codegen"
 
     # Start Docker Containers
@@ -61,7 +62,7 @@ function validate_microservices() {
 
     curl http://${ip_address}:8028/generate \
         -X POST \
-        -d '{"inputs":"Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception.","parameters":{"max_new_tokens":1024, "do_sample": true}}' \
+        -d '{"inputs":"def print_hello_world():","parameters":{"max_new_tokens":1024, "do_sample": true}}' \
         -H 'Content-Type: application/json' > ${LOG_PATH}/generate.log
     exit_code=$?
     if [ $exit_code -ne 0 ]; then
@@ -73,7 +74,7 @@ function validate_microservices() {
 
     curl http://${ip_address}:9000/v1/chat/completions \
         -X POST \
-        -d '{"text":"Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception."}' \
+        -d '{"text":"def print_hello_world():"}' \
         -H 'Content-Type: application/json' > ${LOG_PATH}/completions.log
     exit_code=$?
     if [ $exit_code -ne 0 ]; then
@@ -88,12 +89,12 @@ function validate_megaservice() {
     # Curl the Mega Service
     curl http://${ip_address}:6666/v1/codegen -H "Content-Type: application/json" -d '{
         "model": "ise-uiuc/Magicoder-S-DS-6.7B",
-        "messages": "Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception."}' > ${LOG_PATH}/curl_megaservice.log
+        "messages": "def print_hello_world():"}' > ${LOG_PATH}/curl_megaservice.log
 
     echo "Checking response results, make sure the output is reasonable. "
-    local status=true
+    local status=false
     if [[ -f $LOG_PATH/curl_megaservice.log ]] && \
-    [[ $(grep -c "billion" $LOG_PATH/curl_megaservice.log) != 0 ]]; then
+    [[ $(grep -c "Hello" $LOG_PATH/curl_megaservice.log) != 0 ]]; then
         status=true
     fi
 
@@ -109,7 +110,7 @@ function validate_megaservice() {
 }
 
 function stop_docker() {
-    cd $WORKPATH/microservice/gaudi
+    cd $WORKPATH/docker-composer/gaudi
     container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
     for container_name in $container_list; do
         cid=$(docker ps -aq --filter "name=$container_name")

CodeGen/tests/test_codegen_on_xeon.sh

@@ -14,10 +14,10 @@ function build_docker_images() {
     git clone https://github.com/opea-project/GenAIComps.git
     cd GenAIComps
 
-    docker build -t opea/gen-ai-comps:llm-tgi-server -f comps/llms/langchain/docker/Dockerfile .
+    docker build -t opea/gen-ai-comps:llm-tgi-server -f comps/llms/text-generation/tgi/Dockerfile .
 
-    cd $WORKPATH/microservice/xeon
+    cd $WORKPATH
-    docker build --no-cache -t opea/gen-ai-comps:codegen-megaservice-server -f docker/Dockerfile .
+    docker build --no-cache -t opea/gen-ai-comps:codegen-megaservice-server -f Dockerfile .
 
     cd $WORKPATH/ui
     docker build --no-cache -t opea/gen-ai-comps:codegen-ui-server -f docker/Dockerfile .
@@ -26,12 +26,13 @@ function build_docker_images() {
 }
 
 function start_services() {
-    cd $WORKPATH/microservice/xeon
+    cd $WORKPATH/docker-composer/xeon
 
     export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
     export TGI_LLM_ENDPOINT="http://${ip_address}:8028"
     export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
     export MEGA_SERVICE_HOST_IP=${ip_address}
+    export LLM_SERVICE_HOST_IP=${ip_address}
     export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:6666/v1/codegen"
 
     # Start Docker Containers
@@ -49,13 +50,13 @@ function validate_microservices() {
 
     curl http://${ip_address}:8028/generate \
         -X POST \
-        -d '{"inputs":"Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception.","parameters":{"max_new_tokens":256, "do_sample": true}}' \
+        -d '{"inputs":"def print_hello_world():","parameters":{"max_new_tokens":256, "do_sample": true}}' \
         -H 'Content-Type: application/json' > ${LOG_PATH}/generate.log
     sleep 5s
 
     curl http://${ip_address}:9000/v1/chat/completions \
         -X POST \
-        -d '{"text":"Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception."}' \
+        -d '{"text":"def print_hello_world():"}' \
         -H 'Content-Type: application/json' > ${LOG_PATH}/completions.log
     sleep 5s
 }
@@ -64,12 +65,13 @@ function validate_megaservice() {
     # Curl the Mega Service
     curl http://${ip_address}:6666/v1/codegen -H "Content-Type: application/json" -d '{
         "model": "ise-uiuc/Magicoder-S-DS-6.7B",
-        "messages": "Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception."}' > ${LOG_PATH}/curl_megaservice.log
+        "messages": "def print_hello_world():"}' > ${LOG_PATH}/curl_megaservice.log
 
+    sleep 2s
     echo "Checking response results, make sure the output is reasonable. "
-    local status=true
+    local status=false
     if [[ -f $LOG_PATH/curl_megaservice.log ]] && \
-    [[ $(grep -c "billion" $LOG_PATH/curl_megaservice.log) != 0 ]]; then
+    [[ $(grep -c "Hello" $LOG_PATH/curl_megaservice.log) != 0 ]]; then
         status=true
     fi
 
@@ -86,7 +88,7 @@ function validate_megaservice() {
 }
 
 function stop_docker() {
-    cd $WORKPATH/microservice/xeon
+    cd $WORKPATH/docker-composer/xeon
     container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
     for container_name in $container_list; do
         cid=$(docker ps -aq --filter "name=$container_name")

CodeTrans/Dockerfile

@@ -31,7 +31,7 @@ RUN cd /home/user/ && \
 RUN cd /home/user/GenAIComps && pip install --no-cache-dir --upgrade pip && \
     pip install -r /home/user/GenAIComps/requirements.txt
 
-COPY ../code_translation.py /home/user/code_translation.py
+COPY ./code_translation.py /home/user/code_translation.py
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps
 

CodeTrans/README.md

@@ -14,8 +14,8 @@ The Code Translation service can be effortlessly deployed on either Intel Gaudi2
 
 ## Deploy Code Translation on Gaudi
 
-Refer to the [Gaudi Guide](./microservice/gaudi/README.md) for instructions on deploying Code Translation on Gaudi.
+Refer to the [Gaudi Guide](./docker-composer/gaudi/README.md) for instructions on deploying Code Translation on Gaudi.
 
 ## Deploy Code Translation on Xeon
 
-Refer to the [Xeon Guide](./microservice/xeon/README.md) for instructions on deploying Code Translation on Xeon.
+Refer to the [Xeon Guide](./docker-composer/xeon/README.md) for instructions on deploying Code Translation on Xeon.

CodeTrans/code_translation.py

@@ -18,17 +18,25 @@ import os
 
 from comps import CodeTransGateway, MicroService, ServiceOrchestrator
 
-SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0")
+MEGA_SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0")
+MEGA_SERVICE_PORT = os.getenv("MEGA_SERVICE_PORT", 7777)
+LLM_SERVICE_HOST_IP = os.getenv("LLM_SERVICE_HOST_IP", "0.0.0.0")
+LLM_SERVICE_PORT = os.getenv("LLM_SERVICE_PORT", 9000)
 
 
-class MyServiceOrchestrator:
+class CodeTransService:
-    def __init__(self, port=8000):
+    def __init__(self, host="0.0.0.0", port=8000):
+        self.host = host
         self.port = port
         self.megaservice = ServiceOrchestrator()
 
     def add_remote_service(self):
         llm = MicroService(
-            name="llm", host=SERVICE_HOST_IP, port=9000, endpoint="/v1/chat/completions", use_remote_service=True
+            name="llm",
+            host=LLM_SERVICE_HOST_IP,
+            port=LLM_SERVICE_PORT,
+            endpoint="/v1/chat/completions",
+            use_remote_service=True,
         )
         self.megaservice.add(llm)
         self.gateway = CodeTransGateway(megaservice=self.megaservice, host="0.0.0.0", port=self.port)
@@ -55,6 +63,6 @@ class MyServiceOrchestrator:
 
 
 if __name__ == "__main__":
-    service_ochestrator = MyServiceOrchestrator(port=7777)
+    service_ochestrator = CodeTransService(host=MEGA_SERVICE_HOST_IP, port=MEGA_SERVICE_PORT)
     service_ochestrator.add_remote_service()
     asyncio.run(service_ochestrator.schedule())

CodeTrans/codetrans.yaml

@@ -0,0 +1,53 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+opea_micro_services:
+  tgi_service:
+    host: ${TGI_SERVICE_IP}
+    ports: ${TGI_SERVICE_PORT}
+    image: ghcr.io/huggingface/tgi-gaudi:1.2.1
+    volumes:
+      - "./data:/data"
+    runtime: habana
+    cap_add:
+      - SYS_NICE
+    ipc: host
+    environment:
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      HABANA_VISIBLE_DEVICES: all
+      OMPI_MCA_btl_vader_single_copy_mechanism: none
+    model-id: ${LLM_MODEL_ID}
+  llm:
+    host: ${LLM_SERVICE_HOST_IP}
+    ports: ${LLM_SERVICE_PORT}
+    image: opea/gen-ai-comps:llm-tgi-gaudi-server
+    endpoint: /v1/chat/completions
+    environment:
+      TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+  ui:
+    host: ${UI_SERVICE_HOST_IP}
+    ports:
+      - "5173:5173"
+    environment:
+      - CHAT_BASE_URL=${BACKEND_SERVICE_ENDPOINT}
+    image: opea/gen-ai-comps:codetrans-ui-server
+
+opea_mega_service:
+  host: ${MEGA_SERVICE_HOST_IP}
+  ports: ${MEGA_SERVICE_PORT}
+  endpoint: /v1/codetrans
+  image: opea/gen-ai-comps:codetrans-ui-server
+  mega_flow:
+    - llm

CodeTrans/docker-composer/gaudi/README.md

@@ -16,15 +16,15 @@ cd GenAIComps
 ### 2. Build the LLM Docker Image with the following command
 
 ```bash
-docker build -t opea/gen-ai-comps:llm-tgi-gaudi-server --no-cache --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/langchain/docker/Dockerfile .
+docker build -t opea/gen-ai-comps:llm-tgi-gaudi-server --no-cache --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
 ```
 
 ### 3. Build MegaService Docker Image
 
 ```bash
 git clone https://github.com/opea-project/GenAIExamples.git
-cd GenAIExamples/CodeTrans/microservice/gaudi/
+cd GenAIExamples/CodeTrans
-docker build -t opea/gen-ai-comps:codetrans-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
+docker build -t opea/gen-ai-comps:codetrans-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
 ```
 
 ### 4. Build UI Docker Image
@@ -53,12 +53,14 @@ export LLM_MODEL_ID="HuggingFaceH4/mistral-7b-grok"
 export TGI_LLM_ENDPOINT="http://${host_ip}:8008"
 export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
 export MEGA_SERVICE_HOST_IP=${host_ip}
+export LLM_SERVICE_HOST_IP=${host_ip}
 export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:7777/v1/codetrans"
 ```
 
 ### Start Microservice Docker Containers
 
 ```bash
+cd GenAIExamples/CodeTrans/docker-composer/gaudi
 docker compose -f docker_compose.yaml up -d
 ```
 

CodeTrans/docker-composer/gaudi/docker_compose.yaml

@@ -56,6 +56,7 @@ services:
       - https_proxy=${https_proxy}
       - http_proxy=${http_proxy}
       - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP}
+      - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP}
     ipc: host
     restart: always
   codetrans-gaudi-ui-server:

CodeTrans/docker-composer/xeon/README.md

@@ -24,15 +24,15 @@ cd GenAIComps
 ### 2. Build the LLM Docker Image with the following command
 
 ```bash
-docker build -t opea/gen-ai-comps:llm-tgi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/langchain/docker/Dockerfile .
+docker build -t opea/gen-ai-comps:llm-tgi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
 ```
 
 ### 3. Build MegaService Docker Image
 
 ```bash
 git clone https://github.com/opea-project/GenAIExamples.git
-cd GenAIExamples/CodeTrans/microservice/xeon/
+cd GenAIExamples/CodeTrans
-docker build -t opea/gen-ai-comps:codetrans-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
+docker build -t opea/gen-ai-comps:codetrans-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
 ```
 
 ### 4. Build UI Docker Image
@@ -61,12 +61,14 @@ export LLM_MODEL_ID="HuggingFaceH4/mistral-7b-grok"
 export TGI_LLM_ENDPOINT="http://${host_ip}:8008"
 export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
 export MEGA_SERVICE_HOST_IP=${host_ip}
+export LLM_SERVICE_HOST_IP=${host_ip}
 export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:7777/v1/codetrans"
 ```
 
 ### Start Microservice Docker Containers
 
 ```bash
+cd GenAIExamples/CodeTrans/docker-composer/xeon
 docker compose -f docker_compose.yaml up -d
 ```
 

CodeTrans/docker-composer/xeon/docker_compose.yaml

@@ -51,6 +51,7 @@ services:
       - https_proxy=${https_proxy}
       - http_proxy=${http_proxy}
       - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP}
+      - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP}
     ipc: host
     restart: always
   codetrans-xeon-ui-server:

CodeTrans/microservice/xeon/code_translation.py

@@ -1,60 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import asyncio
-import os
-
-from comps import CodeTransGateway, MicroService, ServiceOrchestrator
-
-SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0")
-
-
-class MyServiceOrchestrator:
-    def __init__(self, port=8000):
-        self.port = port
-        self.megaservice = ServiceOrchestrator()
-
-    def add_remote_service(self):
-        llm = MicroService(
-            name="llm", host=SERVICE_HOST_IP, port=9000, endpoint="/v1/chat/completions", use_remote_service=True
-        )
-        self.megaservice.add(llm)
-        self.gateway = CodeTransGateway(megaservice=self.megaservice, host="0.0.0.0", port=self.port)
-
-    async def schedule(self):
-        await self.megaservice.schedule(
-            initial_inputs={
-                "query": """
-    ### System: Please translate the following Golang codes into  Python codes.
-
-    ### Original codes:
-    '''Golang
-
-    \npackage main\n\nimport \"fmt\"\nfunc main() {\n    fmt.Println(\"Hello, World!\");\n
-
-    '''
-
-    ### Translated codes:
-"""
-            }
-        )
-        result_dict = self.megaservice.result_dict
-        print(result_dict)
-
-
-if __name__ == "__main__":
-    service_ochestrator = MyServiceOrchestrator(port=7777)
-    service_ochestrator.add_remote_service()
-    asyncio.run(service_ochestrator.schedule())

CodeTrans/microservice/xeon/docker/Dockerfile

@@ -1,42 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-FROM python:3.11-slim
-
-RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
-    libgl1-mesa-glx \
-    libjemalloc-dev \
-    vim \
-    git
-
-RUN useradd -m -s /bin/bash user && \
-    mkdir -p /home/user && \
-    chown -R user /home/user/
-
-RUN cd /home/user/ && \
-    git clone https://github.com/opea-project/GenAIComps.git
-
-RUN cd /home/user/GenAIComps && pip install --no-cache-dir --upgrade pip && \
-    pip install -r /home/user/GenAIComps/requirements.txt
-
-COPY ../code_translation.py /home/user/code_translation.py
-
-ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps
-
-USER user
-
-WORKDIR /home/user
-
-ENTRYPOINT ["python", "code_translation.py"]

CodeTrans/tests/test_codetrans_on_gaudi.sh

@@ -13,12 +13,12 @@ function build_docker_images() {
     git clone https://github.com/opea-project/GenAIComps.git
     cd GenAIComps
 
-    docker build -t opea/gen-ai-comps:llm-tgi-gaudi-server -f comps/llms/langchain/docker/Dockerfile .
+    docker build -t opea/gen-ai-comps:llm-tgi-gaudi-server -f comps/llms/text-generation/tgi/Dockerfile .
 
     docker pull ghcr.io/huggingface/tgi-gaudi:1.2.1
 
-    cd $WORKPATH/microservice/gaudi
+    cd $WORKPATH
-    docker build --no-cache -t opea/gen-ai-comps:codetrans-megaservice-server -f docker/Dockerfile .
+    docker build --no-cache -t opea/gen-ai-comps:codetrans-megaservice-server -f Dockerfile .
 
     cd $WORKPATH/ui
     docker build --no-cache -t opea/gen-ai-comps:codetrans-ui-server -f docker/Dockerfile .
@@ -27,7 +27,7 @@ function build_docker_images() {
 }
 
 function start_services() {
-    cd $WORKPATH/microservice/gaudi
+    cd $WORKPATH/docker-composer/gaudi
 
     export http_proxy=${http_proxy}
     export https_proxy=${http_proxy}
@@ -35,6 +35,7 @@ function start_services() {
     export TGI_LLM_ENDPOINT="http://${ip_address}:8008"
     export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
     export MEGA_SERVICE_HOST_IP=${ip_address}
+    export LLM_SERVICE_HOST_IP=${ip_address}
     export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:7777/v1/codetrans"
 
     # Start Docker Containers
@@ -96,7 +97,7 @@ function validate_megaservice() {
 }
 
 function stop_docker() {
-    cd $WORKPATH/microservice/gaudi
+    cd $WORKPATH/docker-composer/gaudi
     container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
     for container_name in $container_list; do
         cid=$(docker ps -aq --filter "name=$container_name")

CodeTrans/tests/test_codetrans_on_xeon.sh

@@ -14,10 +14,10 @@ function build_docker_images() {
     git clone https://github.com/opea-project/GenAIComps.git
     cd GenAIComps
 
-    docker build -t opea/gen-ai-comps:llm-tgi-server -f comps/llms/langchain/docker/Dockerfile .
+    docker build -t opea/gen-ai-comps:llm-tgi-server -f comps/llms/text-generation/tgi/Dockerfile .
 
-    cd $WORKPATH/microservice/xeon
+    cd $WORKPATH
-    docker build --no-cache -t opea/gen-ai-comps:codetrans-megaservice-server -f docker/Dockerfile .
+    docker build --no-cache -t opea/gen-ai-comps:codetrans-megaservice-server -f Dockerfile .
 
     cd $WORKPATH/ui
     docker build --no-cache -t opea/gen-ai-comps:codetrans-ui-server -f docker/Dockerfile .
@@ -26,20 +26,21 @@ function build_docker_images() {
 }
 
 function start_services() {
-    cd $WORKPATH/microservice/xeon
+    cd $WORKPATH/docker-composer/xeon
     export http_proxy=${http_proxy}
     export https_proxy=${http_proxy}
     export LLM_MODEL_ID="HuggingFaceH4/mistral-7b-grok"
     export TGI_LLM_ENDPOINT="http://${ip_address}:8008"
     export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
     export MEGA_SERVICE_HOST_IP=${ip_address}
+    export LLM_SERVICE_HOST_IP=${ip_address}
     export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:7777/v1/codetrans"
 
     # Start Docker Containers
     # TODO: Replace the container name with a test-specific name
     docker compose -f docker_compose.yaml up -d
 
-    sleep 1m
+    sleep 2m
 }
 
 function validate_microservices() {
@@ -80,7 +81,7 @@ function validate_megaservice() {
 }
 
 function stop_docker() {
-    cd $WORKPATH/microservice/xeon
+    cd $WORKPATH/docker-composer/xeon
     container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
     for container_name in $container_list; do
         cid=$(docker ps -aq --filter "name=$container_name")

CodeTrans/ui/svelte/package.json

@@ -3,24 +3,13 @@
   "version": "0.0.1",
   "scripts": {
     "dev": "vite dev",
-    "build": "vite build && npm run package",
+    "build": "vite build",
     "preview": "vite preview",
     "package": "svelte-kit sync && svelte-package && publint",
     "prepublishOnly": "npm run package",
     "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
     "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch"
   },
-  "exports": {
-    ".": {
-      "types": "./dist/index.d.ts",
-      "svelte": "./dist/index.js"
-    }
-  },
-  "files": [
-    "dist",
-    "!dist/**/*.test.*",
-    "!dist/**/*.spec.*"
-  ],
   "peerDependencies": {
     "svelte": "^4.0.0"
   },
@@ -45,8 +34,6 @@
     "typescript": "^5.0.0",
     "vite": "^5.0.11"
   },
-  "svelte": "./dist/index.js",
-  "types": "./dist/index.d.ts",
   "type": "module",
   "dependencies": {
     "prismjs": "^1.29.0",

DocSum/Dockerfile

@@ -30,7 +30,7 @@ RUN cd /home/user/ && \
 RUN cd /home/user/GenAIComps && pip install --no-cache-dir --upgrade pip && \
     pip install -r /home/user/GenAIComps/requirements.txt
 
-COPY ../docsum.py /home/user/docsum.py
+COPY ./docsum.py /home/user/docsum.py
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps
 

DocSum/README.md

@@ -17,8 +17,8 @@ The Document Summarization service can be effortlessly deployed on either Intel
 
 ## Deploy Document Summarization on Gaudi
 
-Refer to the [Gaudi Guide](./microservice/gaudi/README.md) for instructions on deploying Document Summarization on Gaudi.
+Refer to the [Gaudi Guide](./docker-composer/gaudi/README.md) for instructions on deploying Document Summarization on Gaudi.
 
 ## Deploy Document Summarization on Xeon
 
-Refer to the [Xeon Guide](./microservice/xeon/README.md) for instructions on deploying Document Summarization on Xeon.
+Refer to the [Xeon Guide](./docker-composer/xeon/README.md) for instructions on deploying Document Summarization on Xeon.

DocSum/docker-composer/gaudi/README.md

@@ -31,8 +31,8 @@ To construct the Mega Service, we utilize the [GenAIComps](https://github.com/op
 
 ```bash
 git clone https://github.com/opea-project/GenAIExamples
-cd GenAIExamples/DocSum/microservice/gaudi/
+cd GenAIExamples/DocSum
-docker build -t opea/gen-ai-comps:docsum-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
+docker build -t opea/gen-ai-comps:docsum-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
 ```
 
 ### 4. Build UI Docker Image
@@ -64,6 +64,7 @@ export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
 export TGI_LLM_ENDPOINT="http://${your_ip}:8008"
 export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
 export MEGA_SERVICE_HOST_IP=${host_ip}
+export LLM_SERVICE_HOST_IP=${host_ip}
 export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum"
 ```
 
@@ -72,6 +73,7 @@ Note: Please replace with `host_ip` with you external IP address, do not use loc
 ### Start Microservice Docker Containers
 
 ```bash
+cd GenAIExamples/DocSum/docker-composer/gaudi
 docker compose -f docker_compose.yaml up -d
 ```
 

DocSum/docker-composer/gaudi/docker_compose.yaml

@@ -54,6 +54,7 @@ services:
       - https_proxy=${https_proxy}
       - http_proxy=${http_proxy}
       - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP}
+      - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP}
     ipc: host
     restart: always
   docsum-gaudi-ui-server:

DocSum/docker-composer/xeon/README.md

@@ -33,8 +33,8 @@ To construct the Mega Service, we utilize the [GenAIComps](https://github.com/op
 
 ```bash
 git clone https://github.com/opea-project/GenAIExamples
-cd GenAIExamples/DocSum/microservice/xeon/
+cd GenAIExamples/DocSum
-docker build -t opea/gen-ai-comps:docsum-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
+docker build -t opea/gen-ai-comps:docsum-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
 ```
 
 ### 3. Build UI Docker Image
@@ -65,6 +65,7 @@ export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
 export TGI_LLM_ENDPOINT="http://${your_ip}:8008"
 export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
 export MEGA_SERVICE_HOST_IP=${host_ip}
+export LLM_SERVICE_HOST_IP=${host_ip}
 export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum"
 ```
 
@@ -73,6 +74,7 @@ Note: Please replace with `host_ip` with you external IP address, do not use loc
 ### Start Microservice Docker Containers
 
 ```bash
+cd GenAIExamples/DocSum/docker-composer/xeon
 docker compose -f docker_compose.yaml up -d
 ```
 

DocSum/docker-composer/xeon/docker_compose.yaml

@@ -55,6 +55,7 @@ services:
       - https_proxy=${https_proxy}
       - http_proxy=${http_proxy}
       - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP}
+      - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP}
     ipc: host
     restart: always
   docsum-xeon-ui-server:

DocSum/docsum.py

@@ -17,19 +17,23 @@ import os
 
 from comps import DocSumGateway, MicroService, ServiceOrchestrator, ServiceType
 
-SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0")
+MEGA_SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0")
+MEGA_SERVICE_PORT = os.getenv("MEGA_SERVICE_PORT", 8888)
+LLM_SERVICE_HOST_IP = os.getenv("LLM_SERVICE_HOST_IP", "0.0.0.0")
+LLM_SERVICE_PORT = os.getenv("LLM_SERVICE_PORT", 9000)
 
 
 class DocSumService:
-    def __init__(self, port=8000):
+    def __init__(self, host="0.0.0.0", port=8000):
+        self.host = host
         self.port = port
         self.megaservice = ServiceOrchestrator()
 
     def add_remote_service(self):
         llm = MicroService(
             name="llm",
-            host=SERVICE_HOST_IP,
+            host=LLM_SERVICE_HOST_IP,
-            port=9000,
+            port=LLM_SERVICE_PORT,
             endpoint="/v1/chat/docsum",
             use_remote_service=True,
             service_type=ServiceType.LLM,
@@ -48,6 +52,6 @@ class DocSumService:
 
 
 if __name__ == "__main__":
-    docsum = DocSumService(port=8888)
+    docsum = DocSumService(host=MEGA_SERVICE_HOST_IP, port=MEGA_SERVICE_PORT)
     docsum.add_remote_service()
     asyncio.run(docsum.schedule())

DocSum/docsum.yaml

@@ -0,0 +1,53 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+opea_micro_services:
+  tgi_service:
+    host: ${TGI_SERVICE_IP}
+    ports: ${TGI_SERVICE_PORT}
+    image: ghcr.io/huggingface/tgi-gaudi:1.2.1
+    volumes:
+      - "./data:/data"
+    runtime: habana
+    cap_add:
+      - SYS_NICE
+    ipc: host
+    environment:
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      HABANA_VISIBLE_DEVICES: all
+      OMPI_MCA_btl_vader_single_copy_mechanism: none
+    model-id: ${LLM_MODEL_ID}
+  llm:
+    host: ${LLM_SERVICE_HOST_IP}
+    ports: ${LLM_SERVICE_PORT}
+    image: opea/gen-ai-comps:llm-tgi-gaudi-server
+    endpoint: /v1/chat/completions
+    environment:
+      TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+  ui:
+    host: ${UI_SERVICE_HOST_IP}
+    ports:
+      - "5173:5173"
+    environment:
+      - CHAT_BASE_URL=${BACKEND_SERVICE_ENDPOINT}
+    image: opea/gen-ai-comps:docsum-ui-server
+
+opea_mega_service:
+  host: ${MEGA_SERVICE_HOST_IP}
+  ports: ${MEGA_SERVICE_PORT}
+  endpoint: /v1/docsum
+  image: opea/gen-ai-comps:docsum-megaservice-server
+  mega_flow:
+    - llm

DocSum/microservice/xeon/docker/Dockerfile

@@ -1,41 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-FROM langchain/langchain:latest
-
-RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
-    libgl1-mesa-glx \
-    libjemalloc-dev \
-    vim
-
-RUN useradd -m -s /bin/bash user && \
-    mkdir -p /home/user && \
-    chown -R user /home/user/
-
-RUN cd /home/user/ && \
-    git clone https://github.com/opea-project/GenAIComps.git
-
-RUN cd /home/user/GenAIComps && pip install --no-cache-dir --upgrade pip && \
-    pip install -r /home/user/GenAIComps/requirements.txt
-
-COPY ../docsum.py /home/user/docsum.py
-
-ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps
-
-USER user
-
-WORKDIR /home/user
-
-ENTRYPOINT ["python", "docsum.py"]

DocSum/microservice/xeon/docsum.py

@@ -1,53 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import asyncio
-import os
-
-from comps import DocSumGateway, MicroService, ServiceOrchestrator, ServiceType
-
-SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0")
-
-
-class DocSumService:
-    def __init__(self, port=8000):
-        self.port = port
-        self.megaservice = ServiceOrchestrator()
-
-    def add_remote_service(self):
-        llm = MicroService(
-            name="llm",
-            host=SERVICE_HOST_IP,
-            port=9000,
-            endpoint="/v1/chat/docsum",
-            use_remote_service=True,
-            service_type=ServiceType.LLM,
-        )
-        self.megaservice.add(llm)
-        self.gateway = DocSumGateway(megaservice=self.megaservice, host="0.0.0.0", port=self.port)
-
-    async def schedule(self):
-        await self.megaservice.schedule(
-            initial_inputs={
-                "text": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."
-            }
-        )
-        result_dict = self.megaservice.result_dict
-        print(result_dict)
-
-
-if __name__ == "__main__":
-    docsum = DocSumService(port=8888)
-    docsum.add_remote_service()
-    asyncio.run(docsum.schedule())

DocSum/tests/test_docsum_on_gaudi.sh

@@ -18,8 +18,8 @@ function build_docker_images() {
 
     docker pull ghcr.io/huggingface/tgi-gaudi:1.2.1
 
-    cd $WORKPATH/microservice/gaudi
+    cd $WORKPATH
-    docker build --no-cache -t opea/gen-ai-comps:docsum-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
+    docker build --no-cache -t opea/gen-ai-comps:docsum-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
 
     cd $WORKPATH/ui
     docker build --no-cache -t opea/gen-ai-comps:docsum-ui-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
@@ -28,12 +28,13 @@ function build_docker_images() {
 }
 
 function start_services() {
-    cd $WORKPATH/microservice/gaudi
+    cd $WORKPATH/docker-composer/gaudi
 
     export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
     export TGI_LLM_ENDPOINT="http://${ip_address}:8008"
     export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
     export MEGA_SERVICE_HOST_IP=${ip_address}
+    export LLM_SERVICE_HOST_IP=${ip_address}
     export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/docsum"
 
     # Start Docker Containers
@@ -98,7 +99,7 @@ function validate_megaservice() {
 }
 
 function stop_docker() {
-    cd $WORKPATH/microservice/gaudi
+    cd $WORKPATH/docker-composer/gaudi
     container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
     for container_name in $container_list; do
         cid=$(docker ps -aq --filter "name=$container_name")

DocSum/tests/test_docsum_on_xeon.sh

@@ -16,8 +16,8 @@ function build_docker_images() {
 
     docker build -t opea/gen-ai-comps:llm-docsum-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/docsum/langchain/docker/Dockerfile .
 
-    cd $WORKPATH/microservice/xeon
+    cd $WORKPATH
-    docker build --no-cache -t opea/gen-ai-comps:docsum-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
+    docker build --no-cache -t opea/gen-ai-comps:docsum-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
 
     cd $WORKPATH/ui
     docker build --no-cache -t opea/gen-ai-comps:docsum-ui-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
@@ -26,12 +26,13 @@ function build_docker_images() {
 }
 
 function start_services() {
-    cd $WORKPATH/microservice/xeon
+    cd $WORKPATH/docker-composer/xeon
 
     export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
     export TGI_LLM_ENDPOINT="http://${ip_address}:8008"
     export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
     export MEGA_SERVICE_HOST_IP=${ip_address}
+    export LLM_SERVICE_HOST_IP=${ip_address}
     export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/docsum"
 
     # Start Docker Containers
@@ -97,7 +98,7 @@ function validate_megaservice() {
 }
 
 function stop_docker() {
-    cd $WORKPATH/microservice/xeon
+    cd $WORKPATH/docker-composer/xeon
     container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
     for container_name in $container_list; do
         cid=$(docker ps -aq --filter "name=$container_name")