audioqna trigger ut

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>

.github/workflows/_run-docker-compose.yml

@@ -134,6 +134,7 @@ jobs:
           SERVING_TOKEN: ${{ secrets.SERVING_TOKEN }}
           IMAGE_REPO: ${{ inputs.registry }}
           IMAGE_TAG: ${{ inputs.tag }}
+          opea_branch: "refactor_comps"
           example: ${{ inputs.example }}
           hardware: ${{ inputs.hardware }}
           test_case: ${{ matrix.test_case }}

.github/workflows/pr-check-duplicated-image.yml

@@ -5,7 +5,7 @@ name: Check Duplicated Images
 
 on:
   pull_request:
-    branches: [main]
+    branches: [main, genaicomps_refactor]
     types: [opened, reopened, ready_for_review, synchronize]
     paths:
       - "**/docker_image_build/*.yaml"

.github/workflows/pr-docker-compose-e2e.yml

@@ -4,8 +4,8 @@
 name: E2E test with docker compose
 
 on:
-  pull_request_target:
-    branches: ["main", "*rc"]
+  pull_request:
+    branches: ["main", "*rc", "genaicomps_refactor"]
     types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
     paths:
       - "**/Dockerfile**"

.github/workflows/pr-dockerfile-path-and-build-yaml-scan.yml

@@ -5,7 +5,7 @@ name: Compose file and dockerfile path checking
 
 on:
   pull_request:
-    branches: [main]
+    branches: [main, genaicomps_refactor]
     types: [opened, reopened, ready_for_review, synchronize]
 
 jobs:
@@ -22,6 +22,7 @@ jobs:
         run: |
           cd ..
           git clone https://github.com/opea-project/GenAIComps.git
+          cd GenAIComps && git checkout refactor_comps
 
       - name: Check for Missing Dockerfile Paths in GenAIComps
         run: |

.github/workflows/pr-link-path-scan.yml

@@ -5,7 +5,7 @@ name: Check hyperlinks and relative path validity
 
 on:
   pull_request:
-    branches: [main]
+    branches: [main, genaicomps_refactor]
     types: [opened, reopened, ready_for_review, synchronize]
 
 jobs:

AudioQnA/audioqna.py

@@ -40,6 +40,15 @@ def align_inputs(self, inputs, cur_node, runtime_graph, llm_parameters_dict, **k
     return inputs
 
 
+def align_inputs(self, inputs, cur_node, runtime_graph, llm_parameters_dict, **kwargs):
+    if self.services[cur_node].service_type == ServiceType.TTS:
+        new_inputs = {}
+        new_inputs["text"] = inputs["choices"][0]["text"]
+        return new_inputs
+    else:
+        return inputs
+
+
 class AudioQnAService:
     def __init__(self, host="0.0.0.0", port=8000):
         self.host = host

AudioQnA/docker_compose/amd/gpu/rocm/compose.yaml

@@ -43,6 +43,12 @@ services:
       HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       HF_HUB_DISABLE_PROGRESS_BARS: 1
       HF_HUB_ENABLE_HF_TRANSFER: 0
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:3006/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 100
     command: --model-id ${LLM_MODEL_ID}
     cap_add:
       - SYS_PTRACE

AudioQnA/docker_compose/intel/cpu/xeon/compose.yaml

@@ -37,6 +37,12 @@ services:
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
       HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:3006/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 100
     command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0
   audioqna-xeon-backend-server:
     image: ${REGISTRY:-opea}/audioqna:${TAG:-latest}

AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml

@@ -58,6 +58,11 @@ services:
     cap_add:
       - SYS_NICE
     ipc: host
+    healthcheck:
+      test: ["CMD-SHELL", "sleep 500 && exit 0"]
+      interval: 1s
+      timeout: 505s
+      retries: 1
     command: --model-id ${LLM_MODEL_ID} --max-input-length 1024 --max-total-tokens 2048
   audioqna-gaudi-backend-server:
     image: ${REGISTRY:-opea}/audioqna:${TAG:-latest}

AudioQnA/docker_image_build/build.yaml

@@ -44,7 +44,7 @@ services:
   llm-tgi:
     build:
       context: GenAIComps
-      dockerfile: comps/llms/text-generation/tgi/Dockerfile
+      dockerfile: comps/llms/src/text-generation/Dockerfile
     extends: audioqna
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
   speecht5-gaudi:

AudioQnA/tests/test_compose_on_gaudi.sh

@@ -2,7 +2,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-set -e
+set -xe
 IMAGE_REPO=${IMAGE_REPO:-"opea"}
 IMAGE_TAG=${IMAGE_TAG:-"latest"}
 echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
@@ -45,25 +45,7 @@ function start_services() {
 
     # Start Docker Containers
     docker compose up -d > ${LOG_PATH}/start_services_with_compose.log
-    n=0
-    until [[ "$n" -ge 200 ]]; do
-       docker logs tgi-gaudi-server > $LOG_PATH/tgi_service_start.log
-       if grep -q Connected $LOG_PATH/tgi_service_start.log; then
-           break
-       fi
-       sleep 5s
-       n=$((n+1))
-    done
-
-    n=0
-    until [[ "$n" -ge 100 ]]; do
-       docker logs whisper-service > $LOG_PATH/whisper_service_start.log
-       if grep -q "Uvicorn server setup on port" $LOG_PATH/whisper_service_start.log; then
-           break
-       fi
-       sleep 5s
-       n=$((n+1))
-    done
+    sleep 20s
 }
 
 

AudioQnA/tests/test_compose_on_rocm.sh

@@ -2,7 +2,7 @@
 # Copyright (C) 2024 Advanced Micro Devices, Inc.
 # SPDX-License-Identifier: Apache-2.0
 
-set -ex
+set -xe
 IMAGE_REPO=${IMAGE_REPO:-"opea"}
 IMAGE_TAG=${IMAGE_TAG:-"latest"}
 echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
@@ -47,15 +47,7 @@ function start_services() {
 
     # Start Docker Containers
     docker compose up -d > ${LOG_PATH}/start_services_with_compose.log
-    n=0
-    until [[ "$n" -ge 200 ]]; do
-       docker logs tgi-service > $LOG_PATH/tgi_service_start.log
-       if grep -q Connected $LOG_PATH/tgi_service_start.log; then
-           break
-       fi
-       sleep 5s
-       n=$((n+1))
-    done
+    sleep 24s
 }
 function validate_megaservice() {
     response=$(http_proxy="" curl http://${ip_address}:3008/v1/audioqna -XPOST -d '{"audio": "UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA", "max_tokens":64}' -H 'Content-Type: application/json')

AudioQnA/tests/test_compose_on_xeon.sh

@@ -2,7 +2,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-set -e
+set -xe
 IMAGE_REPO=${IMAGE_REPO:-"opea"}
 IMAGE_TAG=${IMAGE_TAG:-"latest"}
 echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
@@ -46,15 +46,7 @@ function start_services() {
 
     # Start Docker Containers
     docker compose up -d > ${LOG_PATH}/start_services_with_compose.log
-    n=0
-    until [[ "$n" -ge 200 ]]; do
-       docker logs tgi-service > $LOG_PATH/tgi_service_start.log
-       if grep -q Connected $LOG_PATH/tgi_service_start.log; then
-           break
-       fi
-       sleep 5s
-       n=$((n+1))
-    done
+    sleep 20s
 }
 
 

AvatarChatbot/docker_compose/intel/cpu/xeon/compose.yaml

@@ -38,6 +38,12 @@ services:
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
       HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:3006/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 100
     command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0
   wav2lip-service:
     image: ${REGISTRY:-opea}/wav2lip:${TAG:-latest}

AvatarChatbot/docker_compose/intel/hpu/gaudi/README.md

@@ -19,6 +19,10 @@ docker build -t opea/whisper-gaudi:latest --build-arg https_proxy=$https_proxy -
 
 ### 3. Build LLM Image
 
+```bash
+docker build --no-cache -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/text-generation/Dockerfile .
+```
+
 Intel Xeon optimized image hosted in huggingface repo will be used for TGI service: ghcr.io/huggingface/tgi-gaudi:2.0.6 (https://github.com/huggingface/tgi-gaudi)
 
 ### 4. Build TTS Image

AvatarChatbot/docker_compose/intel/hpu/gaudi/compose.yaml

@@ -61,7 +61,12 @@ services:
     cap_add:
       - SYS_NICE
     ipc: host
-    command: --model-id ${LLM_MODEL_ID} --max-input-length 128 --max-total-tokens 256
+    healthcheck:
+      test: ["CMD-SHELL", "sleep 500 && exit 0"]
+      interval: 1s
+      timeout: 505s
+      retries: 1
+    command: --model-id ${LLM_MODEL_ID} --max-input-length 1024 --max-total-tokens 2048
   wav2lip-service:
     image: ${REGISTRY:-opea}/wav2lip-gaudi:${TAG:-latest}
     container_name: wav2lip-service

AvatarChatbot/docker_image_build/build.yaml

@@ -32,7 +32,7 @@ services:
   llm-tgi:
     build:
       context: GenAIComps
-      dockerfile: comps/llms/text-generation/tgi/Dockerfile
+      dockerfile: comps/llms/src/text-generation/Dockerfile
     extends: avatarchatbot
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
   speecht5-gaudi:

AvatarChatbot/tests/test_compose_on_gaudi.sh

@@ -72,19 +72,9 @@ function start_services() {
 
     # Start Docker Containers
     docker compose up -d > ${LOG_PATH}/start_services_with_compose.log
-
-    n=0
-    until [[ "$n" -ge 200 ]]; do
-       docker logs tgi-gaudi-server > $LOG_PATH/tgi_service_start.log
-       if grep -q Connected $LOG_PATH/tgi_service_start.log; then
-           break
-       fi
-       sleep 5s
-       n=$((n+1))
-    done
-
+    sleep 60s
     echo "All services are up and running"
-    sleep 5s
+
 }
 
 

AvatarChatbot/tests/test_compose_on_xeon.sh

@@ -72,17 +72,8 @@ function start_services() {
 
     # Start Docker Containers
     docker compose up -d
-    n=0
-    until [[ "$n" -ge 100 ]]; do
-       docker logs tgi-service > $LOG_PATH/tgi_service_start.log
-       if grep -q Connected $LOG_PATH/tgi_service_start.log; then
-           break
-       fi
-       sleep 5s
-       n=$((n+1))
-    done
+    sleep 20s
     echo "All services are up and running"
-    sleep 5s
 }
 
 

ChatQnA/docker_compose/amd/gpu/rocm/README.md

@@ -138,7 +138,7 @@ cd ../../../..
 
 ```bash
 cd GenAIComps
-docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/nginx/Dockerfile .
+docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/3rd_parties/nginx/src/Dockerfile .
 ```
 
 Then run the command `docker images`, you will have the following 5 Docker Images:

ChatQnA/docker_compose/intel/cpu/aipc/README.md

@@ -55,7 +55,7 @@ docker build --no-cache -t opea/chatqna-ui:latest --build-arg https_proxy=$https
 
 ```bash
 cd GenAIComps
-docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/nginx/Dockerfile .
+docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/3rd_parties/nginx/src/Dockerfile .
 ```
 
 Then run the command `docker images`, you will have the following 6 Docker Images:

ChatQnA/docker_compose/intel/cpu/xeon/README.md

@@ -161,7 +161,7 @@ docker build --no-cache -t opea/chatqna-conversation-ui:latest --build-arg https
 
 ```bash
 cd GenAIComps
-docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/nginx/Dockerfile .
+docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/3rd_parties/nginx/src/Dockerfile .
 ```
 
 Then run the command `docker images`, you will have the following 5 Docker Images:

ChatQnA/docker_compose/intel/cpu/xeon/README_pinecone.md

@@ -164,7 +164,7 @@ docker build --no-cache -t opea/chatqna-conversation-ui:latest --build-arg https
 
 ```bash
 cd GenAIComps
-docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/nginx/Dockerfile .
+docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/3rd_parties/nginx/src/Dockerfile .
 ```
 
 Then run the command `docker images`, you will have the following 5 Docker Images:

ChatQnA/docker_compose/intel/cpu/xeon/README_qdrant.md

@@ -122,7 +122,7 @@ cd ../../../..
 
 ```bash
 cd GenAIComps
-docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/nginx/Dockerfile .
+docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/3rd_parties/nginx/src/Dockerfile .
 ```
 
 Then run the command `docker images`, you will have the following 5 Docker Images:

ChatQnA/docker_compose/intel/hpu/gaudi/README.md

@@ -151,7 +151,7 @@ docker build --no-cache -t opea/chatqna-conversation-ui:latest --build-arg https
 
 ```bash
 cd GenAIComps
-docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/nginx/Dockerfile .
+docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/3rd_parties/nginx/src/Dockerfile .
 ```
 
 Then run the command `docker images`, you will have the following 5 Docker Images:

ChatQnA/docker_compose/nvidia/gpu/README.md

@@ -148,7 +148,7 @@ cd ../../..
 
 ```bash
 cd GenAIComps
-docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/nginx/Dockerfile .
+docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/3rd_parties/nginx/src/Dockerfile .
 ```
 
 Then run the command `docker images`, you will have the following 5 Docker Images:

ChatQnA/docker_image_build/build.yaml

@@ -44,7 +44,7 @@ services:
   embedding-tei:
     build:
       context: GenAIComps
-      dockerfile: comps/embeddings/tei/langchain/Dockerfile
+      dockerfile: comps/embeddings/src/Dockerfile
     extends: chatqna
     image: ${REGISTRY:-opea}/embedding-tei:${TAG:-latest}
   retriever-redis:
@@ -68,25 +68,25 @@ services:
   reranking-tei:
     build:
       context: GenAIComps
-      dockerfile: comps/reranks/tei/Dockerfile
+      dockerfile: comps/reranks/src/Dockerfile
     extends: chatqna
     image: ${REGISTRY:-opea}/reranking-tei:${TAG:-latest}
   llm-tgi:
     build:
       context: GenAIComps
-      dockerfile: comps/llms/text-generation/tgi/Dockerfile
+      dockerfile: comps/llms/src/text-generation/Dockerfile
     extends: chatqna
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
   llm-ollama:
     build:
       context: GenAIComps
-      dockerfile: comps/llms/text-generation/ollama/langchain/Dockerfile
+      dockerfile: comps/llms/src/text-generation/Dockerfile
     extends: chatqna
     image: ${REGISTRY:-opea}/llm-ollama:${TAG:-latest}
   llm-vllm:
     build:
       context: GenAIComps
-      dockerfile: comps/llms/text-generation/vllm/langchain/Dockerfile
+      dockerfile: comps/llms/src/text-generation/Dockerfile
     extends: chatqna
     image: ${REGISTRY:-opea}/llm-vllm:${TAG:-latest}
   dataprep-redis:
@@ -128,6 +128,6 @@ services:
   nginx:
     build:
       context: GenAIComps
-      dockerfile: comps/nginx/Dockerfile
+      dockerfile: comps/3rd_parties/nginx/src/Dockerfile
     extends: chatqna
     image: ${REGISTRY:-opea}/nginx:${TAG:-latest}

ChatQnA/tests/test_compose_on_rocm.sh

@@ -76,6 +76,8 @@ function start_services() {
         sleep 1s
         n=$((n+1))
     done
+
+    echo "all containers start!"
 }
 
 function validate_service() {

CodeGen/docker_compose/amd/gpu/rocm/README.md

@@ -10,7 +10,7 @@ git clone https://github.com/opea-project/GenAIComps.git
 cd GenAIComps
 
 ### Build Docker image
-docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
+docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/text-generation/Dockerfile .
 ```
 
 ### Build the MegaService Docker Image

CodeGen/docker_compose/amd/gpu/rocm/compose.yaml

@@ -15,6 +15,12 @@ services:
       https_proxy: ${https_proxy}
       HUGGING_FACE_HUB_TOKEN: ${CODEGEN_HUGGINGFACEHUB_API_TOKEN}
       HUGGINGFACEHUB_API_TOKEN: ${CODEGEN_HUGGINGFACEHUB_API_TOKEN}
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:${CODEGEN_TGI_SERVICE_PORT:-8028}/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 100
     shm_size: 1g
     devices:
       - /dev/kfd:/dev/kfd
@@ -31,7 +37,8 @@ services:
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
     container_name: codegen-llm-server
     depends_on:
-      - codegen-tgi-service
+      codegen-tgi-service:
+        condition: service_healthy
     ports:
       - "${CODEGEN_LLM_SERVICE_PORT:-9000}:9000"
     ipc: host
@@ -39,7 +46,8 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-      TGI_LLM_ENDPOINT: "http://codegen-tgi-service"
+      LLM_ENDPOINT: "http://codegen-tgi-service"
+      LLM_MODEL_ID: ${CODEGEN_LLM_MODEL_ID}
       HUGGINGFACEHUB_API_TOKEN: ${CODEGEN_HUGGINGFACEHUB_API_TOKEN}
     restart: unless-stopped
   codegen-backend-server:

CodeGen/docker_compose/intel/cpu/xeon/README.md

@@ -19,7 +19,7 @@ Should the Docker image you seek not yet be available on Docker Hub, you can bui
 ```bash
 git clone https://github.com/opea-project/GenAIComps.git
 cd GenAIComps
-docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
+docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/text-generation/Dockerfile .
 ```
 
 ### 2. Build the MegaService Docker Image

CodeGen/docker_compose/intel/cpu/xeon/compose.yaml

@@ -15,12 +15,19 @@ services:
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
       HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:8028/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 100
     command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0
   llm:
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
     container_name: llm-tgi-server
     depends_on:
-      - tgi-service
+      tgi-service:
+        condition: service_healthy
     ports:
       - "9000:9000"
     ipc: host
@@ -28,7 +35,8 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-      TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      LLM_MODEL_ID: ${LLM_MODEL_ID}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
     restart: unless-stopped
   codegen-xeon-backend-server:

CodeGen/docker_compose/intel/hpu/gaudi/README.md

@@ -11,7 +11,7 @@ First of all, you need to build the Docker images locally. This step can be igno
 ```bash
 git clone https://github.com/opea-project/GenAIComps.git
 cd GenAIComps
-docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
+docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/text-generation/Dockerfile .
 ```
 
 ### 2. Build the MegaService Docker Image

CodeGen/docker_compose/intel/hpu/gaudi/compose.yaml

@@ -20,6 +20,11 @@ services:
       LIMIT_HPU_GRAPH: true
       USE_FLASH_ATTENTION: true
       FLASH_ATTENTION_RECOMPUTE: true
+    healthcheck:
+      test: ["CMD-SHELL", "sleep 500 && exit 0"]
+      interval: 1s
+      timeout: 505s
+      retries: 1
     runtime: habana
     cap_add:
       - SYS_NICE
@@ -29,7 +34,8 @@ services:
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
     container_name: llm-tgi-gaudi-server
     depends_on:
-      - tgi-service
+      tgi-service:
+        condition: service_healthy
     ports:
       - "9000:9000"
     ipc: host
@@ -37,7 +43,8 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-      TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      LLM_MODEL_ID: ${LLM_MODEL_ID}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
     restart: unless-stopped
   codegen-gaudi-backend-server:

CodeGen/docker_image_build/build.yaml

@@ -26,6 +26,6 @@ services:
   llm-tgi:
     build:
       context: GenAIComps
-      dockerfile: comps/llms/text-generation/tgi/Dockerfile
+      dockerfile: comps/llms/src/text-generation/Dockerfile
     extends: codegen
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}

CodeGen/tests/test_compose_on_gaudi.sh

@@ -34,6 +34,7 @@ function start_services() {
     export MEGA_SERVICE_HOST_IP=${ip_address}
     export LLM_SERVICE_HOST_IP=${ip_address}
     export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:7778/v1/codegen"
+    export host_ip=${ip_address}
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 

CodeGen/tests/test_compose_on_rocm.sh

@@ -39,6 +39,7 @@ function start_services() {
     export CODEGEN_BACKEND_SERVICE_PORT=7778
     export CODEGEN_BACKEND_SERVICE_URL="http://${ip_address}:${CODEGEN_BACKEND_SERVICE_PORT}/v1/codegen"
     export CODEGEN_UI_SERVICE_PORT=5173
+    export host_ip=${ip_address}
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 

CodeGen/tests/test_compose_on_xeon.sh

@@ -35,6 +35,7 @@ function start_services() {
     export MEGA_SERVICE_HOST_IP=${ip_address}
     export LLM_SERVICE_HOST_IP=${ip_address}
     export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:7778/v1/codegen"
+    export host_ip=${ip_address}
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 

CodeGen/ui/svelte/src/routes/+page.svelte

@@ -34,22 +34,21 @@
 		const eventSource = await fetchTextStream(query);
 
 		eventSource.addEventListener("message", (e: any) => {
-			let Msg = e.data;
-			console.log("Msg", Msg);
+			let res = e.data;
 
-			if (Msg.startsWith("b")) {
-				const trimmedData = Msg.slice(2, -1);
-				if (trimmedData.includes("'''")) {
-					deleteFlag = true;
-				} else if (deleteFlag && trimmedData.includes("\\n")) {
-					deleteFlag = false;
-				} else if (trimmedData !== "</s>" && !deleteFlag) {
-					code_output += trimmedData.replace(/\\n/g, "\n");
-				}
-			} else if (Msg === "[DONE]") {
+			if (res === "[DONE]") {
 				deleteFlag = false;
 				loading = false;
 				query = '';
+			} else {
+				let Msg = JSON.parse(res).choices[0].text;
+				if (Msg.includes("'''")) {
+					deleteFlag = true;
+				} else if (deleteFlag && Msg.includes("\\n")) {
+					deleteFlag = false;
+				} else if (Msg !== "</s>" && !deleteFlag) {
+					code_output += Msg.replace(/\\n/g, "\n");
+				}
 			}
 		});
 		eventSource.stream();

CodeTrans/docker_compose/amd/gpu/rocm/README.md

@@ -10,7 +10,7 @@ git clone https://github.com/opea-project/GenAIComps.git
 cd GenAIComps
 
 ### Build Docker image
-docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
+docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/text-generation/Dockerfile .
 ```
 
 ### Build the MegaService Docker Image

CodeTrans/docker_compose/amd/gpu/rocm/compose.yaml

@@ -17,6 +17,12 @@ services:
       TGI_LLM_ENDPOINT: ${CODETRANS_TGI_LLM_ENDPOINT}
       HUGGING_FACE_HUB_TOKEN: ${CODEGEN_HUGGINGFACEHUB_API_TOKEN}
       HUGGINGFACEHUB_API_TOKEN: ${CODEGEN_HUGGINGFACEHUB_API_TOKEN}
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:8008/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 100
     devices:
       - /dev/kfd:/dev/kfd
       - /dev/dri/:/dev/dri/
@@ -31,6 +37,9 @@ services:
   codetrans-llm-server:
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
     container_name: codetrans-llm-server
+    depends_on:
+      codetrans-tgi-service:
+        condition: service_healthy
     ports:
       - "${CODETRANS_LLM_SERVICE_PORT:-9000}:9000"
     ipc: host
@@ -38,7 +47,8 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-      TGI_LLM_ENDPOINT: "http://codetrans-tgi-service"
+      LLM_ENDPOINT: "http://codetrans-tgi-service"
+      LLM_MODEL_ID: ${CODETRANS_LLM_MODEL_ID}
       HUGGINGFACEHUB_API_TOKEN: ${CODETRANS_HUGGINGFACEHUB_API_TOKEN}
     restart: unless-stopped
   codetrans-backend-server:

CodeTrans/docker_compose/intel/cpu/xeon/README.md

@@ -19,7 +19,7 @@ First of all, you need to build Docker Images locally and install the python pac
 ```bash
 git clone https://github.com/opea-project/GenAIComps.git
 cd GenAIComps
-docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
+docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/text-generation/Dockerfile .
 ```
 
 ### 2. Build MegaService Docker Image
@@ -41,7 +41,7 @@ docker build -t opea/codetrans-ui:latest --build-arg https_proxy=$https_proxy --
 
 ```bash
 cd GenAIComps
-docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/nginx/Dockerfile .
+docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/3rd_parties/nginx/src/Dockerfile .
 ```
 
 Then run the command `docker images`, you will have the following Docker Images:

CodeTrans/docker_compose/intel/cpu/xeon/compose.yaml

@@ -15,10 +15,19 @@ services:
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
       HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:8008/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 100
     command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0
   llm:
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
     container_name: llm-tgi-server
+    depends_on:
+      tgi-service:
+        condition: service_healthy
     ports:
       - "9000:9000"
     ipc: host
@@ -26,7 +35,8 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-      TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      LLM_MODEL_ID: ${LLM_MODEL_ID}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
     restart: unless-stopped
   codetrans-xeon-backend-server:

CodeTrans/docker_compose/intel/hpu/gaudi/README.md

@@ -11,7 +11,7 @@ First of all, you need to build Docker Images locally and install the python pac
 ```bash
 git clone https://github.com/opea-project/GenAIComps.git
 cd GenAIComps
-docker build -t opea/llm-tgi:latest --no-cache --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
+docker build -t opea/llm-tgi:latest --no-cache --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/text-generation/Dockerfile .
 ```
 
 ### 2. Build MegaService Docker Image
@@ -33,7 +33,7 @@ docker build -t opea/codetrans-ui:latest --build-arg https_proxy=$https_proxy --
 
 ```bash
 cd GenAIComps
-docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/nginx/Dockerfile .
+docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/3rd_parties/nginx/src/Dockerfile .
 ```
 
 Then run the command `docker images`, you will have the following Docker Images:

CodeTrans/docker_compose/intel/hpu/gaudi/compose.yaml

@@ -20,6 +20,11 @@ services:
       LIMIT_HPU_GRAPH: true
       USE_FLASH_ATTENTION: true
       FLASH_ATTENTION_RECOMPUTE: true
+    healthcheck:
+      test: ["CMD-SHELL", "sleep 500 && exit 0"]
+      interval: 1s
+      timeout: 505s
+      retries: 1
     runtime: habana
     cap_add:
       - SYS_NICE
@@ -28,6 +33,9 @@ services:
   llm:
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
     container_name: llm-tgi-gaudi-server
+    depends_on:
+      tgi-service:
+        condition: service_healthy
     ports:
       - "9000:9000"
     ipc: host
@@ -35,7 +43,8 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-      TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      LLM_MODEL_ID: ${LLM_MODEL_ID}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
     restart: unless-stopped
   codetrans-gaudi-backend-server:

CodeTrans/docker_image_build/build.yaml

@@ -20,12 +20,12 @@ services:
   llm-tgi:
     build:
       context: GenAIComps
-      dockerfile: comps/llms/text-generation/tgi/Dockerfile
+      dockerfile: comps/llms/src/text-generation/Dockerfile
     extends: codetrans
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
   nginx:
     build:
       context: GenAIComps
-      dockerfile: comps/nginx/Dockerfile
+      dockerfile: comps/3rd_parties/nginx/src/Dockerfile
     extends: codetrans
     image: ${REGISTRY:-opea}/nginx:${TAG:-latest}

CodeTrans/tests/test_compose_on_gaudi.sh

@@ -43,6 +43,7 @@ function start_services() {
     export BACKEND_SERVICE_IP=${ip_address}
     export BACKEND_SERVICE_PORT=7777
     export NGINX_PORT=80
+    export host_ip=${ip_address}
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 

CodeTrans/tests/test_compose_on_rocm.sh

@@ -45,6 +45,7 @@ function start_services() {
     export CODETRANS_BACKEND_SERVICE_PORT=7777
     export CODETRANS_NGINX_PORT=8088
     export CODETRANS_BACKEND_SERVICE_URL="http://${ip_address}:${CODETRANS_BACKEND_SERVICE_PORT}/v1/codetrans"
+    export host_ip=${ip_address}
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 

CodeTrans/tests/test_compose_on_xeon.sh

@@ -42,6 +42,7 @@ function start_services() {
     export BACKEND_SERVICE_IP=${ip_address}
     export BACKEND_SERVICE_PORT=7777
     export NGINX_PORT=80
+    export host_ip=${ip_address}
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 

CodeTrans/ui/svelte/src/routes/+page.svelte

@@ -47,16 +47,16 @@
 
   const languagesTag = {
     // 'TypeScript': typescript,
-    'Python': python,
-    'C': c,
-    'C++': cpp,
+    Python: python,
+    C: c,
+    "C++": cpp,
     // 'C#': csharp,
-    'Go': go,
-    'Java': java,
-    'JavaScript': javascript,
+    Go: go,
+    Java: java,
+    JavaScript: javascript,
     // 'Swift': swift,
     // 'Ruby': ruby,
-    'Rust': rust,
+    Rust: rust,
     // 'PHP': php,
     // 'Kotlin': kotlin,
     // 'Objective-C': objectivec,
@@ -103,21 +103,20 @@
     const eventSource = await fetchTextStream(input, langFrom, langTo);
 
     eventSource.addEventListener("message", (e: any) => {
-      let Msg = e.data;
-      console.log('Msg', Msg);
+      let res = e.data;
 
-      if (Msg.startsWith("b")) {
-        const trimmedData = Msg.slice(2, -1);
-        if (trimmedData.includes("'''")) {
-          deleteFlag = true;
-        } else if (deleteFlag && trimmedData.includes("\\n")) {
-          deleteFlag = false;
-        } else if (trimmedData !== "</s>" && !deleteFlag) {
-          output += trimmedData.replace(/\\n/g, "\n");
-        }
-      } else if (Msg === "[DONE]") {
+      if (res === "[DONE]") {
         deleteFlag = false;
         loading = false;
+      } else {
+        let Msg = JSON.parse(res).choices[0].text;
+        if (Msg.includes("'''")) {
+          deleteFlag = true;
+        } else if (deleteFlag && Msg.includes("\\n")) {
+          deleteFlag = false;
+        } else if (Msg !== "</s>" && !deleteFlag) {
+          output += Msg.replace(/\\n/g, "\n");
+        }
       }
     });
     eventSource.stream();
@@ -202,7 +201,9 @@
           data-testid="code-output"
         >
           {#if output !== ""}
-            <div class="bg-[#282c34] p-2 px-6 text-white flex justify-end border-2 border-none border-b-gray-800">
+            <div
+              class="bg-[#282c34] p-2 px-6 text-white flex justify-end border-2 border-none border-b-gray-800"
+            >
               <button
                 class="border px-3 py-1 rounded border-none"
                 on:click={() => {

DocIndexRetriever/docker_compose/intel/cpu/xeon/README.md

@@ -9,7 +9,7 @@ DocRetriever are the most widely adopted use case for leveraging the different m
   ```bash
   git clone https://github.com/opea-project/GenAIComps.git
   cd GenAIComps
-  docker build -t opea/embedding-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/tei/langchain/Dockerfile .
+  docker build -t opea/embedding-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/src/Dockerfile .
   ```
 
 - Retriever Vector store Image
@@ -21,7 +21,7 @@ DocRetriever are the most widely adopted use case for leveraging the different m
 - Rerank TEI Image
 
   ```bash
-  docker build -t opea/reranking-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/tei/Dockerfile .
+  docker build -t opea/reranking-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/src/Dockerfile .
   ```
 
 - Dataprep Image

DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml

@@ -28,8 +28,10 @@ services:
       INDEX_NAME: ${INDEX_NAME}
       TEI_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      LOGFLAG: ${LOGFLAG}
   tei-embedding-service:
     image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+    entrypoint: /bin/sh -c "apt-get update && apt-get install -y curl && text-embeddings-router --json-output --model-id ${EMBEDDING_MODEL_ID} --auto-truncate"
     container_name: tei-embedding-server
     ports:
       - "6006:80"
@@ -41,7 +43,12 @@ services:
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
-    command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:6006/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 60
   embedding:
     image: ${REGISTRY:-opea}/embedding-tei:${TAG:-latest}
     container_name: embedding-tei-server
@@ -49,13 +56,15 @@ services:
       - "6000:6000"
     ipc: host
     depends_on:
-      - tei-embedding-service
+      tei-embedding-service:
+        condition: service_healthy
     environment:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       TEI_EMBEDDING_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
+      LOGFLAG: ${LOGFLAG}
     restart: unless-stopped
   retriever:
     image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest}
@@ -73,9 +82,11 @@ services:
       INDEX_NAME: ${INDEX_NAME}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       TEI_EMBEDDING_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
+      LOGFLAG: ${LOGFLAG}
     restart: unless-stopped
   tei-reranking-service:
     image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+    entrypoint: /bin/sh -c "apt-get update && apt-get install -y curl && text-embeddings-router --json-output --model-id ${RERANK_MODEL_ID} --auto-truncate"
     container_name: tei-reranking-server
     ports:
       - "8808:80"
@@ -89,12 +100,18 @@ services:
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       HF_HUB_DISABLE_PROGRESS_BARS: 1
       HF_HUB_ENABLE_HF_TRANSFER: 0
-    command: --model-id ${RERANK_MODEL_ID} --auto-truncate
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:8808/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 60
   reranking:
     image: ${REGISTRY:-opea}/reranking-tei:${TAG:-latest}
     container_name: reranking-tei-xeon-server
     depends_on:
-      - tei-reranking-service
+      tei-reranking-service:
+        condition: service_healthy
     ports:
       - "8000:8000"
     ipc: host
@@ -102,10 +119,12 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
+      RERANK_TYPE: ${RERANK_TYPE}
       TEI_RERANKING_ENDPOINT: ${TEI_RERANKING_ENDPOINT}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       HF_HUB_DISABLE_PROGRESS_BARS: 1
       HF_HUB_ENABLE_HF_TRANSFER: 0
+      LOGFLAG: ${LOGFLAG}
     restart: unless-stopped
   doc-index-retriever-server:
     image: ${REGISTRY:-opea}/doc-index-retriever:${TAG:-latest}

DocIndexRetriever/docker_compose/intel/cpu/xeon/compose_without_rerank.yaml

@@ -26,22 +26,29 @@ services:
       REDIS_URL: redis://redis-vector-db:6379
       REDIS_HOST: redis-vector-db
       INDEX_NAME: ${INDEX_NAME:-rag-redis}
-      TEI_EMBEDDING_ENDPOINT: http://tei-embedding-service:80
+      TEI_ENDPOINT: http://tei-embedding-service:80
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      LOGFLAG: ${LOGFLAG}
   tei-embedding-service:
     image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+    entrypoint: /bin/sh -c "apt-get update && apt-get install -y curl && text-embeddings-router --json-output --model-id ${EMBEDDING_MODEL_ID} --auto-truncate"
     container_name: tei-embedding-server
     ports:
       - "6006:80"
     volumes:
-      - "/home/ligang/models:/data"
+      - "/mnt/models:/data"
     shm_size: 1g
     environment:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
-    command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:6006/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 60
   embedding:
     image: ${REGISTRY:-opea}/embedding-tei:${TAG:-latest}
     container_name: embedding-tei-server
@@ -49,13 +56,15 @@ services:
       - "6000:6000"
     ipc: host
     depends_on:
-      - tei-embedding-service
+      tei-embedding-service:
+        condition: service_healthy
     environment:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       TEI_EMBEDDING_ENDPOINT: http://tei-embedding-service:80
+      LOGFLAG: ${LOGFLAG}
     restart: unless-stopped
   retriever:
     image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest}
@@ -73,6 +82,7 @@ services:
       INDEX_NAME: ${INDEX_NAME:-rag-redis}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       TEI_EMBEDDING_ENDPOINT: http://tei-embedding-service:80
+      LOGFLAG: ${LOGFLAG}
     restart: unless-stopped
   doc-index-retriever-server:
     image: ${REGISTRY:-opea}/doc-index-retriever:${TAG:-latest}

DocIndexRetriever/docker_compose/intel/hpu/gaudi/README.md

@@ -9,7 +9,7 @@ DocRetriever are the most widely adopted use case for leveraging the different m
   ```bash
   git clone https://github.com/opea-project/GenAIComps.git
   cd GenAIComps
-  docker build -t opea/embedding-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/tei/langchain/Dockerfile .
+  docker build -t opea/embedding-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/src/Dockerfile .
   ```
 
 - Retriever Vector store Image
@@ -21,7 +21,7 @@ DocRetriever are the most widely adopted use case for leveraging the different m
 - Rerank TEI Image
 
   ```bash
-  docker build -t opea/reranking-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/tei/Dockerfile .
+  docker build -t opea/reranking-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/src/Dockerfile .
   ```
 
 - Dataprep Image

DocIndexRetriever/docker_compose/intel/hpu/gaudi/compose.yaml

@@ -29,6 +29,7 @@ services:
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
   tei-embedding-service:
     image: ghcr.io/huggingface/tei-gaudi:1.5.0
+    entrypoint: /bin/sh -c "apt-get update && apt-get install -y curl && text-embeddings-router --json-output --model-id ${EMBEDDING_MODEL_ID} --auto-truncate"
     container_name: tei-embedding-gaudi-server
     ports:
       - "8090:80"
@@ -47,7 +48,12 @@ services:
       MAX_WARMUP_SEQUENCE_LENGTH: 512
       INIT_HCCL_ON_ACQUIRE: 0
       ENABLE_EXPERIMENTAL_FLAGS: true
-    command: --model-id ${EMBEDDING_MODEL_ID}  --auto-truncate
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:8090/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 60
   embedding:
     image: ${REGISTRY:-opea}/embedding-tei:${TAG:-latest}
     container_name: embedding-tei-server
@@ -55,12 +61,14 @@ services:
       - "6000:6000"
     ipc: host
     depends_on:
-      - tei-embedding-service
+      tei-embedding-service:
+        condition: service_healthy
     environment:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
       TEI_EMBEDDING_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
+      HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       LOGFLAG: ${LOGFLAG}
     restart: unless-stopped
   retriever:
@@ -81,6 +89,7 @@ services:
     restart: unless-stopped
   tei-reranking-service:
     image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+    entrypoint: /bin/sh -c "apt-get update && apt-get install -y curl && text-embeddings-router --json-output --model-id ${RERANK_MODEL_ID} --auto-truncate"
     container_name: tei-reranking-gaudi-server
     ports:
       - "8808:80"
@@ -94,12 +103,18 @@ services:
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       HF_HUB_DISABLE_PROGRESS_BARS: 1
       HF_HUB_ENABLE_HF_TRANSFER: 0
-    command: --model-id ${RERANK_MODEL_ID} --auto-truncate
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:8808/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 60
   reranking:
     image: ${REGISTRY:-opea}/reranking-tei:${TAG:-latest}
     container_name: reranking-tei-gaudi-server
     depends_on:
-      - tei-reranking-service
+      tei-reranking-service:
+        condition: service_healthy
     ports:
       - "8000:8000"
     ipc: host
@@ -107,6 +122,7 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
+      RERANK_TYPE: ${RERANK_TYPE}
       TEI_RERANKING_ENDPOINT: ${TEI_RERANKING_ENDPOINT}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       HF_HUB_DISABLE_PROGRESS_BARS: 1

DocIndexRetriever/docker_image_build/build.yaml

@@ -14,7 +14,7 @@ services:
   embedding-tei:
     build:
       context: GenAIComps
-      dockerfile: comps/embeddings/tei/langchain/Dockerfile
+      dockerfile: comps/embeddings/src/Dockerfile
     extends: doc-index-retriever
     image: ${REGISTRY:-opea}/embedding-tei:${TAG:-latest}
   retriever-redis:
@@ -26,7 +26,7 @@ services:
   reranking-tei:
     build:
       context: GenAIComps
-      dockerfile: comps/reranks/tei/Dockerfile
+      dockerfile: comps/reranks/src/Dockerfile
     extends: doc-index-retriever
     image: ${REGISTRY:-opea}/reranking-tei:${TAG:-latest}
   dataprep-redis:

DocIndexRetriever/retrieval_tool.py

@@ -21,10 +21,29 @@ RERANK_SERVICE_HOST_IP = os.getenv("RERANK_SERVICE_HOST_IP", "0.0.0.0")
 RERANK_SERVICE_PORT = os.getenv("RERANK_SERVICE_PORT", 8000)
 
 
+def align_inputs(self, inputs, cur_node, runtime_graph, llm_parameters_dict, **kwargs):
+    if self.services[cur_node].service_type == ServiceType.EMBEDDING:
+        inputs["input"] = inputs["text"]
+        del inputs["text"]
+    return inputs
+
+
+def align_outputs(self, data, cur_node, inputs, runtime_graph, llm_parameters_dict, **kwargs):
+    next_data = {}
+    if self.services[cur_node].service_type == ServiceType.EMBEDDING:
+        next_data = {"text": inputs["input"], "embedding": [item["embedding"] for item in data["data"]]}
+    else:
+        next_data = data
+
+    return next_data
+
+
 class RetrievalToolService:
     def __init__(self, host="0.0.0.0", port=8000):
         self.host = host
         self.port = port
+        ServiceOrchestrator.align_inputs = align_inputs
+        ServiceOrchestrator.align_outputs = align_outputs
         self.megaservice = ServiceOrchestrator()
         self.endpoint = str(MegaServiceEndpoint.RETRIEVALTOOL)
 

DocIndexRetriever/tests/test_compose_on_gaudi.sh

@@ -46,6 +46,9 @@ function start_services() {
     export RETRIEVER_SERVICE_HOST_IP=${ip_address}
     export RERANK_SERVICE_HOST_IP=${ip_address}
     export LLM_SERVICE_HOST_IP=${ip_address}
+    export host_ip=${ip_address}
+    export RERANK_TYPE="tei"
+    export LOGFLAG=true
 
     # Start Docker Containers
     docker compose up -d

DocIndexRetriever/tests/test_compose_on_xeon.sh

@@ -47,6 +47,9 @@ function start_services() {
     export RETRIEVER_SERVICE_HOST_IP=${ip_address}
     export RERANK_SERVICE_HOST_IP=${ip_address}
     export LLM_SERVICE_HOST_IP=${ip_address}
+    export host_ip=${ip_address}
+    export RERANK_TYPE="tei"
+    export LOGFLAG=true
 
     # Start Docker Containers
     docker compose up -d

DocIndexRetriever/tests/test_compose_without_rerank_on_xeon.sh

@@ -42,6 +42,8 @@ function start_services() {
     export MEGA_SERVICE_HOST_IP=${ip_address}
     export EMBEDDING_SERVICE_HOST_IP=${ip_address}
     export RETRIEVER_SERVICE_HOST_IP=${ip_address}
+    export host_ip=${ip_address}
+    export LOGFLAG=true
 
     # Start Docker Containers
     docker compose -f compose_without_rerank.yaml up -d

FaqGen/docker_compose/amd/gpu/rocm/README.md

@@ -10,7 +10,7 @@ git clone https://github.com/opea-project/GenAIComps.git
 cd GenAIComps
 
 ### Build Docker image
-docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
+docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/text-generation/Dockerfile .
 ```
 
 ## 🚀 Start Microservices and MegaService

GraphRAG/docker_image_build/build.yaml

@@ -36,7 +36,7 @@ services:
         https_proxy: ${https_proxy}
         no_proxy: ${no_proxy}
       context: GenAIComps
-      dockerfile: comps/nginx/Dockerfile
+      dockerfile: comps/3rd_parties/nginx/src/Dockerfile
     image: ${REGISTRY:-opea}/nginx:${TAG:-latest}
   graphrag-ui:
     build:

MultimodalQnA/docker_compose/amd/gpu/rocm/README.md

@@ -25,13 +25,13 @@ Build embedding-multimodal-bridgetower docker image
 ```bash
 git clone https://github.com/opea-project/GenAIComps.git
 cd GenAIComps
-docker build --no-cache -t opea/embedding-multimodal-bridgetower:latest --build-arg EMBEDDER_PORT=$EMBEDDER_PORT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal/bridgetower/Dockerfile .
+docker build --no-cache -t opea/embedding-multimodal-bridgetower:latest --build-arg EMBEDDER_PORT=$EMBEDDER_PORT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/src/integrations/dependency/bridgetower/Dockerfile .
 ```
 
 Build embedding-multimodal microservice image
 
 ```bash
-docker build --no-cache -t opea/embedding-multimodal:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal/multimodal_langchain/Dockerfile .
+docker build --no-cache -t opea/embedding-multimodal:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/src/Dockerfile .
 ```
 
 ### 2. Build LVM Images

MultimodalQnA/docker_compose/intel/cpu/xeon/README.md

@@ -112,13 +112,13 @@ Build embedding-multimodal-bridgetower docker image
 ```bash
 git clone https://github.com/opea-project/GenAIComps.git
 cd GenAIComps
-docker build --no-cache -t opea/embedding-multimodal-bridgetower:latest --build-arg EMBEDDER_PORT=$EMBEDDER_PORT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal/bridgetower/Dockerfile .
+docker build --no-cache -t opea/embedding-multimodal-bridgetower:latest --build-arg EMBEDDER_PORT=$EMBEDDER_PORT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/src/integrations/dependency/bridgetower/Dockerfile .
 ```
 
 Build embedding-multimodal microservice image
 
 ```bash
-docker build --no-cache -t opea/embedding-multimodal:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal/multimodal_langchain/Dockerfile .
+docker build --no-cache -t opea/embedding-multimodal:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/src/Dockerfile .
 ```
 
 ### 2. Build retriever-multimodal-redis Image

MultimodalQnA/docker_compose/intel/hpu/gaudi/README.md

@@ -63,13 +63,13 @@ Build embedding-multimodal-bridgetower docker image
 ```bash
 git clone https://github.com/opea-project/GenAIComps.git
 cd GenAIComps
-docker build --no-cache -t opea/embedding-multimodal-bridgetower:latest --build-arg EMBEDDER_PORT=$EMBEDDER_PORT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal/bridgetower/Dockerfile .
+docker build --no-cache -t opea/embedding-multimodal-bridgetower:latest --build-arg EMBEDDER_PORT=$EMBEDDER_PORT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/src/integrations/dependency/bridgetower/Dockerfile .
 ```
 
 Build embedding-multimodal microservice image
 
 ```bash
-docker build --no-cache -t opea/embedding-multimodal:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal/multimodal_langchain/Dockerfile .
+docker build --no-cache -t opea/embedding-multimodal:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/src/Dockerfile .
 ```
 
 ### 2. Build retriever-multimodal-redis Image

MultimodalQnA/docker_image_build/build.yaml

@@ -20,13 +20,13 @@ services:
   embedding-multimodal-bridgetower:
     build:
       context: GenAIComps
-      dockerfile: comps/embeddings/multimodal/bridgetower/Dockerfile
+      dockerfile: comps/embeddings/src/integrations/dependency/bridgetower/Dockerfile
     extends: multimodalqna
     image: ${REGISTRY:-opea}/embedding-multimodal-bridgetower:${TAG:-latest}
   embedding-multimodal:
     build:
       context: GenAIComps
-      dockerfile: comps/embeddings/multimodal/multimodal_langchain/Dockerfile
+      dockerfile: comps/embeddings/src/Dockerfile
     extends: multimodalqna
     image: ${REGISTRY:-opea}/embedding-multimodal:${TAG:-latest}
   retriever-redis:

ProductivitySuite/docker_compose/intel/cpu/xeon/README.md

@@ -13,7 +13,7 @@ First of all, you need to build Docker Images locally and install the python pac
 ```bash
 git clone https://github.com/opea-project/GenAIComps.git
 cd GenAIComps
-docker build --no-cache -t opea/embedding-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/tei/langchain/Dockerfile .
+docker build --no-cache -t opea/embedding-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/src/Dockerfile .
 ```
 
 ### 2. Build Retriever Image
@@ -25,7 +25,7 @@ docker build --no-cache -t opea/retriever-redis:latest --build-arg https_proxy=$
 ### 3. Build Rerank Image
 
 ```bash
-docker build --no-cache -t opea/reranking-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/tei/Dockerfile .
+docker build --no-cache -t opea/reranking-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/src/Dockerfile .
 ```
 
 ### 4. Build LLM Image
@@ -33,7 +33,7 @@ docker build --no-cache -t opea/reranking-tei:latest --build-arg https_proxy=$ht
 #### Use TGI as backend
 
 ```bash
-docker build --no-cache -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
+docker build --no-cache -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/text-generation/Dockerfile .
 ```
 
 ### 5. Build Dataprep Image

ProductivitySuite/docker_compose/intel/cpu/xeon/compose.yaml

@@ -20,7 +20,7 @@ services:
     depends_on:
       - redis-vector-db
     ports:
-      - "6007:6007"
+      - "6007:5000"
     environment:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
@@ -28,10 +28,13 @@ services:
       REDIS_URL: ${REDIS_URL}
       REDIS_HOST: redis-vector-db
       INDEX_NAME: ${INDEX_NAME}
-      TEI_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
+      TEI_EMBEDDING_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      DATAPREP_TYPE: ${DATAPREP_TYPE}
+      LOGFLAG: ${LOGFLAG}
   tei-embedding-service:
     image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+    entrypoint: /bin/sh -c "apt-get update && apt-get install -y curl && text-embeddings-router --json-output --model-id ${EMBEDDING_MODEL_ID} --auto-truncate"
     container_name: tei-embedding-server
     ports:
       - "6006:80"
@@ -42,12 +45,18 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-    command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:6006/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 60
   embedding:
     image: ${REGISTRY:-opea}/embedding-tei:${TAG:-latest}
     container_name: embedding-tei-server
     depends_on:
-      - tei-embedding-service
+      tei-embedding-service:
+        condition: service_healthy
     ports:
       - "6000:6000"
     ipc: host
@@ -56,9 +65,8 @@ services:
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
       TEI_EMBEDDING_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
-      LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY}
-      LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2}
-      LANGCHAIN_PROJECT: "opea-embedding-service"
+      HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      LOGFLAG: ${LOGFLAG}
     restart: unless-stopped
   retriever:
     image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest}
@@ -77,9 +85,12 @@ services:
       INDEX_NAME: ${INDEX_NAME}
       TEI_EMBEDDING_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      RETRIEVER_TYPE: ${RETRIEVER_TYPE}
+      LOGFLAG: ${LOGFLAG}
     restart: unless-stopped
   tei-reranking-service:
     image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+    entrypoint: /bin/sh -c "apt-get update && apt-get install -y curl && text-embeddings-router --json-output --model-id ${RERANK_MODEL_ID} --auto-truncate"
     container_name: tei-reranking-server
     ports:
       - "8808:80"
@@ -93,12 +104,18 @@ services:
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       HF_HUB_DISABLE_PROGRESS_BARS: 1
       HF_HUB_ENABLE_HF_TRANSFER: 0
-    command: --model-id ${RERANK_MODEL_ID} --auto-truncate
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:8808/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 60
   reranking:
     image: ${REGISTRY:-opea}/reranking-tei:${TAG:-latest}
     container_name: reranking-tei-xeon-server
     depends_on:
-      - tei-reranking-service
+      tei-reranking-service:
+        condition: service_healthy
     ports:
       - "8000:8000"
     ipc: host
@@ -106,13 +123,12 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
+      RERANK_TYPE: ${RERANK_TYPE}
       TEI_RERANKING_ENDPOINT: ${TEI_RERANKING_ENDPOINT}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       HF_HUB_DISABLE_PROGRESS_BARS: 1
       HF_HUB_ENABLE_HF_TRANSFER: 0
-      LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY}
-      LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2}
-      LANGCHAIN_PROJECT: "opea-reranking-service"
+      LOGFLAG: ${LOGFLAG}
     restart: unless-stopped
   tgi_service:
     image: ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu
@@ -129,12 +145,19 @@ services:
       HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       HF_HUB_DISABLE_PROGRESS_BARS: 1
       HF_HUB_ENABLE_HF_TRANSFER: 0
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:9009/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 100
     command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0
   llm:
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
     container_name: llm-tgi-server
     depends_on:
-      - tgi_service
+      tgi_service:
+        condition: service_healthy
     ports:
       - "9000:9000"
     ipc: host
@@ -142,13 +165,12 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-      TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT_CHATQNA}
+      LLM_ENDPOINT: ${TGI_LLM_ENDPOINT_CHATQNA}
+      LLM_MODEL_ID: ${LLM_MODEL_ID}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       HF_HUB_DISABLE_PROGRESS_BARS: 1
       HF_HUB_ENABLE_HF_TRANSFER: 0
-      LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY}
-      LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2}
-      LANGCHAIN_PROJECT: "opea-llm-service"
+      LOGFLAG: ${LOGFLAG}
     restart: unless-stopped
   chatqna-xeon-backend-server:
     image: ${REGISTRY:-opea}/chatqna:${TAG:-latest}
@@ -194,12 +216,19 @@ services:
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
       HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:8028/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 30
     command: --model-id ${LLM_MODEL_ID_CODEGEN}
   llm_codegen:
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
     container_name: llm-tgi-server-codegen
     depends_on:
-      - tgi_service_codegen
+      tgi_service_codegen:
+        condition: service_healthy
     ports:
       - "9001:9000"
     ipc: host
@@ -207,11 +236,10 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-      TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT_CODEGEN}
+      LLM_ENDPOINT: ${TGI_LLM_ENDPOINT_CODEGEN}
+      LLM_MODEL_ID: ${LLM_MODEL_ID}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
-      LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY}
-      LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2}
-      LANGCHAIN_PROJECT: "opea-llm-service"
+      LOGFLAG: ${LOGFLAG}
     restart: unless-stopped
   codegen-xeon-backend-server:
     image: ${REGISTRY:-opea}/codegen:${TAG:-latest}
@@ -227,6 +255,7 @@ services:
       MEGA_SERVICE_HOST_IP: ${MEGA_SERVICE_HOST_IP}
       LLM_SERVICE_HOST_IP: ${LLM_SERVICE_HOST_IP_CODEGEN}
       LLM_SERVICE_PORT: ${LLM_SERVICE_HOST_PORT_CODEGEN}
+      LOGFLAG: ${LOGFLAG}
     ipc: host
     restart: always
   llm_faqgen:
@@ -243,9 +272,6 @@ services:
       https_proxy: ${https_proxy}
       TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT_FAQGEN}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
-      LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY}
-      LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2}
-      LANGCHAIN_PROJECT: "opea-llm-service"
     restart: unless-stopped
   faqgen-xeon-backend-server:
     image: ${REGISTRY:-opea}/faqgen:${TAG:-latest}
@@ -262,6 +288,7 @@ services:
       MEGA_SERVICE_HOST_IP: ${MEGA_SERVICE_HOST_IP}
       LLM_SERVICE_PORT: ${LLM_SERVICE_HOST_PORT_FAQGEN}
       LLM_SERVICE_HOST_IP: ${LLM_SERVICE_HOST_IP_FAQGEN}
+      LOGFLAG: ${LOGFLAG}
     ipc: host
     restart: always
   llm_docsum_server:
@@ -278,9 +305,6 @@ services:
       https_proxy: ${https_proxy}
       TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT_DOCSUM}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
-      LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY}
-      LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2}
-      LANGCHAIN_PROJECT: "opea-llm-service"
     restart: unless-stopped
   docsum-xeon-backend-server:
     image: ${REGISTRY:-opea}/docsum:${TAG:-latest}
@@ -297,6 +321,7 @@ services:
       LLM_SERVICE_PORT: ${LLM_SERVICE_HOST_PORT_DOCSUM}
       MEGA_SERVICE_HOST_IP: ${MEGA_SERVICE_HOST_IP}
       LLM_SERVICE_HOST_IP: ${LLM_SERVICE_HOST_IP_DOCSUM}
+      LOGFLAG: ${LOGFLAG}
     ipc: host
     restart: always
   mongo:
@@ -323,6 +348,7 @@ services:
       MONGO_HOST: ${MONGO_HOST}
       MONGO_PORT: ${MONGO_PORT}
       COLLECTION_NAME: ${COLLECTION_NAME}
+      LOGFLAG: ${LOGFLAG}
     restart: unless-stopped
 
   promptregistry-mongo:
@@ -338,6 +364,7 @@ services:
       MONGO_HOST: ${MONGO_HOST}
       MONGO_PORT: ${MONGO_PORT}
       COLLECTION_NAME: ${PROMPT_COLLECTION_NAME}
+      LOGFLAG: ${LOGFLAG}
     restart: unless-stopped
   keycloak:
     image: quay.io/keycloak/keycloak:25.0.2
@@ -371,6 +398,7 @@ services:
       - APP_PROMPT_SERVICE_GET_ENDPOINT=${PROMPT_SERVICE_GET_ENDPOINT}
       - APP_PROMPT_SERVICE_CREATE_ENDPOINT=${PROMPT_SERVICE_CREATE_ENDPOINT}
       - APP_KEYCLOAK_SERVICE_ENDPOINT=${KEYCLOAK_SERVICE_ENDPOINT}
+      - LOGFLAG=${LOGFLAG}
     ipc: host
     restart: always
 networks:

ProductivitySuite/docker_image_build/build.yaml

@@ -14,7 +14,7 @@ services:
   embedding-tei:
     build:
       context: GenAIComps
-      dockerfile: comps/embeddings/tei/langchain/Dockerfile
+      dockerfile: comps/embeddings/src/Dockerfile
     extends: chatqna
     image: ${REGISTRY:-opea}/embedding-tei:${TAG:-latest}
   retriever-redis:
@@ -26,13 +26,13 @@ services:
   reranking-tei:
     build:
       context: GenAIComps
-      dockerfile: comps/reranks/tei/Dockerfile
+      dockerfile: comps/reranks/src/Dockerfile
     extends: chatqna
     image: ${REGISTRY:-opea}/reranking-tei:${TAG:-latest}
   llm-tgi:
     build:
       context: GenAIComps
-      dockerfile: comps/llms/text-generation/tgi/Dockerfile
+      dockerfile: comps/llms/src/text-generation/Dockerfile
     extends: chatqna
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
   dataprep-redis:

ProductivitySuite/tests/test_compose_on_xeon.sh

@@ -30,9 +30,12 @@ function start_services() {
     cd $WORKPATH/docker_compose/intel/cpu/xeon/
 
     export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
+    export RERANK_TYPE="tei"
     export RERANK_MODEL_ID="BAAI/bge-reranker-base"
     export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
     export LLM_MODEL_ID_CODEGEN="Intel/neural-chat-7b-v3-3"
+    export DATAPREP_TYPE="redis"
+    export RETRIEVER_TYPE="redis"
     export TEI_EMBEDDING_ENDPOINT="http://${ip_address}:6006"
     export TEI_RERANKING_ENDPOINT="http://${ip_address}:8808"
     export TGI_LLM_ENDPOINT="http://${ip_address}:9009"
@@ -58,8 +61,8 @@ function start_services() {
     export DATAPREP_DELETE_FILE_ENDPOINT="http://${ip_address}:6009/v1/dataprep/delete_file"
     export BACKEND_SERVICE_ENDPOINT_CODEGEN="http://${ip_address}:7778/v1/codegen"
     export BACKEND_SERVICE_ENDPOINT_DOCSUM="http://${ip_address}:8890/v1/docsum"
-    export DATAPREP_SERVICE_ENDPOINT="http://${ip_address}:6007/v1/dataprep"
-    export DATAPREP_GET_FILE_ENDPOINT="http://${ip_address}:6008/v1/dataprep/get_file"
+    export DATAPREP_SERVICE_ENDPOINT="http://${ip_address}:6007/v1/dataprep/ingest"
+    export DATAPREP_GET_FILE_ENDPOINT="http://${ip_address}:6008/v1/dataprep/get"
     export CHAT_HISTORY_CREATE_ENDPOINT="http://${ip_address}:6012/v1/chathistory/create"
     export CHAT_HISTORY_CREATE_ENDPOINT="http://${ip_address}:6012/v1/chathistory/create"
     export CHAT_HISTORY_DELETE_ENDPOINT="http://${ip_address}:6012/v1/chathistory/delete"
@@ -78,6 +81,7 @@ function start_services() {
     export EMBEDDING_SERVER_PORT=6006
     export LLM_SERVER_PORT=9009
     export PROMPT_COLLECTION_NAME="prompt"
+    export host_ip=${ip_address}
 
     # Start Docker Containers
     docker compose up -d > ${LOG_PATH}/start_services_with_compose.log
@@ -101,6 +105,8 @@ function start_services() {
         sleep 5s
         n=$((n+1))
     done
+
+    sleep 10s
 }
 
 function validate_service() {
@@ -162,38 +168,38 @@ function validate_microservices() {
     # embedding microservice
     validate_service \
         "${ip_address}:6000/v1/embeddings" \
-        '"text":"What is Deep Learning?","embedding":[' \
+        '"embedding":[' \
         "embedding-microservice" \
         "embedding-tei-server" \
-        '{"text":"What is Deep Learning?"}'
+        '{"input":"What is Deep Learning?"}'
 
     sleep 1m # retrieval can't curl as expected, try to wait for more time
 
     # test /v1/dataprep upload file
     echo "Deep learning is a subset of machine learning that utilizes neural networks with multiple layers to analyze various levels of abstract data representations. It enables computers to identify patterns and make decisions with minimal human intervention by learning from large amounts of data." > $LOG_PATH/dataprep_file.txt
     validate_service \
-        "http://${ip_address}:6007/v1/dataprep" \
+        "http://${ip_address}:6007/v1/dataprep/ingest" \
         "Data preparation succeeded" \
         "dataprep_upload_file" \
         "dataprep-redis-server"
 
     # test /v1/dataprep upload link
     validate_service \
-        "http://${ip_address}:6007/v1/dataprep" \
+        "http://${ip_address}:6007/v1/dataprep/ingest" \
         "Data preparation succeeded" \
         "dataprep_upload_link" \
         "dataprep-redis-server"
 
     # test /v1/dataprep/get_file
     validate_service \
-        "http://${ip_address}:6007/v1/dataprep/get_file" \
+        "http://${ip_address}:6007/v1/dataprep/get" \
         '{"name":' \
         "dataprep_get" \
         "dataprep-redis-server"
 
     # test /v1/dataprep/delete_file
     validate_service \
-        "http://${ip_address}:6007/v1/dataprep/delete_file" \
+        "http://${ip_address}:6007/v1/dataprep/delete" \
         '{"status":true}' \
         "dataprep_del" \
         "dataprep-redis-server"

SearchQnA/docker_compose/intel/cpu/xeon/README.md

@@ -9,7 +9,7 @@ This document outlines the deployment process for a SearchQnA application utiliz
 ```bash
 git clone https://github.com/opea-project/GenAIComps.git
 cd GenAIComps
-docker build --no-cache -t opea/embedding-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/tei/langchain/Dockerfile .
+docker build --no-cache -t opea/embedding-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/src/Dockerfile .
 ```
 
 ### 2. Build Retriever Image
@@ -21,13 +21,13 @@ docker build --no-cache -t opea/web-retriever-chroma:latest --build-arg https_pr
 ### 3. Build Rerank Image
 
 ```bash
-docker build --no-cache -t opea/reranking-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/tei/Dockerfile .
+docker build --no-cache -t opea/reranking-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/src/Dockerfile .
 ```
 
 ### 4. Build LLM Image
 
 ```bash
-docker build --no-cache -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
+docker build --no-cache -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/text-generation/Dockerfile .
 ```
 
 ### 5. Build MegaService Docker Image

SearchQnA/docker_compose/intel/cpu/xeon/compose.yaml

@@ -4,6 +4,7 @@
 services:
   tei-embedding-service:
     image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+    entrypoint: /bin/sh -c "apt-get update && apt-get install -y curl && text-embeddings-router --json-output --model-id ${EMBEDDING_MODEL_ID} --auto-truncate"
     container_name: tei-embedding-server
     ports:
       - "3001:80"
@@ -14,12 +15,18 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-    command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:3001/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 60
   embedding:
     image: ${REGISTRY:-opea}/embedding-tei:${TAG:-latest}
     container_name: embedding-tei-server
     depends_on:
-      - tei-embedding-service
+      tei-embedding-service:
+        condition: service_healthy
     ports:
       - "3002:6000"
     ipc: host
@@ -29,6 +36,7 @@ services:
       https_proxy: ${https_proxy}
       TEI_EMBEDDING_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
       HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      LOGFLAG: ${LOGFLAG}
     restart: unless-stopped
   web-retriever:
     image: ${REGISTRY:-opea}/web-retriever-chroma:${TAG:-latest}
@@ -43,9 +51,11 @@ services:
       TEI_EMBEDDING_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
       GOOGLE_API_KEY: ${GOOGLE_API_KEY}
       GOOGLE_CSE_ID: ${GOOGLE_CSE_ID}
+      LOGFLAG: ${LOGFLAG}
     restart: unless-stopped
   tei-reranking-service:
     image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+    entrypoint: /bin/sh -c "apt-get update && apt-get install -y curl && text-embeddings-router --json-output --model-id ${RERANK_MODEL_ID} --auto-truncate"
     container_name: tei-reranking-server
     ports:
       - "3004:80"
@@ -56,12 +66,18 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-    command: --model-id ${RERANK_MODEL_ID} --auto-truncate
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:3004/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 60
   reranking:
     image: ${REGISTRY:-opea}/reranking-tei:${TAG:-latest}
     container_name: reranking-tei-xeon-server
     depends_on:
-      - tei-reranking-service
+      tei-reranking-service:
+        condition: service_healthy
     ports:
       - "3005:8000"
     ipc: host
@@ -69,8 +85,10 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
+      RERANK_TYPE: ${RERANK_TYPE}
       TEI_RERANKING_ENDPOINT: ${TEI_RERANKING_ENDPOINT}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      LOGFLAG: ${LOGFLAG}
     restart: unless-stopped
   tgi-service:
     image: ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu
@@ -85,12 +103,19 @@ services:
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
       HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:3006/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 100
     command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0
   llm:
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
     container_name: llm-tgi-server
     depends_on:
-      - tgi-service
+      tgi-service:
+        condition: service_healthy
     ports:
       - "3007:9000"
     ipc: host
@@ -98,8 +123,10 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-      TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      LLM_MODEL_ID: ${LLM_MODEL_ID}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      LOGFLAG: ${LOGFLAG}
     restart: unless-stopped
   searchqna-xeon-backend-server:
     image: ${REGISTRY:-opea}/searchqna:${TAG:-latest}
@@ -127,6 +154,7 @@ services:
       - WEB_RETRIEVER_SERVICE_PORT=${WEB_RETRIEVER_SERVICE_PORT}
       - RERANK_SERVICE_PORT=${RERANK_SERVICE_PORT}
       - LLM_SERVICE_PORT=${LLM_SERVICE_PORT}
+      - LOGFLAG=${LOGFLAG}
     ipc: host
     restart: always
   searchqna-xeon-ui-server:

SearchQnA/docker_compose/intel/hpu/gaudi/README.md

@@ -11,7 +11,7 @@ First of all, you need to build Docker Images locally. This step can be ignored
 ```bash
 git clone https://github.com/opea-project/GenAIComps.git
 cd GenAIComps
-docker build --no-cache -t opea/embedding-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/tei/langchain/Dockerfile .
+docker build --no-cache -t opea/embedding-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/src/Dockerfile .
 ```
 
 ### 2. Build Retriever Image
@@ -23,13 +23,13 @@ docker build --no-cache -t opea/web-retriever-chroma:latest --build-arg https_pr
 ### 3. Build Rerank Image
 
 ```bash
-docker build --no-cache -t opea/reranking-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/tei/Dockerfile .
+docker build --no-cache -t opea/reranking-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/src/Dockerfile .
 ```
 
 ### 4. Build LLM Image
 
 ```bash
-docker build --no-cache -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
+docker build --no-cache -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/text-generation/Dockerfile .
 ```
 
 ### 5. Build MegaService Docker Image

SearchQnA/docker_compose/intel/hpu/gaudi/compose.yaml

@@ -4,6 +4,7 @@
 services:
   tei-embedding-service:
     image: ghcr.io/huggingface/tei-gaudi:1.5.0
+    entrypoint: /bin/sh -c "apt-get update && apt-get install -y curl && text-embeddings-router --json-output --model-id ${EMBEDDING_MODEL_ID} --auto-truncate"
     container_name: tei-embedding-gaudi-server
     ports:
       - "3001:80"
@@ -22,12 +23,18 @@ services:
       MAX_WARMUP_SEQUENCE_LENGTH: 512
       INIT_HCCL_ON_ACQUIRE: 0
       ENABLE_EXPERIMENTAL_FLAGS: true
-    command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:3001/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 60
   embedding:
     image: ${REGISTRY:-opea}/embedding-tei:${TAG:-latest}
-    container_name: embedding-tei-server
+    container_name: embedding-tei-gaudi-server
     depends_on:
-      - tei-embedding-service
+      tei-embedding-service:
+        condition: service_healthy
     ports:
       - "3002:6000"
     ipc: host
@@ -36,6 +43,8 @@ services:
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
       TEI_EMBEDDING_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
+      HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      LOGFLAG: ${LOGFLAG}
     restart: unless-stopped
   web-retriever:
     image: ${REGISTRY:-opea}/web-retriever-chroma:${TAG:-latest}
@@ -53,6 +62,7 @@ services:
     restart: unless-stopped
   tei-reranking-service:
     image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+    entrypoint: /bin/sh -c "apt-get update && apt-get install -y curl && text-embeddings-router --json-output --model-id ${RERANK_MODEL_ID} --auto-truncate"
     container_name: tei-reranking-server
     ports:
       - "3004:80"
@@ -63,12 +73,18 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-    command: --model-id ${RERANK_MODEL_ID} --auto-truncate
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:3004/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 60
   reranking:
     image: ${REGISTRY:-opea}/reranking-tei:${TAG:-latest}
-    container_name: reranking-tei-xeon-server
+    container_name: reranking-tei-gaudi-server
     depends_on:
-      - tei-reranking-service
+      tei-reranking-service:
+        condition: service_healthy
     ports:
       - "3005:8000"
     ipc: host
@@ -76,8 +92,10 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
+      RERANK_TYPE: ${RERANK_TYPE}
       TEI_RERANKING_ENDPOINT: ${TEI_RERANKING_ENDPOINT}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      LOGFLAG: ${LOGFLAG}
     restart: unless-stopped
   tgi-service:
     image: ghcr.io/huggingface/tgi-gaudi:2.0.6
@@ -99,6 +117,11 @@ services:
       LIMIT_HPU_GRAPH: true
       USE_FLASH_ATTENTION: true
       FLASH_ATTENTION_RECOMPUTE: true
+    healthcheck:
+      test: ["CMD-SHELL", "sleep 800 && exit 0"]
+      interval: 1s
+      timeout: 805s
+      retries: 1
     runtime: habana
     cap_add:
       - SYS_NICE
@@ -108,7 +131,8 @@ services:
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
     container_name: llm-tgi-gaudi-server
     depends_on:
-      - tgi-service
+      tgi-service:
+        condition: service_healthy
     ports:
       - "3007:9000"
     ipc: host
@@ -116,10 +140,12 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-      TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      LLM_MODEL_ID: ${LLM_MODEL_ID}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       HF_HUB_DISABLE_PROGRESS_BARS: 1
       HF_HUB_ENABLE_HF_TRANSFER: 0
+      LOGFLAG: ${LOGFLAG}
     restart: unless-stopped
   searchqna-gaudi-backend-server:
     image: ${REGISTRY:-opea}/searchqna:${TAG:-latest}
@@ -147,6 +173,7 @@ services:
       - WEB_RETRIEVER_SERVICE_PORT=${WEB_RETRIEVER_SERVICE_PORT}
       - RERANK_SERVICE_PORT=${RERANK_SERVICE_PORT}
       - LLM_SERVICE_PORT=${LLM_SERVICE_PORT}
+      - LOGFLAG=${LOGFLAG}
     ipc: host
     restart: always
   searchqna-gaudi-ui-server:

SearchQnA/docker_image_build/build.yaml

@@ -20,7 +20,7 @@ services:
   embedding-tei:
     build:
       context: GenAIComps
-      dockerfile: comps/embeddings/tei/langchain/Dockerfile
+      dockerfile: comps/embeddings/src/Dockerfile
     extends: searchqna
     image: ${REGISTRY:-opea}/embedding-tei:${TAG:-latest}
   web-retriever-chroma:
@@ -32,12 +32,12 @@ services:
   reranking-tei:
     build:
       context: GenAIComps
-      dockerfile: comps/reranks/tei/Dockerfile
+      dockerfile: comps/reranks/src/Dockerfile
     extends: searchqna
     image: ${REGISTRY:-opea}/reranking-tei:${TAG:-latest}
   llm-tgi:
     build:
       context: GenAIComps
-      dockerfile: comps/llms/text-generation/tgi/Dockerfile
+      dockerfile: comps/llms/src/text-generation/Dockerfile
     extends: searchqna
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}

SearchQnA/searchqna.py

@@ -27,10 +27,21 @@ LLM_SERVICE_HOST_IP = os.getenv("LLM_SERVICE_HOST_IP", "0.0.0.0")
 LLM_SERVICE_PORT = int(os.getenv("LLM_SERVICE_PORT", 9000))
 
 
+def align_outputs(self, data, cur_node, inputs, runtime_graph, llm_parameters_dict, **kwargs):
+    next_data = {}
+    if self.services[cur_node].service_type == ServiceType.EMBEDDING:
+        next_data = {"text": inputs["input"], "embedding": data["data"][0]["embedding"], "k": 1}
+        return next_data
+
+    else:
+        return data
+
+
 class SearchQnAService:
     def __init__(self, host="0.0.0.0", port=8000):
         self.host = host
         self.port = port
+        ServiceOrchestrator.align_outputs = align_outputs
         self.megaservice = ServiceOrchestrator()
         self.endpoint = str(MegaServiceEndpoint.SEARCH_QNA)
 
@@ -88,7 +99,7 @@ class SearchQnAService:
             streaming=stream_opt,
         )
         result_dict, runtime_graph = await self.megaservice.schedule(
-            initial_inputs={"text": prompt}, llm_parameters=parameters
+            initial_inputs={"input": prompt}, llm_parameters=parameters
         )
         for node, response in result_dict.items():
             # Here it suppose the last microservice in the megaservice is LLM.
@@ -99,7 +110,8 @@ class SearchQnAService:
             ):
                 return response
         last_node = runtime_graph.all_leaves()[-1]
-        response = result_dict[last_node]["text"]
+        print(f"================= result: {result_dict[last_node]}")
+        response = result_dict[last_node]["choices"][0]["text"]
         choices = []
         usage = UsageInfo()
         choices.append(

SearchQnA/tests/test_compose_on_gaudi.sh

@@ -38,6 +38,7 @@ function start_services() {
     export EMBEDDING_MODEL_ID=BAAI/bge-base-en-v1.5
     export TEI_EMBEDDING_ENDPOINT=http://$ip_address:3001
     export RERANK_MODEL_ID=BAAI/bge-reranker-base
+    export RERANK_TYPE="tei"
     export TEI_RERANKING_ENDPOINT=http://$ip_address:3004
 
     export TGI_LLM_ENDPOINT=http://$ip_address:3006
@@ -54,34 +55,29 @@ function start_services() {
     export RERANK_SERVICE_PORT=3005
     export LLM_SERVICE_PORT=3007
     export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:3008/v1/searchqna"
+    export host_ip=${ip_address}
+    export LOGFLAG=true
 
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 
     # Start Docker Containers
     docker compose up -d > ${LOG_PATH}/start_services_with_compose.log
-    n=0
-    until [[ "$n" -ge 100 ]]; do
-        docker logs tgi-gaudi-server > $LOG_PATH/tgi_service_start.log
-        if grep -q Connected $LOG_PATH/tgi_service_start.log; then
-            break
-        fi
-        sleep 5s
-        n=$((n+1))
-    done
+
+    sleep 10s
 }
 
 
 function validate_megaservice() {
-    result=$(http_proxy="" curl http://${ip_address}:3008/v1/searchqna -XPOST -d '{"messages": "What is black myth wukong?", "stream": "False"}' -H 'Content-Type: application/json')
+    result=$(curl http://${ip_address}:3008/v1/searchqna -X POST -d '{"messages": "What is the capital of China?", "stream": "False"}' -H 'Content-Type: application/json')
     echo $result
 
     docker logs web-retriever-chroma-server > ${LOG_PATH}/web-retriever-chroma-server.log
     docker logs searchqna-gaudi-backend-server > ${LOG_PATH}/searchqna-gaudi-backend-server.log
     docker logs tei-embedding-gaudi-server > ${LOG_PATH}/tei-embedding-gaudi-server.log
-    docker logs embedding-tei-server > ${LOG_PATH}/embedding-tei-server.log
+    docker logs embedding-tei-gaudi-server > ${LOG_PATH}/embedding-tei-gaudi-server.log
 
-    if [[ $result == *"the"* ]]; then
+    if [[ $result == *"capital"* ]]; then
         echo "Result correct."
     else
         echo "Result wrong."

SearchQnA/tests/test_compose_on_xeon.sh

@@ -36,6 +36,7 @@ function start_services() {
     export EMBEDDING_MODEL_ID=BAAI/bge-base-en-v1.5
     export TEI_EMBEDDING_ENDPOINT=http://$ip_address:3001
     export RERANK_MODEL_ID=BAAI/bge-reranker-base
+    export RERANK_TYPE="tei"
     export TEI_RERANKING_ENDPOINT=http://$ip_address:3004
 
     export TGI_LLM_ENDPOINT=http://$ip_address:3006
@@ -52,35 +53,29 @@ function start_services() {
     export RERANK_SERVICE_PORT=3005
     export LLM_SERVICE_PORT=3007
     export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:3008/v1/searchqna"
+    export host_ip=${ip_address}
+    export LOGFLAG=true
 
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 
     # Start Docker Containers
     docker compose up -d > ${LOG_PATH}/start_services_with_compose.log
-    n=0
-    until [[ "$n" -ge 100 ]]; do
-        docker logs tgi-service > $LOG_PATH/tgi_service_start.log
-        if grep -q Connected $LOG_PATH/tgi_service_start.log; then
-            break
-        fi
-        sleep 5s
-        n=$((n+1))
-    done
+
+    sleep 10s
 }
 
 
 function validate_megaservice() {
-    result=$(http_proxy="" curl http://${ip_address}:3008/v1/searchqna -XPOST -d '{"messages": "What is black myth wukong?", "stream": "False"}' -H 'Content-Type: application/json')
+    result=$(curl http://${ip_address}:3008/v1/searchqna -X POST -d '{"messages": "What is the capital of China?", "stream": "False"}' -H 'Content-Type: application/json')
     echo $result
 
-    if [[ $result == *"the"* ]]; then
-        docker logs web-retriever-chroma-server
-        docker logs searchqna-xeon-backend-server
+    docker logs web-retriever-chroma-server > ${LOG_PATH}/web_retriever.log
+    docker logs searchqna-xeon-backend-server > ${LOG_PATH}/searchqna_backend.log
+
+    if [[ $result == *"capital"* ]]; then
         echo "Result correct."
     else
-        docker logs web-retriever-chroma-server
-        docker logs searchqna-xeon-backend-server
         echo "Result wrong."
         exit 1
     fi

SearchQnA/ui/svelte/src/routes/+page.svelte

@@ -92,37 +92,9 @@
 		);
 
 		eventSource.addEventListener("message", (e: any) => {
-			let Msg = e.data;
-			if (Msg.startsWith("b")) {
-				let trimmedData = Msg.slice(2, -1);
+			let res = e.data;
 
-				if (/\\x[\dA-Fa-f]{2}/.test(trimmedData)) {
-					trimmedData = decodeEscapedBytes(trimmedData);
-				} else if (/\\u[\dA-Fa-f]{4}/.test(trimmedData)) {
-					trimmedData = decodeUnicode(trimmedData);
-				}
-
-				if (trimmedData !== "</s>") {
-				        trimmedData = trimmedData.replace(/\\n/g, "\n");
-				}
-				if (chatMessages[chatMessages.length - 1].role == MessageRole.User) {
-
-					chatMessages = [
-						...chatMessages,
-						{
-							role: MessageRole.Assistant,
-							type: MessageType.Text,
-							content: trimmedData,
-							time: getCurrentTimeStamp(),
-						},
-					];
-					console.log("? chatMessages", chatMessages);
-				} else {
-					let content = chatMessages[chatMessages.length - 1].content as string;
-					chatMessages[chatMessages.length - 1].content = content + trimmedData;
-				}
-				scrollToBottom(scrollToDiv);
-			} else if (Msg === "[DONE]") {
+			if (res === "[DONE]") {
 				let startTime = chatMessages[chatMessages.length - 1].time;
 
 				loading = false;
@@ -133,6 +105,27 @@
 					chatMessages[chatMessages.length - 1].time = totalTime;
 				}
 				storeMessages();
+			} else {
+				let Msg = JSON.parse(res).choices[0].text;
+				if (Msg !== "</s>") {
+					if (chatMessages[chatMessages.length - 1].role == MessageRole.User) {
+						chatMessages = [
+							...chatMessages,
+							{
+								role: MessageRole.Assistant,
+								type: MessageType.Text,
+								content: Msg,
+								time: getCurrentTimeStamp(),
+							},
+						];
+						console.log("? chatMessages", chatMessages);
+					} else {
+						let content = chatMessages[chatMessages.length - 1]
+							.content as string;
+						chatMessages[chatMessages.length - 1].content = content + Msg;
+					}
+					scrollToBottom(scrollToDiv);
+				}
 			}
 		});
 		eventSource.stream();

SearchQnA/ui/svelte/tests/searchQnA.spec.ts

@@ -9,7 +9,7 @@ test.beforeEach(async ({ page }) => {
 });
 
 // Constants definition
-const CHAT_ITEMS = ["What is black myth wukong?"];
+const CHAT_ITEMS = ["What is the capital of China?"];
 
 // Helper function: Enter message to chat
 async function enterMessageToChat(page: Page, message: string) {

Translation/docker_compose/intel/cpu/xeon/README.md

@@ -34,7 +34,7 @@ Follow the instructions below to build the docker images from source.
 ```bash
 git clone https://github.com/opea-project/GenAIComps.git
 cd GenAIComps
-docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
+docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/text-generation/Dockerfile .
 ```
 
 ### 2. Build MegaService Docker Image
@@ -60,7 +60,7 @@ docker build -t opea/translation-ui:latest --build-arg https_proxy=$https_proxy
 
 ```bash
 cd GenAIComps
-docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/nginx/Dockerfile .
+docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/3rd_parties/nginx/src/Dockerfile .
 ```
 
 Then run the command `docker images`, you will have the following Docker Images:

Translation/docker_compose/intel/cpu/xeon/compose.yaml

@@ -14,6 +14,12 @@ services:
       HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       HF_HUB_DISABLE_PROGRESS_BARS: 1
       HF_HUB_ENABLE_HF_TRANSFER: 0
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:8008/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 100
     volumes:
       - "./data:/data"
     shm_size: 1g
@@ -22,7 +28,8 @@ services:
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
     container_name: llm-tgi-server
     depends_on:
-      - tgi-service
+      tgi-service:
+        condition: service_healthy
     ports:
       - "9000:9000"
     ipc: host
@@ -30,7 +37,8 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-      TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      LLM_MODEL_ID: ${LLM_MODEL_ID}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       HF_HUB_DISABLE_PROGRESS_BARS: 1
       HF_HUB_ENABLE_HF_TRANSFER: 0

Translation/docker_compose/intel/hpu/gaudi/README.md

@@ -26,7 +26,7 @@ Follow the instructions below to build the docker images from source.
 ```bash
 git clone https://github.com/opea-project/GenAIComps.git
 cd GenAIComps
-docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
+docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/text-generation/Dockerfile .
 ```
 
 ### 2. Build MegaService Docker Image
@@ -52,7 +52,7 @@ docker build -t opea/translation-ui:latest --build-arg https_proxy=$https_proxy
 
 ```bash
 cd GenAIComps
-docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/nginx/Dockerfile .
+docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/3rd_parties/nginx/src/Dockerfile .
 ```
 
 Then run the command `docker images`, you will have the following four Docker Images:

Translation/docker_compose/intel/hpu/gaudi/compose.yaml

@@ -20,6 +20,11 @@ services:
       LIMIT_HPU_GRAPH: true
       USE_FLASH_ATTENTION: true
       FLASH_ATTENTION_RECOMPUTE: true
+    healthcheck:
+      test: ["CMD-SHELL", "sleep 500 && exit 0"]
+      interval: 1s
+      timeout: 505s
+      retries: 1
     runtime: habana
     cap_add:
       - SYS_NICE
@@ -31,7 +36,8 @@ services:
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
     container_name: llm-tgi-gaudi-server
     depends_on:
-      - tgi-service
+      tgi-service:
+        condition: service_healthy
     ports:
       - "9000:9000"
     ipc: host
@@ -39,7 +45,8 @@ services:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-      TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      LLM_MODEL_ID: ${LLM_MODEL_ID}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       HF_HUB_DISABLE_PROGRESS_BARS: 1
       HF_HUB_ENABLE_HF_TRANSFER: 0

Translation/docker_image_build/build.yaml

@@ -20,12 +20,12 @@ services:
   llm-tgi:
     build:
       context: GenAIComps
-      dockerfile: comps/llms/text-generation/tgi/Dockerfile
+      dockerfile: comps/llms/src/text-generation/Dockerfile
     extends: translation
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
   nginx:
     build:
       context: GenAIComps
-      dockerfile: comps/nginx/Dockerfile
+      dockerfile: comps/3rd_parties/nginx/src/Dockerfile
     extends: translation
     image: ${REGISTRY:-opea}/nginx:${TAG:-latest}

Translation/tests/test_compose_on_xeon.sh

@@ -41,6 +41,7 @@ function start_services() {
     export BACKEND_SERVICE_NAME=translation
     export BACKEND_SERVICE_IP=${ip_address}
     export BACKEND_SERVICE_PORT=8888
+    export host_ip=${ip_address}
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 
@@ -167,7 +168,7 @@ function main() {
 
     validate_microservices
     validate_megaservice
-    validate_frontend
+    # validate_frontend
 
     stop_docker
     echo y | docker system prune

Translation/ui/svelte/.env

@@ -1 +1 @@
-BASE_URL = 'http://10.7.5.135:8888/v1/translation'
+BASE_URL = 'http://backend_address:8888/v1/translation'

VideoQnA/docker_compose/intel/cpu/xeon/README.md

@@ -53,7 +53,7 @@ First of all, you need to build Docker Images locally and install the python pac
 ```bash
 git clone https://github.com/opea-project/GenAIComps.git
 cd GenAIComps
-docker build -t opea/embedding-multimodal-clip:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal_clip/Dockerfile .
+docker build -t opea/embedding-multimodal-clip:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/src/integrations/dependency/clip/Dockerfile .
 ```
 
 ### 2. Build Retriever Image
@@ -65,7 +65,7 @@ docker build -t opea/retriever-vdms:latest --build-arg https_proxy=$https_proxy
 ### 3. Build Rerank Image
 
 ```bash
-docker build -t opea/reranking-videoqna:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy  -f comps/reranks/videoqna/Dockerfile .
+docker build -t opea/reranking-videoqna:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy  -f comps/reranks/src/Dockerfile .
 ```
 
 ### 4. Build LVM Image (Xeon)

VideoQnA/docker_image_build/build.yaml

@@ -26,7 +26,7 @@ services:
   embedding-multimodal-clip:
     build:
       context: GenAIComps
-      dockerfile: comps/embeddings/multimodal_clip/Dockerfile
+      dockerfile: comps/embeddings/src/integrations/dependency/clip/Dockerfile
     extends: videoqna
     image: ${REGISTRY:-opea}/embedding-multimodal-clip:${TAG:-latest}
   retriever-vdms:
@@ -38,7 +38,7 @@ services:
   reranking-videoqna:
     build:
       context: GenAIComps
-      dockerfile: comps/reranks/videoqna/Dockerfile
+      dockerfile: comps/reranks/src/Dockerfile
     extends: videoqna
     image: ${REGISTRY:-opea}/reranking-videoqna:${TAG:-latest}
   video-llama-lvm-server:

VideoQnA/tests/test_compose_on_xeon.sh

@@ -248,4 +248,4 @@ function main() {
 
 }
 
-main
+# main

VisualQnA/docker_compose/amd/gpu/rocm/README.md

@@ -12,7 +12,7 @@ First of all, you need to build Docker Images locally and install the python pac
 git clone https://github.com/opea-project/GenAIComps.git
 cd GenAIComps
 docker build --no-cache -t opea/lvm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/lvms/tgi-llava/Dockerfile .
-docker build --no-cache -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/nginx/Dockerfile .
+docker build --no-cache -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/3rd_parties/nginx/src/Dockerfile .
 ```
 
 ### 2. Build MegaService Docker Image

VisualQnA/docker_compose/intel/cpu/xeon/README.md

@@ -42,7 +42,7 @@ First of all, you need to build Docker Images locally and install the python pac
 git clone https://github.com/opea-project/GenAIComps.git
 cd GenAIComps
 docker build --no-cache -t opea/lvm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/lvms/tgi-llava/Dockerfile .
-docker build --no-cache -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/nginx/Dockerfile .
+docker build --no-cache -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/3rd_parties/nginx/src/Dockerfile .
 ```
 
 ### 2. Build MegaService Docker Image

VisualQnA/docker_compose/intel/cpu/xeon/compose.yaml

@@ -16,12 +16,19 @@ services:
       https_proxy: ${https_proxy}
       HF_HUB_DISABLE_PROGRESS_BARS: 1
       HF_HUB_ENABLE_HF_TRANSFER: 0
+      host_ip: ${host_ip}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:8399/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 60
     command: --model-id ${LVM_MODEL_ID} --max-input-length 4096 --max-total-tokens 8192 --cuda-graphs 0
   lvm-tgi:
     image: ${REGISTRY:-opea}/lvm-tgi:${TAG:-latest}
     container_name: lvm-tgi-xeon-server
     depends_on:
-      - llava-tgi-service
+      llava-tgi-service:
+        condition: service_healthy
     ports:
       - "9399:9399"
     ipc: host

VisualQnA/docker_compose/intel/hpu/gaudi/README.md

@@ -12,7 +12,7 @@ First of all, you need to build Docker Images locally. This step can be ignored
 git clone https://github.com/opea-project/GenAIComps.git
 cd GenAIComps
 docker build --no-cache -t opea/lvm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/lvms/tgi-llava/Dockerfile .
-docker build --no-cache -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/nginx/Dockerfile .
+docker build --no-cache -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/3rd_parties/nginx/src/Dockerfile .
 ```
 
 ### 2. Pull TGI Gaudi Image

VisualQnA/docker_image_build/build.yaml

@@ -26,6 +26,6 @@ services:
   nginx:
     build:
       context: GenAIComps
-      dockerfile: comps/nginx/Dockerfile
+      dockerfile: comps/3rd_parties/nginx/src/Dockerfile
     extends: visualqna
     image: ${REGISTRY:-opea}/nginx:${TAG:-latest}

VisualQnA/tests/test_compose_on_xeon.sh

@@ -41,6 +41,7 @@ function start_services() {
     export BACKEND_SERVICE_IP=${ip_address}
     export BACKEND_SERVICE_PORT=8888
     export NGINX_PORT=80
+    export host_ip=${ip_address}
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 
@@ -48,7 +49,7 @@ function start_services() {
     docker compose up -d > ${LOG_PATH}/start_services_with_compose.log
 
     n=0
-    until [[ "$n" -ge 100 ]]; do
+    until [[ "$n" -ge 200 ]]; do
         docker logs lvm-tgi-xeon-server > ${LOG_PATH}/lvm_tgi_service_start.log
         if grep -q Connected ${LOG_PATH}/lvm_tgi_service_start.log; then
             break

docker_images_list.md

@@ -40,73 +40,71 @@ Take ChatQnA for example. ChatQnA is a chatbot application service based on the
 
 ## Microservice images
 
-| Microservice Images                                                                                                 | Dockerfile                                                                                                                       | Description                                                                                                                                                                                                                |
-| ------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [opea/agent-langchain](https://hub.docker.com/r/opea/comps-agent-langchain)                                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/agent/langchain/Dockerfile)                                    | The docker image exposed the OPEA agent microservice for GenAI application use                                                                                                                                             |
-| [opea/asr](https://hub.docker.com/r/opea/asr)                                                                       | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/asr/src/Dockerfile)                                            | The docker image exposed the OPEA Audio-Speech-Recognition microservice for GenAI application use                                                                                                                          |
-| [opea/chathistory-mongo-server](https://hub.docker.com/r/opea/chathistory-mongo-server)                             | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/chathistory/mongo/Dockerfile)                                  | The docker image exposes OPEA Chat History microservice which based on MongoDB database, designed to allow user to store, retrieve and manage chat conversations                                                           |
-| [opea/dataprep-milvus](https://hub.docker.com/r/opea/dataprep-milvus)                                               | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/milvus/langchain/Dockerfile)                          | The docker image exposed the OPEA dataprep microservice based on milvus vectordb for GenAI application use                                                                                                                 |
-| [opea/dataprep-multimodal-vdms](https://hub.docker.com/r/opea/dataprep-multimodal-vdms)                             | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/vdms/multimodal_langchain/Dockerfile)                 | This docker image exposes an OPEA dataprep microservice based on a multi-modal VDMS for use by GenAI applications.                                                                                                         |
-| [opea/dataprep-multimodal-redis](https://hub.docker.com/r/opea/dataprep-multimodal-redis)                           | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/redis/langchain/Dockerfile)                           | This docker image exposes an OPEA dataprep microservice based on a multi-modal redis for use by GenAI applications.                                                                                                        |
-| [opea/dataprep-on-ray-redis](https://hub.docker.com/r/opea/dataprep-on-ray-redis)                                   | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/redis/langchain_ray/Dockerfile)                       | The docker image exposed the OPEA dataprep microservice based on redis vectordb and optimized ray for GenAI application use                                                                                                |
-| [opea/dataprep-pgvector](https://hub.docker.com/r/opea/dataprep-pgvector)                                           | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/pgvector/langchain/Dockerfile)                        | The docker image exposed the OPEA dataprep microservice based on pgvector vectordb for GenAI application use                                                                                                               |
-| [opea/dataprep-pinecone](https://hub.docker.com/r/opea/dataprep-pinecone)                                           | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/pinecone/langchain/Dockerfile)                        | The docker image exposed the OPEA dataprep microservice based on pincone vectordb for GenAI application use                                                                                                                |
-| [opea/dataprep-qdrant](https://hub.docker.com/r/opea/dataprep-qdrant)                                               | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/qdrant/langchain/Dockerfile)                          | The docker image exposed the OPEA dataprep microservice based on qdrant vectordb for GenAI application use                                                                                                                 |
-| [opea/dataprep-redis](https://hub.docker.com/r/opea/dataprep-redis)                                                 | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/redis/langchain/Dockerfile)                           | The docker image exposed the OPEA dataprep microservice based on redis vectordb Langchain framework for GenAI application use                                                                                              |
-| [opea/dataprep-redis-llama-index](https://hub.docker.com/r/opea/dataprep-redis-llama-index)                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/redis/llama_index/Dockerfile)                         | The docker image exposed the OPEA dataprep microservice based on redis vectordb LlamaIndex framework for GenAI application use                                                                                             |
-| [opea/dataprep-vdms](https://hub.docker.com/r/opea/dataprep-vdms)                                                   | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/vdms/langchain/Dockerfile)                            | This docker image exposes an OPEA dataprep microservice based on VDMS vectordb for use by GenAI applications.                                                                                                              |
-| [opea/embedding-langchain-mosec](https://hub.docker.com/r/opea/embedding-langchain-mosec)                           | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/embeddings/mosec/langchain/Dockerfile)                         | The docker image exposed the OPEA mosec embedding microservice base on Langchain framework for GenAI application use                                                                                                       |
-| [opea/embedding-langchain-mosec-endpoint](https://hub.docker.com/r/opea/embedding-langchain-mosec-endpoint)         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/embeddings/mosec/langchain/dependency/Dockerfile)              | The docker image exposed the OPEA mosec embedding endpoint microservice base on Langchain framework for GenAI application use                                                                                              |
-| [opea/embedding-multimodal-clip](https://hub.docker.com/r/opea/embedding-multimodal-clip)                           | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/embeddings/multimodal_clip/Dockerfile)                         | The docker image exposes OPEA multimodal CLIP-based embedded microservices for use by GenAI applications                                                                                                                   |
-| [opea/embedding-multimodal](https://hub.docker.com/r/opea/embedding-multimodal)                                     | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/embeddings/multimodal/multimodal_langchain/Dockerfile)         | The docker image exposes OPEA multimodal embedded microservices for use by GenAI applications                                                                                                                              |
-| [opea/embedding-multimodal-bridgetower](https://hub.docker.com/r/opea/embedding-multimodal-bridgetower)             | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/embeddings/multimodal/bridgetower/Dockerfile)                  | The docker image exposes OPEA multimodal embedded microservices based on bridgetower for use by GenAI applications                                                                                                         |
-| [opea/embedding-multimodal-bridgetower-gaudi](https://hub.docker.com/r/opea/embedding-multimodal-bridgetower-gaudi) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/embeddings/multimodal/bridgetower/Dockerfile.intel_hpu)        | The docker image exposes OPEA multimodal embedded microservices based on bridgetower for use by GenAI applications on the Gaudi                                                                                            |
-| [opea/embedding-tei](https://hub.docker.com/r/opea/embedding-tei)                                                   | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/embeddings/tei/langchain/Dockerfile)                           | The docker image exposed the OPEA embedding microservice upon tei docker image for GenAI application use                                                                                                                   |
-| [opea/embedding-tei-llama-index](https://hub.docker.com/r/opea/embedding-tei-llama-index)                           | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/embeddings/tei/llama_index/Dockerfile)                         | The docker image exposed the OPEA embedding microservice upon tei docker image base on LlamaIndex framework for GenAI application use                                                                                      |
-| [opea/feedbackmanagement](https://hub.docker.com/r/opea/feedbackmanagement)                                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/feedback_management/mongo/Dockerfile)                          | The docker image exposes that the OPEA feedback management microservice uses a MongoDB database for GenAI applications.                                                                                                    |
-| [opea/finetuning](https://hub.docker.com/r/opea/finetuning)                                                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/finetuning/Dockerfile)                                         | The docker image exposed the OPEA Fine-tuning microservice for GenAI application use                                                                                                                                       |
-| [opea/finetuning-gaudi](https://hub.docker.com/r/opea/finetuning-gaudi)                                             | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/finetuning/Dockerfile.intel_hpu)                               | The docker image exposed the OPEA Fine-tuning microservice for GenAI application use on the Gaudi                                                                                                                          |
-| [opea/gmcrouter](https://hub.docker.com/r/opea/gmcrouter)                                                           | [Link](https://github.com/opea-project/GenAIInfra/blob/main/microservices-connector/Dockerfile.manager)                          | The docker image served as one of key parts of the OPEA GenAI Microservice Connector(GMC) to route the traffic among the microservices defined in GMC                                                                      |
-| [opea/gmcmanager](https://hub.docker.com/r/opea/gmcmanager)                                                         | [Link](https://github.com/opea-project/GenAIInfra/blob/main/microservices-connector/Dockerfile.router)                           | The docker image served as one of key parts of the OPEA GenAI Microservice Connector(GMC) to be controller manager to handle GMC CRD                                                                                       |
-| [opea/guardrails-tgi](https://hub.docker.com/r/opea/guardrails-tgi)                                                 | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/guardrails/llama_guard/langchain/Dockerfile)                   | The docker image exposed the OPEA guardrail microservice to provide content review for GenAI application use                                                                                                               |
-| [opea/guardrails-toxicity-detection](https://hub.docker.com/r/opea/guardrails-toxicity-detection)                   | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/guardrails/toxicity_detection/Dockerfile)                      | The docker image exposed the OPEA guardrail microservice to provide toxicity detection for GenAI application use                                                                                                           |
-| [opea/guardrails-pii-detection](https://hub.docker.com/r/opea/guardrails-pii-detection)                             | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/guardrails/pii_detection/Dockerfile)                           | The docker image exposed the OPEA guardrail microservice to provide PII detection for GenAI application use                                                                                                                |
-| [opea/llm-docsum-tgi](https://hub.docker.com/r/opea/llm-docsum-tgi)                                                 | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/llms/summarization/tgi/langchain/Dockerfile)                   | This docker image is designed to build a document summarization microservice using the HuggingFace Text Generation Inference(TGI) framework. The microservice accepts document input and generates a document summary.     |
-| [opea/llm-faqgen-tgi](https://hub.docker.com/r/opea/llm-faqgen-tgi)                                                 | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/llms/faq-generation/tgi/langchain/Dockerfile)                  | This docker image is designed to build a frequently asked questions microservice using the HuggingFace Text Generation Inference(TGI) framework. The microservice accepts document input and generates a FAQ.              |
-| [opea/llm-native](https://hub.docker.com/r/opea/llm-native)                                                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/llms/text-generation/native/langchain/Dockerfile)              | The docker image exposed the OPEA LLM microservice based on native for GenAI application use                                                                                                                               |
-| [opea/llm-ollama](https://hub.docker.com/r/opea/llm-ollama)                                                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/llms/text-generation/ollama/langchain/Dockerfile)              | The docker image exposed the OPEA LLM microservice based on ollama for GenAI application use                                                                                                                               |
-| [opea/llm-tgi](https://hub.docker.com/r/opea/llm-tgi)                                                               | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/llms/text-generation/tgi/Dockerfile)                           | The docker image exposed the OPEA LLM microservice upon TGI docker image for GenAI application use                                                                                                                         |
-| [opea/llm-vllm](https://hub.docker.com/r/opea/llm-vllm)                                                             | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/llms/text-generation/vllm/langchain/Dockerfile)                | The docker image exposed the OPEA LLM microservice upon vLLM docker image for GenAI application use                                                                                                                        |
-| [opea/llm-vllm-llamaindex](https://hub.docker.com/r/opea/llm-vllm-llamaindex)                                       | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/llms/text-generation/vllm/llama_index/Dockerfile)              | This docker image exposes OPEA LLM microservices to the llamaindex framework's vLLM Docker image for use by GenAI applications                                                                                             |
-| [opea/llava-gaudi](https://hub.docker.com/r/opea/llava-hpu)                                                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/lvms/llava/dependency/Dockerfile.intel_hpu)                    | The docker image exposed the OPEA microservice running LLaVA as a large visual model (LVM) service for GenAI application use on the Gaudi                                                                                  |
-| [opea/lvm-tgi](https://hub.docker.com/r/opea/lvm-tgi)                                                               | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/lvms/tgi-llava/Dockerfile)                                     | This docker image is designed to build a large visual model (LVM) microservice using the HuggingFace Text Generation Inference(TGI) framework. The microservice accepts document input and generates a answer to question. |
-| [opea/lvm-llava](https://hub.docker.com/r/opea/lvm-llava)                                                           | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/lvms/llava/dependency/Dockerfile)                              | The docker image exposed the OPEA microservice running LLaVA as a large visual model (LVM) server for GenAI application use                                                                                                |
-| [opea/lvm-llava-svc](https://hub.docker.com/r/opea/lvm-llava-svc)                                                   | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/lvms/llava/Dockerfile)                                         | The docker image exposed the OPEA microservice running LLaVA as a large visual model (LVM) service for GenAI application use                                                                                               |
-| [opea/lvm-video-llama](https://hub.docker.com/r/opea/lvm-video-llama)                                               | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/lvms/video-llama/Dockerfile)                                   | The docker image exposed the OPEA microservice running Video-Llama as a large visual model (LVM) for GenAI application use                                                                                                 |
-| [opea/nginx](https://hub.docker.com/r/opea/nginx)                                                                   | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/nginx/Dockerfile)                                              | The docker image exposed the OPEA nginx microservice for GenAI application use                                                                                                                                             |
-| [opea/promptregistry-mongo-server](https://hub.docker.com/r/opea/promptregistry-mongo-server)                       | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/prompt_registry/mongo/Dockerfile)                              | The docker image exposes the OPEA Prompt Registry microservices which based on MongoDB database, designed to store and retrieve user's preferred prompts                                                                   |
-| [opea/reranking-videoqna](https://hub.docker.com/r/opea/reranking-videoqna)                                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/reranks/videoqna/Dockerfile)                                   | The docker image exposed the OPEA reranking microservice for reranking the results of VideoQnA use casesfor GenAI application use                                                                                          |
-| [opea/reranking-fastrag](https://hub.docker.com/r/opea/reranking-fastrag)                                           | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/reranks/fastrag/Dockerfile)                                    | The docker image exposed the OPEA reranking microservice base on fastrag for GenAI application use                                                                                                                         |
-| [opea/reranking-langchain-mosec](https://hub.docker.com/r/opea/reranking-langchain-mosec)                           | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/reranks/mosec/langchain/Dockerfile)                            | The docker image exposed the OPEA mosec reranking microservice base on Langchain framework for GenAI application use                                                                                                       |
-| [opea/reranking-langchain-mosec-endpoint](https://hub.docker.com/r/opea/reranking-langchain-mosec-endpoint)         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/reranks/mosec/langchain/dependency/Dockerfile)                 | The docker image exposed the OPEA mosec reranking endpoint microservice base on Langchain framework for GenAI application use                                                                                              |
-| [opea/reranking-tei](https://hub.docker.com/r/opea/reranking-tei)                                                   | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/reranks/tei/Dockerfile)                                        | The docker image exposed the OPEA reranking microservice based on tei docker image for GenAI application use                                                                                                               |
-| [opea/retriever-milvus](https://hub.docker.com/r/opea/retriever-milvus)                                             | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/milvus/langchain/Dockerfile)                        | The docker image exposed the OPEA retrieval microservice based on milvus vectordb for GenAI application use                                                                                                                |
-| [opea/retriever-pathway](https://hub.docker.com/r/opea/retriever-pathway)                                           | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/pathway/langchain/Dockerfile)                       | The docker image exposed the OPEA retrieval microservice with pathway for GenAI application use                                                                                                                            |
-| [opea/retriever-pgvector](https://hub.docker.com/r/opea/retriever-pgvector)                                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/pgvector/langchain/Dockerfile)                      | The docker image exposed the OPEA retrieval microservice based on pgvector vectordb for GenAI application use                                                                                                              |
-| [opea/retriever-pinecone](https://hub.docker.com/r/opea/retriever-pinecone)                                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/pinecone/langchain/Dockerfile)                      | The docker image exposed the OPEA retrieval microservice based on pinecone vectordb for GenAI application use                                                                                                              |
-| [opea/retriever-qdrant](https://hub.docker.com/r/opea/retriever-qdrant)                                             | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/qdrant/haystack/Dockerfile)                         | The docker image exposed the OPEA retrieval microservice based on qdrant vectordb for GenAI application use                                                                                                                |
-| [opea/retriever-redis](https://hub.docker.com/r/opea/retriever-redis)                                               | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/redis/langchain/Dockerfile)                         | The docker image exposed the OPEA retrieval microservice based on redis vectordb for GenAI application use                                                                                                                 |
-| [opea/retriever-redis-llamaindex](https://hub.docker.com/r/opea/retriever-redis-llamaindex)                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/redis/llama_index/Dockerfile)                       | The docker image exposed the OPEA retriever service based on LlamaIndex for GenAI application use                                                                                                                          |
-| [opea/retriever-vdms](https://hub.docker.com/r/opea/retriever-vdms)                                                 | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/vdms/langchain/Dockerfile)                          | The docker image exposed the OPEA retriever service based on Visual Data Management System for GenAI application use                                                                                                       |
-| [opea/speecht5](https://hub.docker.com/r/opea/speecht5)                                                             | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/tts/src/integrations/dependency/speecht5/Dockerfile)           | The docker image exposed the OPEA SpeechT5 service for GenAI application use                                                                                                                                               |
-| [opea/speecht5-gaudi](https://hub.docker.com/r/opea/speecht5-gaudi)                                                 | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/tts/src/integrations/dependency/speecht5/Dockerfile.intel_hpu) | The docker image exposed the OPEA SpeechT5 service on Gaudi2 for GenAI application use                                                                                                                                     |
-| [opea/tei-gaudi](https://hub.docker.com/r/opea/tei-gaudi/tags)                                                      | [Link](https://github.com/huggingface/tei-gaudi/blob/habana-main/Dockerfile-hpu)                                                 | The docker image powered by HuggingFace Text Embedding Inference (TEI) on Gaudi2 for deploying and serving Embedding Models                                                                                                |
-| [opea/vectorstore-pathway](https://hub.docker.com/r/opea/vectorstore-pathway)                                       | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/vectorstores/pathway/Dockerfile)                               | The docker image exposed the OPEA Vectorstores microservice with Pathway for GenAI application use                                                                                                                         |
-| [opea/video-llama-lvm-server](https://hub.docker.com/r/opea/video-llama-lvm-server)                                 | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/lvms/video-llama/dependency/Dockerfile)                        | The docker image exposed the OPEA microservice running Video-Llama as a large visual model (LVM) server for GenAI application use                                                                                          |
-| [opea/tts](https://hub.docker.com/r/opea/tts)                                                                       | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/tts/src/Dockerfile)                                            | The docker image exposed the OPEA Text-To-Speech microservice for GenAI application use                                                                                                                                    |
-| [opea/vllm](https://hub.docker.com/r/opea/vllm)                                                                     | [Link](https://github.com/vllm-project/vllm/blob/main/Dockerfile.cpu)                                                            | The docker image powered by vllm-project for deploying and serving vllm Models                                                                                                                                             |
-| [opea/vllm-gaudi]()                                                                                                 | [Link](https://github.com/HabanaAI/vllm-fork/blob/habana_main/Dockerfile.hpu)                                                    | The docker image powered by vllm-fork for deploying and serving vllm-gaudi Models                                                                                                                                          |
-| [opea/vllm-openvino](https://hub.docker.com/r/opea/vllm-openvino)                                                   | [Link](https://github.com/vllm-project/vllm/blob/main/Dockerfile.openvino)                                                       | The docker image powered by vllm-project for deploying and serving vllm Models of the Openvino Framework                                                                                                                   |
-| [opea/web-retriever-chroma](https://hub.docker.com/r/opea/web-retriever-chroma)                                     | [Link](https://github.com/opea-project/GenAIComps/tree/main/comps/web_retrievers/chroma/langchain/Dockerfile)                    | The docker image exposed the OPEA retrieval microservice based on chroma vectordb for GenAI application use                                                                                                                |
-| [opea/whisper](https://hub.docker.com/r/opea/whisper)                                                               | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/asr/src/integrations/dependency/whisper/Dockerfile)            | The docker image exposed the OPEA Whisper service for GenAI application use                                                                                                                                                |
-| [opea/whisper-gaudi](https://hub.docker.com/r/opea/whisper-gaudi)                                                   | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/asr/src/integrations/dependency/whisper/Dockerfile.intel_hpu)  | The docker image exposed the OPEA Whisper service on Gaudi2 for GenAI application use                                                                                                                                      |
+| Microservice Images                                                                                                 | Dockerfile                                                                                                                                 | Description                                                                                                                                                                                                                |
+| ------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [opea/agent-langchain](https://hub.docker.com/r/opea/comps-agent-langchain)                                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/agent/langchain/Dockerfile)                                              | The docker image exposed the OPEA agent microservice for GenAI application use                                                                                                                                             |
+| [opea/asr](https://hub.docker.com/r/opea/asr)                                                                       | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/asr/whisper/Dockerfile)                                                  | The docker image exposed the OPEA Audio-Speech-Recognition microservice for GenAI application use                                                                                                                          |
+| [opea/chathistory-mongo-server](https://hub.docker.com/r/opea/chathistory-mongo-server)                             | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/chathistory/mongo/Dockerfile)                                            | The docker image exposes OPEA Chat History microservice which based on MongoDB database, designed to allow user to store, retrieve and manage chat conversations                                                           |
+| [opea/dataprep-milvus](https://hub.docker.com/r/opea/dataprep-milvus)                                               | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/milvus/langchain/Dockerfile)                                    | The docker image exposed the OPEA dataprep microservice based on milvus vectordb for GenAI application use                                                                                                                 |
+| [opea/dataprep-multimodal-vdms](https://hub.docker.com/r/opea/dataprep-multimodal-vdms)                             | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/vdms/multimodal_langchain/Dockerfile)                           | This docker image exposes an OPEA dataprep microservice based on a multi-modal VDMS for use by GenAI applications.                                                                                                         |
+| [opea/dataprep-multimodal-redis](https://hub.docker.com/r/opea/dataprep-multimodal-redis)                           | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/redis/langchain/Dockerfile)                                     | This docker image exposes an OPEA dataprep microservice based on a multi-modal redis for use by GenAI applications.                                                                                                        |
+| [opea/dataprep-on-ray-redis](https://hub.docker.com/r/opea/dataprep-on-ray-redis)                                   | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/redis/langchain_ray/Dockerfile)                                 | The docker image exposed the OPEA dataprep microservice based on redis vectordb and optimized ray for GenAI application use                                                                                                |
+| [opea/dataprep-pgvector](https://hub.docker.com/r/opea/dataprep-pgvector)                                           | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/pgvector/langchain/Dockerfile)                                  | The docker image exposed the OPEA dataprep microservice based on pgvector vectordb for GenAI application use                                                                                                               |
+| [opea/dataprep-pinecone](https://hub.docker.com/r/opea/dataprep-pinecone)                                           | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/pinecone/langchain/Dockerfile)                                  | The docker image exposed the OPEA dataprep microservice based on pincone vectordb for GenAI application use                                                                                                                |
+| [opea/dataprep-qdrant](https://hub.docker.com/r/opea/dataprep-qdrant)                                               | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/qdrant/langchain/Dockerfile)                                    | The docker image exposed the OPEA dataprep microservice based on qdrant vectordb for GenAI application use                                                                                                                 |
+| [opea/dataprep-redis](https://hub.docker.com/r/opea/dataprep-redis)                                                 | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/redis/langchain/Dockerfile)                                     | The docker image exposed the OPEA dataprep microservice based on redis vectordb Langchain framework for GenAI application use                                                                                              |
+| [opea/dataprep-redis-llama-index](https://hub.docker.com/r/opea/dataprep-redis-llama-index)                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/redis/llama_index/Dockerfile)                                   | The docker image exposed the OPEA dataprep microservice based on redis vectordb LlamaIndex framework for GenAI application use                                                                                             |
+| [opea/dataprep-vdms](https://hub.docker.com/r/opea/dataprep-vdms)                                                   | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/vdms/langchain/Dockerfile)                                      | This docker image exposes an OPEA dataprep microservice based on VDMS vectordb for use by GenAI applications.                                                                                                              |
+| [opea/embedding-langchain-mosec](https://hub.docker.com/r/opea/embedding-langchain-mosec)                           | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/3rd_parties/nginx/src/Dockerfile)                                        | The docker image exposed the OPEA mosec embedding microservice base on Langchain framework for GenAI application use                                                                                                       |
+| [opea/embedding-langchain-mosec-endpoint](https://hub.docker.com/r/opea/embedding-langchain-mosec-endpoint)         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/3rd_parties/mosec/deployment/docker/Dockerfile)                          | The docker image exposed the OPEA mosec embedding endpoint microservice base on Langchain framework for GenAI application use                                                                                              |
+| [opea/embedding-multimodal-clip](https://hub.docker.com/r/opea/embedding-multimodal-clip)                           | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/embeddings/src/integrations/dependency/clip/Dockerfile)                  | The docker image exposes OPEA multimodal CLIP-based embedded microservices for use by GenAI applications                                                                                                                   |
+| [opea/embedding-multimodal](https://hub.docker.com/r/opea/embedding-multimodal)                                     | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/embeddings/src/Dockerfile)                                               | The docker image exposes OPEA multimodal embedded microservices for use by GenAI applications                                                                                                                              |
+| [opea/embedding-multimodal-bridgetower](https://hub.docker.com/r/opea/embedding-multimodal-bridgetower)             | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/embeddings/src/integrations/dependency/bridgetower/Dockerfile)           | The docker image exposes OPEA multimodal embedded microservices based on bridgetower for use by GenAI applications                                                                                                         |
+| [opea/embedding-multimodal-bridgetower-gaudi](https://hub.docker.com/r/opea/embedding-multimodal-bridgetower-gaudi) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/embeddings/src/integrations/dependency/bridgetower/Dockerfile.intel_hpu) | The docker image exposes OPEA multimodal embedded microservices based on bridgetower for use by GenAI applications on the Gaudi                                                                                            |
+| [opea/embedding-tei](https://hub.docker.com/r/opea/embedding-tei)                                                   | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/embeddings/src/Dockerfile)                                               | The docker image exposed the OPEA embedding microservice upon tei docker image for GenAI application use                                                                                                                   |
+| [opea/feedbackmanagement](https://hub.docker.com/r/opea/feedbackmanagement)                                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/feedback_management/mongo/Dockerfile)                                    | The docker image exposes that the OPEA feedback management microservice uses a MongoDB database for GenAI applications.                                                                                                    |
+| [opea/finetuning](https://hub.docker.com/r/opea/finetuning)                                                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/finetuning/Dockerfile)                                                   | The docker image exposed the OPEA Fine-tuning microservice for GenAI application use                                                                                                                                       |
+| [opea/finetuning-gaudi](https://hub.docker.com/r/opea/finetuning-gaudi)                                             | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/finetuning/Dockerfile.intel_hpu)                                         | The docker image exposed the OPEA Fine-tuning microservice for GenAI application use on the Gaudi                                                                                                                          |
+| [opea/gmcrouter](https://hub.docker.com/r/opea/gmcrouter)                                                           | [Link](https://github.com/opea-project/GenAIInfra/blob/main/microservices-connector/Dockerfile.manager)                                    | The docker image served as one of key parts of the OPEA GenAI Microservice Connector(GMC) to route the traffic among the microservices defined in GMC                                                                      |
+| [opea/gmcmanager](https://hub.docker.com/r/opea/gmcmanager)                                                         | [Link](https://github.com/opea-project/GenAIInfra/blob/main/microservices-connector/Dockerfile.router)                                     | The docker image served as one of key parts of the OPEA GenAI Microservice Connector(GMC) to be controller manager to handle GMC CRD                                                                                       |
+| [opea/guardrails-tgi](https://hub.docker.com/r/opea/guardrails-tgi)                                                 | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/guardrails/llama_guard/langchain/Dockerfile)                             | The docker image exposed the OPEA guardrail microservice to provide content review for GenAI application use                                                                                                               |
+| [opea/guardrails-toxicity-detection](https://hub.docker.com/r/opea/guardrails-toxicity-detection)                   | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/guardrails/toxicity_detection/Dockerfile)                                | The docker image exposed the OPEA guardrail microservice to provide toxicity detection for GenAI application use                                                                                                           |
+| [opea/guardrails-pii-detection](https://hub.docker.com/r/opea/guardrails-pii-detection)                             | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/guardrails/pii_detection/Dockerfile)                                     | The docker image exposed the OPEA guardrail microservice to provide PII detection for GenAI application use                                                                                                                |
+| [opea/llm-docsum-tgi](https://hub.docker.com/r/opea/llm-docsum-tgi)                                                 | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/llms/summarization/tgi/langchain/Dockerfile)                             | This docker image is designed to build a document summarization microservice using the HuggingFace Text Generation Inference(TGI) framework. The microservice accepts document input and generates a document summary.     |
+| [opea/llm-faqgen-tgi](https://hub.docker.com/r/opea/llm-faqgen-tgi)                                                 | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/llms/faq-generation/tgi/langchain/Dockerfile)                            | This docker image is designed to build a frequently asked questions microservice using the HuggingFace Text Generation Inference(TGI) framework. The microservice accepts document input and generates a FAQ.              |
+| [opea/llm-native](https://hub.docker.com/r/opea/llm-native)                                                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/llms/src/text-generation/Dockerfile)                                     | The docker image exposed the OPEA LLM microservice based on native for GenAI application use                                                                                                                               |
+| [opea/llm-ollama](https://hub.docker.com/r/opea/llm-ollama)                                                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/llms/src/text-generation/Dockerfile)                                     | The docker image exposed the OPEA LLM microservice based on ollama for GenAI application use                                                                                                                               |
+| [opea/llm-tgi](https://hub.docker.com/r/opea/llm-tgi)                                                               | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/llms/src/text-generation/Dockerfile)                                     | The docker image exposed the OPEA LLM microservice upon TGI docker image for GenAI application use                                                                                                                         |
+| [opea/llm-vllm](https://hub.docker.com/r/opea/llm-vllm)                                                             | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/llms/src/text-generation/Dockerfile)                                     | The docker image exposed the OPEA LLM microservice upon vLLM docker image for GenAI application use                                                                                                                        |
+| [opea/llava-gaudi](https://hub.docker.com/r/opea/llava-hpu)                                                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/lvms/llava/dependency/Dockerfile.intel_hpu)                              | The docker image exposed the OPEA microservice running LLaVA as a large visual model (LVM) service for GenAI application use on the Gaudi                                                                                  |
+| [opea/lvm-tgi](https://hub.docker.com/r/opea/lvm-tgi)                                                               | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/lvms/tgi-llava/Dockerfile)                                               | This docker image is designed to build a large visual model (LVM) microservice using the HuggingFace Text Generation Inference(TGI) framework. The microservice accepts document input and generates a answer to question. |
+| [opea/lvm-llava](https://hub.docker.com/r/opea/lvm-llava)                                                           | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/lvms/llava/dependency/Dockerfile)                                        | The docker image exposed the OPEA microservice running LLaVA as a large visual model (LVM) server for GenAI application use                                                                                                |
+| [opea/lvm-llava-svc](https://hub.docker.com/r/opea/lvm-llava-svc)                                                   | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/lvms/llava/Dockerfile)                                                   | The docker image exposed the OPEA microservice running LLaVA as a large visual model (LVM) service for GenAI application use                                                                                               |
+| [opea/lvm-video-llama](https://hub.docker.com/r/opea/lvm-video-llama)                                               | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/lvms/video-llama/Dockerfile)                                             | The docker image exposed the OPEA microservice running Video-Llama as a large visual model (LVM) for GenAI application use                                                                                                 |
+| [opea/nginx](https://hub.docker.com/r/opea/nginx)                                                                   | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/3rd_parties/nginx/src/Dockerfile)                                        | The docker image exposed the OPEA nginx microservice for GenAI application use                                                                                                                                             |
+| [opea/promptregistry-mongo-server](https://hub.docker.com/r/opea/promptregistry-mongo-server)                       | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/prompt_registry/mongo/Dockerfile)                                        | The docker image exposes the OPEA Prompt Registry microservices which based on MongoDB database, designed to store and retrieve user's preferred prompts                                                                   |
+| [opea/reranking-videoqna](https://hub.docker.com/r/opea/reranking-videoqna)                                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/reranks/src/Dockerfile)                                                  | The docker image exposed the OPEA reranking microservice for reranking the results of VideoQnA use casesfor GenAI application use                                                                                          |
+| [opea/reranking-fastrag](https://hub.docker.com/r/opea/reranking-fastrag)                                           | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/reranks/src/Dockerfile)                                                  | The docker image exposed the OPEA reranking microservice base on fastrag for GenAI application use                                                                                                                         |
+| [opea/reranking-langchain-mosec](https://hub.docker.com/r/opea/reranking-langchain-mosec)                           | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/reranks/src/Dockerfile)                                                  | The docker image exposed the OPEA mosec reranking microservice base on Langchain framework for GenAI application use                                                                                                       |
+| [opea/reranking-langchain-mosec-endpoint](https://hub.docker.com/r/opea/reranking-langchain-mosec-endpoint)         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/reranks/src/Dockerfile)                                                  | The docker image exposed the OPEA mosec reranking endpoint microservice base on Langchain framework for GenAI application use                                                                                              |
+| [opea/reranking-tei](https://hub.docker.com/r/opea/reranking-tei)                                                   | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/reranks/src/Dockerfile)                                                  | The docker image exposed the OPEA reranking microservice based on tei docker image for GenAI application use                                                                                                               |
+| [opea/retriever-milvus](https://hub.docker.com/r/opea/retriever-milvus)                                             | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/milvus/langchain/Dockerfile)                                  | The docker image exposed the OPEA retrieval microservice based on milvus vectordb for GenAI application use                                                                                                                |
+| [opea/retriever-pathway](https://hub.docker.com/r/opea/retriever-pathway)                                           | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/pathway/langchain/Dockerfile)                                 | The docker image exposed the OPEA retrieval microservice with pathway for GenAI application use                                                                                                                            |
+| [opea/retriever-pgvector](https://hub.docker.com/r/opea/retriever-pgvector)                                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/pgvector/langchain/Dockerfile)                                | The docker image exposed the OPEA retrieval microservice based on pgvector vectordb for GenAI application use                                                                                                              |
+| [opea/retriever-pinecone](https://hub.docker.com/r/opea/retriever-pinecone)                                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/pinecone/langchain/Dockerfile)                                | The docker image exposed the OPEA retrieval microservice based on pinecone vectordb for GenAI application use                                                                                                              |
+| [opea/retriever-qdrant](https://hub.docker.com/r/opea/retriever-qdrant)                                             | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/qdrant/haystack/Dockerfile)                                   | The docker image exposed the OPEA retrieval microservice based on qdrant vectordb for GenAI application use                                                                                                                |
+| [opea/retriever-redis](https://hub.docker.com/r/opea/retriever-redis)                                               | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/redis/langchain/Dockerfile)                                   | The docker image exposed the OPEA retrieval microservice based on redis vectordb for GenAI application use                                                                                                                 |
+| [opea/retriever-redis-llamaindex](https://hub.docker.com/r/opea/retriever-redis-llamaindex)                         | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/redis/llama_index/Dockerfile)                                 | The docker image exposed the OPEA retriever service based on LlamaIndex for GenAI application use                                                                                                                          |
+| [opea/retriever-vdms](https://hub.docker.com/r/opea/retriever-vdms)                                                 | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/retrievers/vdms/langchain/Dockerfile)                                    | The docker image exposed the OPEA retriever service based on Visual Data Management System for GenAI application use                                                                                                       |
+| [opea/speecht5](https://hub.docker.com/r/opea/speecht5)                                                             | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/tts/speecht5/dependency/Dockerfile)                                      | The docker image exposed the OPEA SpeechT5 service for GenAI application use                                                                                                                                               |
+| [opea/speecht5-gaudi](https://hub.docker.com/r/opea/speecht5-gaudi)                                                 | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/tts/speecht5/dependency/Dockerfile.intel_hpu)                            | The docker image exposed the OPEA SpeechT5 service on Gaudi2 for GenAI application use                                                                                                                                     |
+| [opea/tei-gaudi](https://hub.docker.com/r/opea/tei-gaudi/tags)                                                      | [Link](https://github.com/huggingface/tei-gaudi/blob/habana-main/Dockerfile-hpu)                                                           | The docker image powered by HuggingFace Text Embedding Inference (TEI) on Gaudi2 for deploying and serving Embedding Models                                                                                                |
+| [opea/vectorstore-pathway](https://hub.docker.com/r/opea/vectorstore-pathway)                                       | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/vectorstores/pathway/Dockerfile)                                         | The docker image exposed the OPEA Vectorstores microservice with Pathway for GenAI application use                                                                                                                         |
+| [opea/video-llama-lvm-server](https://hub.docker.com/r/opea/video-llama-lvm-server)                                 | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/lvms/video-llama/dependency/Dockerfile)                                  | The docker image exposed the OPEA microservice running Video-Llama as a large visual model (LVM) server for GenAI application use                                                                                          |
+| [opea/tts](https://hub.docker.com/r/opea/tts)                                                                       | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/tts/speecht5/Dockerfile)                                                 | The docker image exposed the OPEA Text-To-Speech microservice for GenAI application use                                                                                                                                    |
+| [opea/vllm](https://hub.docker.com/r/opea/vllm)                                                                     | [Link](https://github.com/vllm-project/vllm/blob/main/Dockerfile.cpu)                                                                      | The docker image powered by vllm-project for deploying and serving vllm Models                                                                                                                                             |
+| [opea/vllm-gaudi]()                                                                                                 | [Link](https://github.com/HabanaAI/vllm-fork/blob/habana_main/Dockerfile.hpu)                                                              | The docker image powered by vllm-fork for deploying and serving vllm-gaudi Models                                                                                                                                          |
+| [opea/vllm-openvino](https://hub.docker.com/r/opea/vllm-openvino)                                                   | [Link](https://github.com/vllm-project/vllm/blob/main/Dockerfile.openvino)                                                                 | The docker image powered by vllm-project for deploying and serving vllm Models of the Openvino Framework                                                                                                                   |
+| [opea/web-retriever-chroma](https://hub.docker.com/r/opea/web-retriever-chroma)                                     | [Link](https://github.com/opea-project/GenAIComps/tree/main/comps/web_retrievers/chroma/langchain/Dockerfile)                              | The docker image exposed the OPEA retrieval microservice based on chroma vectordb for GenAI application use                                                                                                                |
+| [opea/whisper](https://hub.docker.com/r/opea/whisper)                                                               | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/asr/whisper/dependency/Dockerfile)                                       | The docker image exposed the OPEA Whisper service for GenAI application use                                                                                                                                                |
+| [opea/whisper-gaudi](https://hub.docker.com/r/opea/whisper-gaudi)                                                   | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/asr/whisper/dependency/Dockerfile.intel_hpu)                             | The docker image exposed the OPEA Whisper service on Gaudi2 for GenAI application use                                                                                                                                      |