Reduce the verification delay time to 1s

Signed-off-by: ZePan110 <ze.pan@intel.com>

.github/CODEOWNERS

@@ -1,16 +1,18 @@
-* liang1.lv@intel.com feng.tian@intel.com suyue.chen@intel.com
+# Code owners will review PRs within their respective folders.
+
+* liang1.lv@intel.com feng.tian@intel.com suyue.chen@intel.com kaokao.lv@intel.com minmin.hou@intel.com rita.brugarolas.brufau@intel.com
 /.github/ suyue.chen@intel.com ze.pan@intel.com
-/AgentQnA/ kaokao.lv@intel.com minmin.hou@intel.com
+/AgentQnA/ abolfazl.shahbazi@intel.com kaokao.lv@intel.com minmin.hou@intel.com
 /AudioQnA/ sihan.chen@intel.com wenjiao.yue@intel.com
 /AvatarChatbot/ chun.tao@intel.com kaokao.lv@intel.com
 /ChatQnA/ liang1.lv@intel.com letong.han@intel.com
 /CodeGen/ liang1.lv@intel.com
 /CodeTrans/ sihan.chen@intel.com
 /DBQnA/ supriya.krishnamurthi@intel.com liang1.lv@intel.com
-/DocIndexRetriever/ kaokao.lv@intel.com chendi.xue@intel.com
+/DocIndexRetriever/ abolfazl.shahbazi@intel.com kaokao.lv@intel.com chendi.xue@intel.com
 /DocSum/ letong.han@intel.com
 /EdgeCraftRAG/ yongbo.zhu@intel.com mingyuan.qi@intel.com
-/FaqGen/ yogesh.pandey@intel.com
+/FinanceAgent/ abolfazl.shahbazi@intel.com kaokao.lv@intel.com minmin.hou@intel.com rita.brugarolas.brufau@intel.com
 /GraphRAG/ rita.brugarolas.brufau@intel.com abolfazl.shahbazi@intel.com
 /InstructionTuning/ xinyu.ye@intel.com kaokao.lv@intel.com
 /MultimodalQnA/ melanie.h.buehler@intel.com tiep.le@intel.com
@@ -20,4 +22,5 @@
 /Text2Image/ wenjiao.yue@intel.com xinyu.ye@intel.com
 /Translation/ liang1.lv@intel.com sihan.chen@intel.com
 /VideoQnA/ huiling.bao@intel.com
-/VisualQnA/ liang1.lv@intel.com sihan.chen@intel.com
+/VisualQnA/ liang1.lv@intel.com sihan.chen@intel.com
+/WorkflowExecAgent/ joshua.jian.ern.liew@intel.com kaokao.lv@intel.com

.github/env/_build_image.sh

@@ -0,0 +1,5 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+export VLLM_VER=v0.8.3
+export VLLM_FORK_VER=v0.6.6.post1+Gaudi-1.20.0

.github/workflows/_build_image.yml

@@ -75,17 +75,12 @@ jobs:
         run: |
           cd ${{ github.workspace }}/${{ inputs.example }}/docker_image_build
           docker_compose_path=${{ github.workspace }}/${{ inputs.example }}/docker_image_build/build.yaml
+          source ${{ github.workspace }}/.github/env/_build_image.sh
           if [[ $(grep -c "vllm:" ${docker_compose_path}) != 0 ]]; then
-              git clone https://github.com/vllm-project/vllm.git && cd vllm
-              VLLM_VER=v0.8.3
-              echo "Check out vLLM tag ${VLLM_VER}"
-              git checkout ${VLLM_VER} &> /dev/null && cd ../
+              git clone -b ${VLLM_VER} --single-branch https://github.com/vllm-project/vllm.git
           fi
           if [[ $(grep -c "vllm-gaudi:" ${docker_compose_path}) != 0 ]]; then
-              git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
-              VLLM_VER=v0.6.6.post1+Gaudi-1.20.0
-              echo "Check out vLLM tag ${VLLM_VER}"
-              git checkout ${VLLM_VER} &> /dev/null && cd ../
+              git clone -b ${VLLM_FORK_VER} --single-branch https://github.com/HabanaAI/vllm-fork.git
           fi
           git clone --depth 1 --branch ${{ inputs.opea_branch }} https://github.com/opea-project/GenAIComps.git
           cd GenAIComps && git rev-parse HEAD && cd ../

.github/workflows/_helm-e2e.yml

@@ -137,7 +137,7 @@ jobs:
         env:
           example: ${{ inputs.example }}
         run: |
-          if [[ ! "$example" =~ ^[a-zA-Z]{1,20}$ ]] || [[ "$example" =~ \.\. ]] || [[ "$example" == -* || "$example" == *- ]]; then
+          if [[ ! "$example" =~ ^[a-zA-Z0-9]{1,20}$ ]] || [[ "$example" =~ \.\. ]] || [[ "$example" == -* || "$example" == *- ]]; then
             echo "Error: Invalid input - only lowercase alphanumeric and internal hyphens allowed"
             exit 1
           fi

.github/workflows/_run-docker-compose.yml

@@ -204,6 +204,10 @@ jobs:
           if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
 
           echo "Cleaning up images ..."
+          df -h
+          sleep 1
+          docker system df
+          sleep 1
           if [[ "${{ inputs.hardware }}" == "xeon"* ]]; then
               docker system prune -a -f
           else
@@ -213,7 +217,13 @@ jobs:
               docker images --filter reference="opea/comps-base" -q | xargs -r docker rmi && sleep 1s
               docker system prune -f
           fi
+          sleep 5
           docker images
+          sleep 1
+          df -h
+          sleep 1
+          docker system df
+          sleep 1
 
       - name: Publish pipeline artifact
         if: ${{ !cancelled() }}

.github/workflows/daily-update-vllm-version.yml

@@ -0,0 +1,94 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Daily update vLLM & vLLM-fork version
+
+on:
+  schedule:
+    - cron: "30 22 * * *"
+  workflow_dispatch:
+
+env:
+  BRANCH_NAME: "update"
+  USER_NAME: "CICD-at-OPEA"
+  USER_EMAIL: "CICD@opea.dev"
+
+jobs:
+  freeze-tag:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - repo: vLLM
+            repo_name: vllm-project/vllm
+            ver_name: VLLM_VER
+          - repo: vLLM-fork
+            repo_name: HabanaAI/vllm-fork
+            ver_name: VLLM_FORK_VER
+      fail-fast: false
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.ref }}
+
+      - name: Set up Git
+        run: |
+          git config --global user.name ${{ env.USER_NAME }}
+          git config --global user.email ${{ env.USER_EMAIL }}
+          git remote set-url origin https://${{ env.USER_NAME }}:"${{ secrets.ACTION_TOKEN }}"@github.com/${{ github.repository }}.git
+          git fetch
+
+          if git ls-remote https://github.com/${{ github.repository }}.git "refs/heads/${{ env.BRANCH_NAME }}_${{ matrix.repo }}" | grep -q "refs/heads/${{ env.BRANCH_NAME }}_${{ matrix.repo }}"; then
+            echo "branch ${{ env.BRANCH_NAME }}_${{ matrix.repo }} exists"
+            git checkout ${{ env.BRANCH_NAME }}_${{ matrix.repo }}
+          else
+            echo "branch ${{ env.BRANCH_NAME }}_${{ matrix.repo }} not exists"
+            git checkout -b ${{ env.BRANCH_NAME }}_${{ matrix.repo }}
+            git push origin ${{ env.BRANCH_NAME }}_${{ matrix.repo }}
+            echo "branch ${{ env.BRANCH_NAME }}_${{ matrix.repo }} created successfully"
+          fi
+
+      - name: Run script
+        run: |
+          latest_vllm_ver=$(curl -s "https://api.github.com/repos/${{ matrix.repo_name }}/tags" | jq '.[0].name' -)
+          latest_vllm_ver=$(echo "$latest_vllm_ver" | sed 's/"//g')
+          echo "latest_vllm_ver=${latest_vllm_ver}" >> "$GITHUB_ENV"
+          find . -type f -name "*.sh" -exec sed -i "s/${{ matrix.ver_name }}=.*/${{ matrix.ver_name }}=${latest_vllm_ver}/" {} \;
+
+      - name: Commit changes
+        run: |
+          git add .
+          if git diff-index --quiet HEAD --; then
+            echo "No changes detected, skipping commit."
+            exit 1
+          else
+            git commit -s -m "Update ${{ matrix.repo }} version to ${latest_vllm_ver}"
+            git push --set-upstream origin ${{ env.BRANCH_NAME }}_${{ matrix.repo }}
+          fi
+
+      - name: Create Pull Request
+        env:
+          GH_TOKEN: ${{ secrets.ACTION_TOKEN }}
+        run: |
+          pr_count=$(curl -H "Authorization: token ${{ secrets.ACTION_TOKEN }}" -s "https://api.github.com/repos/${{ github.repository }}/pulls?state=all&head=${{ env.USER_NAME }}:${{ env.BRANCH_NAME }}_${{ matrix.repo }}" | jq '. | length')
+          if [ $pr_count -gt 0 ]; then
+            echo "Pull Request exists"
+            pr_number=$(curl -H "Authorization: token ${{ secrets.ACTION_TOKEN }}" -s "https://api.github.com/repos/${{ github.repository }}/pulls?state=all&head=${{ env.USER_NAME }}:${{ env.BRANCH_NAME }}_${{ matrix.repo }}" | jq '.[0].number')
+            gh pr edit ${pr_number} \
+              --title "Update ${{ matrix.repo }} version to ${latest_vllm_ver}" \
+              --body "Update ${{ matrix.repo }} version to ${latest_vllm_ver}"
+            echo "Pull Request updated successfully"
+          else
+            echo "Pull Request does not exists..."
+            gh pr create \
+              -B main \
+              -H ${{ env.BRANCH_NAME }}_${{ matrix.repo }} \
+              --title "Update ${{ matrix.repo }} version to ${latest_vllm_ver}" \
+              --body "Update ${{ matrix.repo }} version to ${latest_vllm_ver}"
+            echo "Pull Request created successfully"
+          fi

.github/workflows/daily_check_issue_and_pr.yml

@@ -26,3 +26,4 @@ jobs:
           close-pr-message: "This PR was closed because it has been stalled for 7 days with no activity."
           repo-token: ${{ secrets.ACTION_TOKEN }}
           start-date: "2025-03-01T00:00:00Z"
+          exempt-issue-labels: "Backlog"

.github/workflows/pr-link-path-scan.yml

@@ -23,6 +23,7 @@ jobs:
       - name: Check the Validity of Hyperlinks
         run: |
           cd ${{github.workspace}}
+          delay=1
           fail="FALSE"
           merged_commit=$(git log -1 --format='%H')
           changed_files="$(git diff --name-status --diff-filter=ARM ${{ github.event.pull_request.base.sha }} ${merged_commit} | awk '/\.md$/ {print $NF}')"
@@ -35,15 +36,20 @@ jobs:
                   # echo $url_line
                   url=$(echo "$url_line"|cut -d '(' -f2 | cut -d ')' -f1|sed 's/\.git$//')
                   path=$(echo "$url_line"|cut -d':' -f1 | cut -d'/' -f2-)
-                  response=$(curl -L -s -o /dev/null -w "%{http_code}" "$url")|| true
-                  if [ "$response" -ne 200 ]; then
-                    echo "**********Validation failed, try again**********"
-                    response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url")
-                    if [ "$response_retry" -eq 200 ]; then
-                      echo "*****Retry successfully*****"
-                    else
-                      echo "Invalid link from ${{github.workspace}}/$path: $url"
-                      fail="TRUE"
+                  if [[ "$url" == "https://platform.openai.com/api-keys"* ]]; then
+                    echo "Link "$url" from ${{github.workspace}}/$path needs to be verified by a real person."
+                  else
+                    sleep $delay
+                    response=$(curl -L -s -o /dev/null -w "%{http_code}" "$url")|| true
+                    if [ "$response" -ne 200 ]; then
+                      echo "**********Validation failed ($response), try again**********"
+                      response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url")
+                      if [ "$response_retry" -eq 200 ]; then
+                        echo "*****Retry successfully*****"
+                      else
+                        echo "Invalid link ($response_retry) from ${{github.workspace}}/$path: $url"
+                        fail="TRUE"
+                      fi
                     fi
                   fi
                 done
@@ -74,6 +80,7 @@ jobs:
       - name: Checking Relative Path Validity
         run: |
           cd ${{github.workspace}}
+          delay=1
           fail="FALSE"
           repo_name=${{ github.event.pull_request.head.repo.full_name }}
           branch="https://github.com/$repo_name/blob/${{ github.event.pull_request.head.ref }}"
@@ -105,14 +112,15 @@ jobs:
                 if [[ "$png_line" == *#* ]]; then
                   if [ -n "changed_files" ] && echo "$changed_files" | grep -q "^${refer_path}$"; then
                     url_dev=$branch$(echo "$real_path" | sed 's|.*/GenAIExamples||')$png_path
+                    sleep $delay
                     response=$(curl -I -L -s -o /dev/null -w "%{http_code}" "$url_dev")
                     if [ "$response" -ne 200 ]; then
-                      echo "**********Validation failed, try again**********"
+                      echo "**********Validation failed ($response), try again**********"
                       response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url_dev")
                       if [ "$response_retry" -eq 200 ]; then
                         echo "*****Retry successfully*****"
                       else
-                        echo "Invalid path from ${{github.workspace}}/$refer_path: $png_path"
+                        echo "Invalid path ($response_retry) from ${{github.workspace}}/$refer_path: $png_path"
                         fail="TRUE"
                       fi
                     else

AgentQnA/README.md

@@ -99,7 +99,7 @@ flowchart LR
 
 #### First, clone the `GenAIExamples` repo.
 
-```
+```bash
 export WORKDIR=<your-work-directory>
 cd $WORKDIR
 git clone https://github.com/opea-project/GenAIExamples.git
@@ -109,7 +109,7 @@ git clone https://github.com/opea-project/GenAIExamples.git
 
 ##### For proxy environments only
 
-```
+```bash
 export http_proxy="Your_HTTP_Proxy"
 export https_proxy="Your_HTTPs_Proxy"
 # Example: no_proxy="localhost, 127.0.0.1, 192.168.1.1"
@@ -118,14 +118,24 @@ export no_proxy="Your_No_Proxy"
 
 ##### For using open-source llms
 
-```
+Set up a [HuggingFace](https://huggingface.co/) account and generate a [user access token](https://huggingface.co/docs/transformers.js/en/guides/private#step-1-generating-a-user-access-token).
+
+Then set an environment variable with the token and another for a directory to download the models:
+
+```bash
 export HUGGINGFACEHUB_API_TOKEN=<your-HF-token>
-export HF_CACHE_DIR=<directory-where-llms-are-downloaded> #so that no need to redownload every time
+export HF_CACHE_DIR=<directory-where-llms-are-downloaded> #  to avoid redownloading models
 ```
 
-##### [Optional] OPANAI_API_KEY to use OpenAI models
+##### [Optional] OPENAI_API_KEY to use OpenAI models or Intel® AI for Enterprise Inference
 
-```
+To use OpenAI models, generate a key following these [instructions](https://platform.openai.com/api-keys).
+
+To use a remote server running Intel® AI for Enterprise Inference, contact the cloud service provider or owner of the on-prem machine for a key to access the desired model on the server.
+
+Then set the environment variable `OPENAI_API_KEY` with the key contents:
+
+```bash
 export OPENAI_API_KEY=<your-openai-key>
 ```
 
@@ -133,16 +143,18 @@ export OPENAI_API_KEY=<your-openai-key>
 
 ##### Gaudi
 
-```
+```bash
 source $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/hpu/gaudi/set_env.sh
 ```
 
 ##### Xeon
 
-```
+```bash
 source $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/cpu/xeon/set_env.sh
 ```
 
+For running
+
 ### 2. Launch the multi-agent system. </br>
 
 We make it convenient to launch the whole system with docker compose, which includes microservices for LLM, agents, UI, retrieval tool, vector database, dataprep, and telemetry. There are 3 docker compose files, which make it easy for users to pick and choose. Users can choose a different retrieval tool other than the `DocIndexRetriever` example provided in our GenAIExamples repo. Users can choose not to launch the telemetry containers.
@@ -184,14 +196,37 @@ docker compose -f $WORKDIR/GenAIExamples/DocIndexRetriever/docker_compose/intel/
 
 #### Launch on Xeon
 
-On Xeon, only OpenAI models are supported. The command below will launch the multi-agent system with the `DocIndexRetriever` as the retrieval tool for the Worker RAG agent.
+On Xeon, OpenAI models and models deployed on a remote server are supported. Both methods require an API key.
 
 ```bash
 export OPENAI_API_KEY=<your-openai-key>
 cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/cpu/xeon
+```
+
+##### OpenAI Models
+
+The command below will launch the multi-agent system with the `DocIndexRetriever` as the retrieval tool for the Worker RAG agent.
+
+```bash
 docker compose -f $WORKDIR/GenAIExamples/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml -f compose_openai.yaml up -d
 ```
 
+##### Models on Remote Server
+
+When models are deployed on a remote server with Intel® AI for Enterprise Inference, a base URL and an API key are required to access them. To run the Agent microservice on Xeon while using models deployed on a remote server, add `compose_remote.yaml` to the `docker compose` command and set additional environment variables.
+
+###### Notes
+
+- `OPENAI_API_KEY` is already set in a previous step.
+- `model` is used to overwrite the value set for this environment variable in `set_env.sh`.
+- `LLM_ENDPOINT_URL` is the base URL given from the owner of the on-prem machine or cloud service provider. It will follow this format: "https://<DNS>". Here is an example: "https://api.inference.example.com".
+
+```bash
+export model=<name-of-model-card>
+export LLM_ENDPOINT_URL=<http-endpoint-of-remote-server>
+docker compose -f $WORKDIR/GenAIExamples/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml -f compose_openai.yaml -f compose_remote.yaml up -d
+```
+
 ### 3. Ingest Data into the vector database
 
 The `run_ingest_data.sh` script will use an example jsonl file to ingest example documents into a vector database. Other ways to ingest data and other types of documents supported can be found in the OPEA dataprep microservice located in the opea-project/GenAIComps repo.
@@ -208,12 +243,18 @@ bash run_ingest_data.sh
 The UI microservice is launched in the previous step with the other microservices.
 To see the UI, open a web browser to `http://${ip_address}:5173` to access the UI. Note the `ip_address` here is the host IP of the UI microservice.
 
-1. `create Admin Account` with a random value
-2. add opea agent endpoint `http://$ip_address:9090/v1` which is a openai compatible api
+1. Click on the arrow above `Get started`. Create an admin account with a name, email, and password.
+2. Add an OpenAI-compatible API endpoint. In the upper right, click on the circle button with the user's initial, go to `Admin Settings`->`Connections`. Under `Manage OpenAI API Connections`, click on the `+` to add a connection. Fill in these fields:
+
+- **URL**: `http://${ip_address}:9090/v1`, do not forget the `v1`
+- **Key**: any value
+- **Model IDs**: any name i.e. `opea-agent`, then press `+` to add it
+
+Click "Save".
 
 ![opea-agent-setting](assets/img/opea-agent-setting.png)
 
-3. test opea agent with ui
+3. Test OPEA agent with UI. Return to `New Chat` and ensure the model (i.e. `opea-agent`) is selected near the upper left. Enter in any prompt to interact with the agent.
 
 ![opea-agent-test](assets/img/opea-agent-test.png)
 

AgentQnA/docker_compose/intel/cpu/xeon/compose_remote.yaml

@@ -0,0 +1,18 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  worker-rag-agent:
+    environment:
+      llm_endpoint_url: ${LLM_ENDPOINT_URL}
+      api_key: ${OPENAI_API_KEY}
+
+  worker-sql-agent:
+    environment:
+      llm_endpoint_url: ${LLM_ENDPOINT_URL}
+      api_key: ${OPENAI_API_KEY}
+
+  supervisor-react-agent:
+    environment:
+      llm_endpoint_url: ${LLM_ENDPOINT_URL}
+      api_key: ${OPENAI_API_KEY}

AgentQnA/tests/step1_build_images.sh

@@ -37,8 +37,8 @@ function build_agent_docker_image_gaudi_vllm() {
     get_genai_comps
 
     git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
-    VLLM_VER=v0.6.6.post1+Gaudi-1.20.0
-    git checkout ${VLLM_VER} &> /dev/null && cd ../
+    VLLM_FORK_VER=v0.6.6.post1+Gaudi-1.20.0
+    git checkout ${VLLM_FORK_VER} &> /dev/null && cd ../
 
     echo "Build agent image with --no-cache..."
     service_list="agent agent-ui vllm-gaudi"

AudioQnA/docker_compose/amd/gpu/rocm/set_env.sh

@@ -6,8 +6,8 @@
 
 # export host_ip=<your External Public IP>    # export host_ip=$(hostname -I | awk '{print $1}')
 
-export host_ip="192.165.1.21"
-export HUGGINGFACEHUB_API_TOKEN=${YOUR_HUGGINGFACEHUB_API_TOKEN}
+export host_ip=${ip_address}
+export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
 # <token>
 
 export LLM_MODEL_ID=Intel/neural-chat-7b-v3-3

AudioQnA/docker_compose/amd/gpu/rocm/set_env_vllm.sh

@@ -6,8 +6,8 @@
 
 # export host_ip=<your External Public IP>    # export host_ip=$(hostname -I | awk '{print $1}')
 
-export host_ip=""
-export external_host_ip=""
+export host_ip=${ip_address}
+export external_host_ip=${ip_address}
 export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
 export HF_CACHE_DIR="./data"
 export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"

AudioQnA/docker_compose/intel/cpu/xeon/set_env.sh

@@ -14,7 +14,8 @@ export MEGA_SERVICE_HOST_IP=${host_ip}
 export WHISPER_SERVER_HOST_IP=${host_ip}
 export SPEECHT5_SERVER_HOST_IP=${host_ip}
 export LLM_SERVER_HOST_IP=${host_ip}
-
+export GPT_SOVITS_SERVER_HOST_IP=${host_ip}
+export GPT_SOVITS_SERVER_PORT=9880
 export WHISPER_SERVER_PORT=7066
 export SPEECHT5_SERVER_PORT=7055
 export LLM_SERVER_PORT=3006

AudioQnA/tests/README.md

@@ -0,0 +1,45 @@
+# AudioQnA E2E test scripts
+
+## Set the required environment variable
+
+```bash
+export HUGGINGFACEHUB_API_TOKEN="Your_Huggingface_API_Token"
+```
+
+## Run test
+
+On Intel Xeon with TGI:
+
+```bash
+bash test_compose_tgi_on_xeon.sh
+```
+
+On Intel Xeon with vLLM:
+
+```bash
+bash test_compose_on_xeon.sh
+```
+
+On Intel Gaudi with TGI:
+
+```bash
+bash test_compose_tgi_on_gaudi.sh
+```
+
+On Intel Gaudi with vLLM:
+
+```bash
+bash test_compose_on_gaudi.sh
+```
+
+On AMD ROCm with TGI:
+
+```bash
+bash test_compose_on_rocm.sh
+```
+
+On AMD ROCm with vLLM:
+
+```bash
+bash test_compose_vllm_on_rocm.sh
+```

AudioQnA/tests/test_compose_multilang_on_xeon.sh

@@ -40,21 +40,8 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/intel/cpu/xeon/
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export LLM_MODEL_ID=meta-llama/Meta-Llama-3-8B-Instruct
-
-    export MEGA_SERVICE_HOST_IP=${ip_address}
-    export WHISPER_SERVER_HOST_IP=${ip_address}
-    export GPT_SOVITS_SERVER_HOST_IP=${ip_address}
-    export LLM_SERVER_HOST_IP=${ip_address}
-
-    export WHISPER_SERVER_PORT=7066
-    export GPT_SOVITS_SERVER_PORT=9880
-    export LLM_SERVER_PORT=3006
-
-    export BACKEND_SERVICE_ENDPOINT=http://${ip_address}:3008/v1/audioqna
     export host_ip=${ip_address}
-
+    source set_env.sh
     # sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 
     # Start Docker Containers

AudioQnA/tests/test_compose_on_gaudi.sh

@@ -27,9 +27,9 @@ function build_docker_images() {
 
     git clone https://github.com/HabanaAI/vllm-fork.git
     cd vllm-fork/
-    VLLM_VER=v0.6.6.post1+Gaudi-1.20.0
-    echo "Check out vLLM tag ${VLLM_VER}"
-    git checkout ${VLLM_VER} &> /dev/null && cd ../
+    VLLM_FORK_VER=v0.6.6.post1+Gaudi-1.20.0
+    echo "Check out vLLM tag ${VLLM_FORK_VER}"
+    git checkout ${VLLM_FORK_VER} &> /dev/null && cd ../
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="audioqna audioqna-ui whisper-gaudi speecht5-gaudi vllm-gaudi"
@@ -40,24 +40,8 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/intel/hpu/gaudi
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export LLM_MODEL_ID=meta-llama/Meta-Llama-3-8B-Instruct
-    export NUM_CARDS=1
-    export BLOCK_SIZE=128
-    export MAX_NUM_SEQS=256
-    export MAX_SEQ_LEN_TO_CAPTURE=2048
-
-    export MEGA_SERVICE_HOST_IP=${ip_address}
-    export WHISPER_SERVER_HOST_IP=${ip_address}
-    export SPEECHT5_SERVER_HOST_IP=${ip_address}
-    export LLM_SERVER_HOST_IP=${ip_address}
-
-    export WHISPER_SERVER_PORT=7066
-    export SPEECHT5_SERVER_PORT=7055
-    export LLM_SERVER_PORT=3006
-
-    export BACKEND_SERVICE_ENDPOINT=http://${ip_address}:3008/v1/audioqna
     export host_ip=${ip_address}
+    source set_env.sh
     # sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 
     # Start Docker Containers

AudioQnA/tests/test_compose_on_rocm.sh

@@ -35,20 +35,7 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/amd/gpu/rocm/
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export LLM_MODEL_ID=Intel/neural-chat-7b-v3-3
-
-    export MEGA_SERVICE_HOST_IP=${ip_address}
-    export WHISPER_SERVER_HOST_IP=${ip_address}
-    export SPEECHT5_SERVER_HOST_IP=${ip_address}
-    export LLM_SERVER_HOST_IP=${ip_address}
-
-    export WHISPER_SERVER_PORT=7066
-    export SPEECHT5_SERVER_PORT=7055
-    export LLM_SERVER_PORT=3006
-
-    export BACKEND_SERVICE_ENDPOINT=http://${ip_address}:3008/v1/audioqna
-
+    source set_env.sh
     # Start Docker Containers
     docker compose up -d > ${LOG_PATH}/start_services_with_compose.log
     n=0

AudioQnA/tests/test_compose_on_xeon.sh

@@ -40,21 +40,8 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/intel/cpu/xeon/
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export LLM_MODEL_ID=meta-llama/Meta-Llama-3-8B-Instruct
-
-    export MEGA_SERVICE_HOST_IP=${ip_address}
-    export WHISPER_SERVER_HOST_IP=${ip_address}
-    export SPEECHT5_SERVER_HOST_IP=${ip_address}
-    export LLM_SERVER_HOST_IP=${ip_address}
-
-    export WHISPER_SERVER_PORT=7066
-    export SPEECHT5_SERVER_PORT=7055
-    export LLM_SERVER_PORT=3006
-
-    export BACKEND_SERVICE_ENDPOINT=http://${ip_address}:3008/v1/audioqna
     export host_ip=${ip_address}
-
+    source set_env.sh
     # sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 
     # Start Docker Containers

AudioQnA/tests/test_compose_tgi_on_gaudi.sh

@@ -34,21 +34,8 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/intel/hpu/gaudi
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export LLM_MODEL_ID=meta-llama/Meta-Llama-3-8B-Instruct
-
-    export MEGA_SERVICE_HOST_IP=${ip_address}
-    export WHISPER_SERVER_HOST_IP=${ip_address}
-    export SPEECHT5_SERVER_HOST_IP=${ip_address}
-    export LLM_SERVER_HOST_IP=${ip_address}
-
-    export WHISPER_SERVER_PORT=7066
-    export SPEECHT5_SERVER_PORT=7055
-    export LLM_SERVER_PORT=3006
-
-    export BACKEND_SERVICE_ENDPOINT=http://${ip_address}:3008/v1/audioqna
     export host_ip=${ip_address}
-
+    source set_env.sh
     # Start Docker Containers
     docker compose -f compose_tgi.yaml up -d > ${LOG_PATH}/start_services_with_compose.log
     n=0

AudioQnA/tests/test_compose_tgi_on_xeon.sh

@@ -34,21 +34,8 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/intel/cpu/xeon/
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export LLM_MODEL_ID=meta-llama/Meta-Llama-3-8B-Instruct
-
-    export MEGA_SERVICE_HOST_IP=${ip_address}
-    export WHISPER_SERVER_HOST_IP=${ip_address}
-    export SPEECHT5_SERVER_HOST_IP=${ip_address}
-    export LLM_SERVER_HOST_IP=${ip_address}
-
-    export WHISPER_SERVER_PORT=7066
-    export SPEECHT5_SERVER_PORT=7055
-    export LLM_SERVER_PORT=3006
-
-    export BACKEND_SERVICE_ENDPOINT=http://${ip_address}:3008/v1/audioqna
     export host_ip=${ip_address}
-
+    source set_env.sh
     # Start Docker Containers
     docker compose -f compose_tgi.yaml up -d > ${LOG_PATH}/start_services_with_compose.log
     n=0

AudioQnA/tests/test_compose_vllm_on_rocm.sh

@@ -33,27 +33,7 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/amd/gpu/rocm/
-
-    export host_ip=${ip_address}
-    export external_host_ip=${ip_address}
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export HF_CACHE_DIR="./data"
-    export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
-    export VLLM_SERVICE_PORT="8081"
-
-    export MEGA_SERVICE_HOST_IP=${host_ip}
-    export WHISPER_SERVER_HOST_IP=${host_ip}
-    export SPEECHT5_SERVER_HOST_IP=${host_ip}
-    export LLM_SERVER_HOST_IP=${host_ip}
-
-    export WHISPER_SERVER_PORT=7066
-    export SPEECHT5_SERVER_PORT=7055
-    export LLM_SERVER_PORT=${VLLM_SERVICE_PORT}
-    export BACKEND_SERVICE_PORT=3008
-    export FRONTEND_SERVICE_PORT=5173
-
-    export BACKEND_SERVICE_ENDPOINT=http://${external_host_ip}:${BACKEND_SERVICE_PORT}/v1/audioqna
-
+    source set_env_vllm.sh
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 
     # Start Docker Containers

AvatarChatbot/Dockerfile

@@ -1,8 +1,9 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
+ARG IMAGE_REPO=opea
 ARG BASE_TAG=latest
-FROM opea/comps-base:$BASE_TAG
+FROM $IMAGE_REPO/comps-base:$BASE_TAG
 
 COPY ./avatarchatbot.py $HOME/avatarchatbot.py
 

AvatarChatbot/docker_compose/amd/gpu/rocm/set_env.sh

@@ -41,7 +41,7 @@ export FACE="/home/user/comps/animation/src/assets/img/avatar5.png"
 # export AUDIO='assets/audio/eg3_ref.wav' # audio file path is optional, will use base64str in the post request as input if is 'None'
 export AUDIO='None'
 export FACESIZE=96
-export OUTFILE="/outputs/result.mp4"
+export OUTFILE="./outputs/result.mp4"
 export GFPGAN_MODEL_VERSION=1.4 # latest version, can roll back to v1.3 if needed
 export UPSCALE_FACTOR=1
-export FPS=10
+export FPS=5

AvatarChatbot/docker_compose/intel/cpu/xeon/set_env.sh

@@ -5,3 +5,32 @@
 pushd "../../../../../" > /dev/null
 source .set_env.sh
 popd > /dev/null
+
+export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export host_ip=$(hostname -I | awk '{print $1}')
+export LLM_MODEL_ID=Intel/neural-chat-7b-v3-3
+export WAV2LIP_ENDPOINT=http://$host_ip:7860
+export MEGA_SERVICE_HOST_IP=${host_ip}
+export WHISPER_SERVER_HOST_IP=${host_ip}
+export WHISPER_SERVER_PORT=7066
+export SPEECHT5_SERVER_HOST_IP=${host_ip}
+export SPEECHT5_SERVER_PORT=7055
+export LLM_SERVER_HOST_IP=${host_ip}
+export LLM_SERVER_PORT=3006
+export ANIMATION_SERVICE_HOST_IP=${host_ip}
+export ANIMATION_SERVICE_PORT=3008
+
+export MEGA_SERVICE_PORT=8888
+
+export DEVICE="cpu"
+export WAV2LIP_PORT=7860
+export INFERENCE_MODE='wav2lip+gfpgan'
+export CHECKPOINT_PATH='/usr/local/lib/python3.11/site-packages/Wav2Lip/checkpoints/wav2lip_gan.pth'
+export FACE="/home/user/comps/animation/src/assets/img/avatar5.png"
+# export AUDIO='assets/audio/eg3_ref.wav' # audio file path is optional, will use base64str in the post request as input if is 'None'
+export AUDIO='None'
+export FACESIZE=96
+export OUTFILE="/outputs/result.mp4"
+export GFPGAN_MODEL_VERSION=1.4 # latest version, can roll back to v1.3 if needed
+export UPSCALE_FACTOR=1
+export FPS=10

AvatarChatbot/docker_compose/intel/hpu/gaudi/set_env.sh

@@ -5,3 +5,35 @@
 pushd "../../../../../" > /dev/null
 source .set_env.sh
 popd > /dev/null
+
+export HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN
+export host_ip=$(hostname -I | awk '{print $1}')
+
+export LLM_MODEL_ID=Intel/neural-chat-7b-v3-3
+
+export WAV2LIP_ENDPOINT=http://$host_ip:7860
+
+export MEGA_SERVICE_HOST_IP=${host_ip}
+export WHISPER_SERVER_HOST_IP=${host_ip}
+export WHISPER_SERVER_PORT=7066
+export SPEECHT5_SERVER_HOST_IP=${host_ip}
+export SPEECHT5_SERVER_PORT=7055
+export LLM_SERVER_HOST_IP=${host_ip}
+export LLM_SERVER_PORT=3006
+export ANIMATION_SERVICE_HOST_IP=${host_ip}
+export ANIMATION_SERVICE_PORT=3008
+
+export MEGA_SERVICE_PORT=8888
+
+export DEVICE="hpu"
+export WAV2LIP_PORT=7860
+export INFERENCE_MODE='wav2lip+gfpgan'
+export CHECKPOINT_PATH='/usr/local/lib/python3.10/dist-packages/Wav2Lip/checkpoints/wav2lip_gan.pth'
+export FACE="/home/user/comps/animation/src/assets/img/avatar1.jpg"
+# export AUDIO='assets/audio/eg3_ref.wav' # audio file path is optional, will use base64str in the post request as input if is 'None'
+export AUDIO='None'
+export FACESIZE=96
+export OUTFILE="/outputs/result.mp4"
+export GFPGAN_MODEL_VERSION=1.4 # latest version, can roll back to v1.3 if needed
+export UPSCALE_FACTOR=1
+export FPS=10

AvatarChatbot/docker_image_build/build.yaml

@@ -5,6 +5,8 @@ services:
   avatarchatbot:
     build:
       args:
+        IMAGE_REPO: ${REGISTRY:-opea}
+        BASE_TAG: ${TAG:-latest}
         http_proxy: ${http_proxy}
         https_proxy: ${https_proxy}
         no_proxy: ${no_proxy}

AvatarChatbot/tests/README.md

@@ -0,0 +1,27 @@
+# AvatarChatbot E2E test scripts
+
+## Set the required environment variable
+
+```bash
+export HUGGINGFACEHUB_API_TOKEN="Your_Huggingface_API_Token"
+```
+
+## Run test
+
+On Intel Xeon with TGI:
+
+```bash
+bash test_compose_on_xeon.sh
+```
+
+On Intel Gaudi with TGI:
+
+```bash
+bash test_compose_on_gaudi.sh
+```
+
+On AMD ROCm with TGI:
+
+```bash
+bash test_compose_on_rocm.sh
+```

AvatarChatbot/tests/test_compose_on_gaudi.sh

@@ -24,19 +24,13 @@ ip_address=$(hostname -I | awk '{print $1}')
 
 function build_docker_images() {
     opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
 
     cd $WORKPATH/docker_image_build
     git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="avatarchatbot whisper-gaudi speecht5-gaudi wav2lip-gaudi animation"
@@ -51,37 +45,7 @@ function build_docker_images() {
 function start_services() {
     cd $WORKPATH/docker_compose/intel/hpu/gaudi
 
-    export HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN
-    export host_ip=$(hostname -I | awk '{print $1}')
-
-    export LLM_MODEL_ID=Intel/neural-chat-7b-v3-3
-
-    export WAV2LIP_ENDPOINT=http://$host_ip:7860
-
-    export MEGA_SERVICE_HOST_IP=${host_ip}
-    export WHISPER_SERVER_HOST_IP=${host_ip}
-    export WHISPER_SERVER_PORT=7066
-    export SPEECHT5_SERVER_HOST_IP=${host_ip}
-    export SPEECHT5_SERVER_PORT=7055
-    export LLM_SERVER_HOST_IP=${host_ip}
-    export LLM_SERVER_PORT=3006
-    export ANIMATION_SERVICE_HOST_IP=${host_ip}
-    export ANIMATION_SERVICE_PORT=3008
-
-    export MEGA_SERVICE_PORT=8888
-
-    export DEVICE="hpu"
-    export WAV2LIP_PORT=7860
-    export INFERENCE_MODE='wav2lip+gfpgan'
-    export CHECKPOINT_PATH='/usr/local/lib/python3.10/dist-packages/Wav2Lip/checkpoints/wav2lip_gan.pth'
-    export FACE="/home/user/comps/animation/src/assets/img/avatar1.jpg"
-    # export AUDIO='assets/audio/eg3_ref.wav' # audio file path is optional, will use base64str in the post request as input if is 'None'
-    export AUDIO='None'
-    export FACESIZE=96
-    export OUTFILE="/outputs/result.mp4"
-    export GFPGAN_MODEL_VERSION=1.4 # latest version, can roll back to v1.3 if needed
-    export UPSCALE_FACTOR=1
-    export FPS=10
+    source set_env.sh
 
     # Start Docker Containers
     docker compose up -d > ${LOG_PATH}/start_services_with_compose.log
@@ -128,19 +92,29 @@ function stop_docker() {
 
 
 function main() {
+    echo "::group::stop_docker"
     stop_docker
-    echo y | docker builder prune --all
-    echo y | docker image prune
+    echo "::endgroup::"
+    docker builder prune --all -f
+    docker image prune -f
 
+    echo "::group::build_docker_images"
     if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
-    start_services
-    # validate_microservices
-    validate_megaservice
-    # validate_frontend
+    echo "::endgroup::"
 
+    echo "::group::start_services"
+    start_services
+    echo "::endgroup::"
+
+    echo "::group::validate_megaservice"
+    validate_megaservice
+    echo "::endgroup::"
+
+    echo "::group::stop_docker"
     stop_docker
-    echo y | docker builder prune --all
-    echo y | docker image prune
+    echo "::endgroup::"
+    docker builder prune --all -f
+    docker image prune -f
 
 }
 

AvatarChatbot/tests/test_compose_on_rocm.sh

@@ -2,7 +2,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-set -e
+set -xe
 IMAGE_REPO=${IMAGE_REPO:-"opea"}
 IMAGE_TAG=${IMAGE_TAG:-"latest"}
 echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
@@ -25,6 +25,10 @@ ip_address=$(hostname -I | awk '{print $1}')
 function build_docker_images() {
     cd $WORKPATH/docker_image_build
     git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="avatarchatbot whisper asr speecht5 tts wav2lip animation"
@@ -38,48 +42,8 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/amd/gpu/rocm
-
-    export HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN
     export OPENAI_API_KEY=$OPENAI_API_KEY
-    export host_ip=${ip_address}
-
-    export TGI_SERVICE_PORT=3006
-    export TGI_LLM_ENDPOINT=http://${host_ip}:${TGI_SERVICE_PORT}
-    export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
-
-    export ASR_ENDPOINT=http://${host_ip}:7066
-    export TTS_ENDPOINT=http://${host_ip}:7055
-    export WAV2LIP_ENDPOINT=http://${host_ip}:7860
-
-    export MEGA_SERVICE_HOST_IP=${host_ip}
-    export ASR_SERVICE_HOST_IP=${host_ip}
-    export TTS_SERVICE_HOST_IP=${host_ip}
-    export LLM_SERVICE_HOST_IP=${host_ip}
-    export ANIMATION_SERVICE_HOST_IP=${host_ip}
-    export WHISPER_SERVER_HOST_IP=${host_ip}
-    export WHISPER_SERVER_PORT=7066
-
-    export SPEECHT5_SERVER_HOST_IP=${host_ip}
-    export SPEECHT5_SERVER_PORT=7055
-
-    export MEGA_SERVICE_PORT=8888
-    export ASR_SERVICE_PORT=3001
-    export TTS_SERVICE_PORT=3002
-    export LLM_SERVICE_PORT=3006
-    export ANIMATION_SERVICE_PORT=3008
-
-    export DEVICE="cpu"
-    export WAV2LIP_PORT=7860
-    export INFERENCE_MODE='wav2lip+gfpgan'
-    export CHECKPOINT_PATH='/usr/local/lib/python3.11/site-packages/Wav2Lip/checkpoints/wav2lip_gan.pth'
-    export FACE="/home/user/comps/animation/src/assets/img/avatar5.png"
-    # export AUDIO='assets/audio/eg3_ref.wav' # audio file path is optional, will use base64str in the post request as input if is 'None'
-    export AUDIO='None'
-    export FACESIZE=96
-    export OUTFILE="./outputs/result.mp4"
-    export GFPGAN_MODEL_VERSION=1.4 # latest version, can roll back to v1.3 if needed
-    export UPSCALE_FACTOR=1
-    export FPS=5
+    source set_env.sh
 
     # Start Docker Containers
     docker compose up -d --force-recreate
@@ -138,11 +102,6 @@ function validate_megaservice() {
 }
 
 
-#function validate_frontend() {
-
-#}
-
-
 function stop_docker() {
     cd $WORKPATH/docker_compose/amd/gpu/rocm
     docker compose down && docker compose rm -f
@@ -151,19 +110,27 @@ function stop_docker() {
 
 function main() {
 
-    echo $OPENAI_API_KEY
-    echo $OPENAI_KEY
-
+    echo "::group::stop_docker"
     stop_docker
+    echo "::endgroup::"
+
+    echo "::group::build_docker_images"
     if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
-    start_services
-    # validate_microservices
-    sleep 30
-    validate_megaservice
-    # validate_frontend
-    stop_docker
+    echo "::endgroup::"
 
-    echo y | docker system prune
+    echo "::group::start_services"
+    start_services
+    echo "::endgroup::"
+
+    echo "::group::validate_megaservice"
+    validate_megaservice
+    echo "::endgroup::"
+
+    echo "::group::stop_docker"
+    stop_docker
+    echo "::endgroup::"
+
+    docker system prune -f
 
 }
 

AvatarChatbot/tests/test_compose_on_xeon.sh

@@ -24,19 +24,13 @@ ip_address=$(hostname -I | awk '{print $1}')
 
 function build_docker_images() {
     opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
 
     cd $WORKPATH/docker_image_build
     git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="avatarchatbot whisper speecht5 wav2lip animation"
@@ -51,37 +45,7 @@ function build_docker_images() {
 function start_services() {
     cd $WORKPATH/docker_compose/intel/cpu/xeon
 
-    export HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN
-    export host_ip=$(hostname -I | awk '{print $1}')
-
-    export LLM_MODEL_ID=Intel/neural-chat-7b-v3-3
-
-    export WAV2LIP_ENDPOINT=http://$host_ip:7860
-
-    export MEGA_SERVICE_HOST_IP=${host_ip}
-    export WHISPER_SERVER_HOST_IP=${host_ip}
-    export WHISPER_SERVER_PORT=7066
-    export SPEECHT5_SERVER_HOST_IP=${host_ip}
-    export SPEECHT5_SERVER_PORT=7055
-    export LLM_SERVER_HOST_IP=${host_ip}
-    export LLM_SERVER_PORT=3006
-    export ANIMATION_SERVICE_HOST_IP=${host_ip}
-    export ANIMATION_SERVICE_PORT=3008
-
-    export MEGA_SERVICE_PORT=8888
-
-    export DEVICE="cpu"
-    export WAV2LIP_PORT=7860
-    export INFERENCE_MODE='wav2lip+gfpgan'
-    export CHECKPOINT_PATH='/usr/local/lib/python3.11/site-packages/Wav2Lip/checkpoints/wav2lip_gan.pth'
-    export FACE="/home/user/comps/animation/src/assets/img/avatar5.png"
-    # export AUDIO='assets/audio/eg3_ref.wav' # audio file path is optional, will use base64str in the post request as input if is 'None'
-    export AUDIO='None'
-    export FACESIZE=96
-    export OUTFILE="/outputs/result.mp4"
-    export GFPGAN_MODEL_VERSION=1.4 # latest version, can roll back to v1.3 if needed
-    export UPSCALE_FACTOR=1
-    export FPS=10
+    source set_env.sh
 
     # Start Docker Containers
     docker compose up -d
@@ -127,16 +91,28 @@ function stop_docker() {
 
 
 function main() {
-    stop_docker
-    if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
-    start_services
-    # validate_microservices
-    validate_megaservice
-    # validate_frontend
-    stop_docker
 
-    echo y | docker builder prune --all
-    echo y | docker image prune
+    echo "::group::stop_docker"
+    stop_docker
+    echo "::endgroup::"
+
+    echo "::group::build_docker_images"
+    if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
+    echo "::endgroup::"
+
+    echo "::group::start_services"
+    start_services
+    echo "::endgroup::"
+
+    echo "::group::validate_megaservice"
+    validate_megaservice
+    echo "::endgroup::"
+
+    echo "::group::stop_docker"
+    stop_docker
+    echo "::endgroup::"
+
+    docker system prune -f
 
 }
 

ChatQnA/docker_compose/amd/gpu/rocm/compose.yaml

@@ -16,7 +16,7 @@ services:
       - chatqna-redis-vector-db
       - chatqna-tei-embedding-service
     ports:
-      - "${CHATQNA_REDIS_DATAPREP_PORT}:5000"
+      - "${CHATQNA_REDIS_DATAPREP_PORT:-18103}:5000"
     environment:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}

ChatQnA/docker_compose/amd/gpu/rocm/compose_faqgen.yaml

@@ -16,7 +16,7 @@ services:
       - chatqna-redis-vector-db
       - chatqna-tei-embedding-service
     ports:
-      - "${CHATQNA_REDIS_DATAPREP_PORT}:5000"
+      - "${CHATQNA_REDIS_DATAPREP_PORT:-18103}:5000"
     environment:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}

ChatQnA/docker_compose/amd/gpu/rocm/compose_faqgen_vllm.yaml

@@ -16,7 +16,7 @@ services:
       - chatqna-redis-vector-db
       - chatqna-tei-embedding-service
     ports:
-      - "${CHATQNA_REDIS_DATAPREP_PORT}:5000"
+      - "${CHATQNA_REDIS_DATAPREP_PORT:-18103}:5000"
     environment:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}

ChatQnA/docker_compose/amd/gpu/rocm/compose_vllm.yaml

@@ -16,7 +16,7 @@ services:
       - chatqna-redis-vector-db
       - chatqna-tei-embedding-service
     ports:
-      - "${CHATQNA_REDIS_DATAPREP_PORT:-5000}:5000"
+      - "${CHATQNA_REDIS_DATAPREP_PORT:-18103}:5000"
     environment:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}

ChatQnA/docker_compose/amd/gpu/rocm/set_env.sh

@@ -2,17 +2,17 @@
 
 # Copyright (C) 2025 Advanced Micro Devices, Inc.
 
-export HOST_IP=''
-export HOST_IP_EXTERNAL=''
+export HOST_IP=${ip_address}
+export HOST_IP_EXTERNAL=${ip_address}
 
 export CHATQNA_EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
 export CHATQNA_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
 export CHATQNA_LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
 export CHATQNA_RERANK_MODEL_ID="BAAI/bge-reranker-base"
 
-export CHATQNA_BACKEND_SERVICE_PORT=18102
-export CHATQNA_FRONTEND_SERVICE_PORT=18101
-export CHATQNA_NGINX_PORT=18104
+export CHATQNA_BACKEND_SERVICE_PORT=8888
+export CHATQNA_FRONTEND_SERVICE_PORT=5173
+export CHATQNA_NGINX_PORT=80
 export CHATQNA_REDIS_DATAPREP_PORT=18103
 export CHATQNA_REDIS_RETRIEVER_PORT=7000
 export CHATQNA_REDIS_VECTOR_INSIGHT_PORT=8001

ChatQnA/docker_compose/amd/gpu/rocm/set_env_faqgen.sh

@@ -2,18 +2,18 @@
 
 # Copyright (C) 2025 Advanced Micro Devices, Inc.
 
-export HOST_IP=''
-export HOST_IP_EXTERNAL=''
+export HOST_IP=${ip_address}
+export HOST_IP_EXTERNAL=${ip_address}
 
 export CHATQNA_EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
 export CHATQNA_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
 export CHATQNA_LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
 export CHATQNA_RERANK_MODEL_ID="BAAI/bge-reranker-base"
 
-export CHATQNA_BACKEND_SERVICE_PORT=18102
-export CHATQNA_FRONTEND_SERVICE_PORT=18101
+export CHATQNA_BACKEND_SERVICE_PORT=8888
+export CHATQNA_FRONTEND_SERVICE_PORT=5173
 export CHATQNA_LLM_FAQGEN_PORT=18011
-export CHATQNA_NGINX_PORT=18104
+export CHATQNA_NGINX_PORT=80
 export CHATQNA_REDIS_DATAPREP_PORT=18103
 export CHATQNA_REDIS_RETRIEVER_PORT=7000
 export CHATQNA_REDIS_VECTOR_INSIGHT_PORT=8001

ChatQnA/docker_compose/amd/gpu/rocm/set_env_faqgen_vllm.sh

@@ -2,18 +2,18 @@
 
 # Copyright (C) 2025 Advanced Micro Devices, Inc.
 
-export HOST_IP=''
-export HOST_IP_EXTERNAL=''
+export HOST_IP=${ip_address}
+export HOST_IP_EXTERNAL=${ip_address}
 
 export CHATQNA_EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
 export CHATQNA_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
 export CHATQNA_LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
 export CHATQNA_RERANK_MODEL_ID="BAAI/bge-reranker-base"
 
-export CHATQNA_BACKEND_SERVICE_PORT=18102
-export CHATQNA_FRONTEND_SERVICE_PORT=18101
+export CHATQNA_BACKEND_SERVICE_PORT=8888
+export CHATQNA_FRONTEND_SERVICE_PORT=5173
 export CHATQNA_LLM_FAQGEN_PORT=18011
-export CHATQNA_NGINX_PORT=18104
+export CHATQNA_NGINX_PORT=80
 export CHATQNA_REDIS_DATAPREP_PORT=18103
 export CHATQNA_REDIS_RETRIEVER_PORT=7000
 export CHATQNA_REDIS_VECTOR_INSIGHT_PORT=8001

ChatQnA/docker_compose/amd/gpu/rocm/set_env_vllm.sh

@@ -2,17 +2,17 @@
 
 # Copyright (C) 2025 Advanced Micro Devices, Inc.
 
-export HOST_IP=''
-export HOST_IP_EXTERNAL=''
+export HOST_IP=${ip_address}
+export HOST_IP_EXTERNAL=${ip_address}
 
 export CHATQNA_EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
 export CHATQNA_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
 export CHATQNA_LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
 export CHATQNA_RERANK_MODEL_ID="BAAI/bge-reranker-base"
 
-export CHATQNA_BACKEND_SERVICE_PORT=18102
-export CHATQNA_FRONTEND_SERVICE_PORT=18101
-export CHATQNA_NGINX_PORT=18104
+export CHATQNA_BACKEND_SERVICE_PORT=8888
+export CHATQNA_FRONTEND_SERVICE_PORT=5173
+export CHATQNA_NGINX_PORT=80
 export CHATQNA_REDIS_DATAPREP_PORT=18103
 export CHATQNA_REDIS_RETRIEVER_PORT=7000
 export CHATQNA_REDIS_VECTOR_INSIGHT_PORT=8001

ChatQnA/docker_compose/install_docker.sh

@@ -1,35 +0,0 @@
-#!/bin/bash
-
-# Copyright (C) 2024 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
-
-# Update the package index
-sudo apt-get -y update
-
-# Install prerequisites
-sudo apt-get -y install ca-certificates curl
-
-# Create the directory for the Docker GPG key
-sudo install -m 0755 -d /etc/apt/keyrings
-
-# Add Docker's official GPG key
-sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
-
-# Set permissions for the GPG key
-sudo chmod a+r /etc/apt/keyrings/docker.asc
-
-# Add Docker repository to the sources list
-echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
-  $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-
-# Update the package index with Docker packages
-sudo apt-get -y update
-
-# Install Docker packages
-sudo apt-get -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
-
-# add existing user
-sudo usermod -aG docker $USER
-
-# Optional: Verify that Docker is installed correctly
-sudo docker --version

ChatQnA/docker_compose/intel/cpu/xeon/README.md

@@ -156,6 +156,7 @@ In the context of deploying a ChatQnA pipeline on an Intel® Xeon® platform, we
 | [compose_faqgen_tgi.yaml](./compose_faqgen_tgi.yaml)         | Enables FAQ generation using TGI as the LLM serving framework. For more details, refer to [README_faqgen.md](./README_faqgen.md).                                     |
 | [compose.telemetry.yaml](./compose.telemetry.yaml)           | Helper file for telemetry features for vllm. Can be used along with any compose files that serves vllm                                                                |
 | [compose_tgi.telemetry.yaml](./compose_tgi.telemetry.yaml)   | Helper file for telemetry features for tgi. Can be used along with any compose files that serves tgi                                                                  |
+| [compose_mariadb.yaml](./compose_mariadb.yaml)               | Uses MariaDB Server as the vector database. All other configurations remain the same as the default                                                                   |
 
 ## ChatQnA with Conversational UI (Optional)
 

ChatQnA/docker_compose/intel/cpu/xeon/README_mariadb.md

@@ -0,0 +1,259 @@
+# Deploying ChatQnA with MariaDB Vector on Intel® Xeon® Processors
+
+This document outlines the deployment process for a ChatQnA application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on Intel® Xeon® servers. The pipeline integrates **MariaDB Vector** as the vector database and includes microservices such as `embedding`, `retriever`, `rerank`, and `llm`.
+
+---
+
+## Table of Contents
+
+1. [Build Docker Images](#build-docker-images)
+2. [Validate Microservices](#validate-microservices)
+3. [Launch the UI](#launch-the-ui)
+4. [Launch the Conversational UI (Optional)](#launch-the-conversational-ui-optional)
+
+---
+
+## Build Docker Images
+
+First of all, you need to build Docker Images locally and install the python package of it.
+
+```bash
+git clone https://github.com/opea-project/GenAIComps.git
+cd GenAIComps
+```
+
+### 1. Build Retriever Image
+
+```bash
+docker build --no-cache -t opea/retriever:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/src/Dockerfile .
+```
+
+### 2. Build Dataprep Image
+
+```bash
+docker build --no-cache -t opea/dataprep:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
+cd ..
+```
+
+### 3. Build MegaService Docker Image
+
+To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `chatqna.py` Python script. Build MegaService Docker image via below command:
+
+```bash
+git clone https://github.com/opea-project/GenAIExamples.git
+cd GenAIExamples/ChatQnA/
+docker build --no-cache -t opea/chatqna:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+cd ../..
+```
+
+### 4. Build UI Docker Image
+
+Build frontend Docker image via below command:
+
+```bash
+cd GenAIExamples/ChatQnA/ui
+docker build --no-cache -t opea/chatqna-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile .
+cd ../../..
+```
+
+### 5. Build Conversational React UI Docker Image (Optional)
+
+Build frontend Docker image that enables Conversational experience with ChatQnA megaservice via below command:
+
+**Export the value of the public IP address of your Xeon server to the `host_ip` environment variable**
+
+```bash
+cd GenAIExamples/ChatQnA/ui
+export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8912/v1/chatqna"
+export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6043/v1/dataprep/ingest"
+docker build --no-cache -t opea/chatqna-conversation-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg DATAPREP_SERVICE_ENDPOINT=$DATAPREP_SERVICE_ENDPOINT -f ./docker/Dockerfile.react .
+cd ../../..
+```
+
+### 6. Build Nginx Docker Image
+
+```bash
+cd GenAIComps
+docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/third_parties/nginx/src/Dockerfile .
+```
+
+Then run the command `docker images`, you will have the following 5 Docker Images:
+
+1. `opea/dataprep:latest`
+2. `opea/retriever:latest`
+3. `opea/chatqna:latest`
+4. `opea/chatqna-ui:latest`
+5. `opea/nginx:latest`
+
+## Start Microservices
+
+### Required Models
+
+By default, the embedding, reranking and LLM models are set to a default value as listed below:
+
+| Service   | Model                               |
+| --------- | ----------------------------------- |
+| Embedding | BAAI/bge-base-en-v1.5               |
+| Reranking | BAAI/bge-reranker-base              |
+| LLM       | meta-llama/Meta-Llama-3-8B-Instruct |
+
+Change the `xxx_MODEL_ID` below for your needs.
+
+### Setup Environment Variables
+
+Since the `compose.yaml` will consume some environment variables, you need to set them up in advance as below.
+
+**Export the value of the public IP address of your Xeon server to the `host_ip` environment variable**
+
+> Change the External_Public_IP below with the actual IPV4 value
+
+```bash
+export host_ip="External_Public_IP"
+```
+
+> Change to your actual Huggingface API Token value
+
+```bash
+export HUGGINGFACEHUB_API_TOKEN="Your_Huggingface_API_Token"
+```
+
+**Append the value of the public IP address to the no_proxy list if you are in a proxy environment**
+
+```bash
+export no_proxy=${your_no_proxy},chatqna-xeon-ui-server,chatqna-xeon-backend-server,dataprep-mariadb-vector,tei-embedding-service,retriever,tei-reranking-service,tgi-service,vllm-service
+```
+
+```bash
+export no_proxy=${your_no_proxy}
+export http_proxy=${your_http_proxy}
+export https_proxy=${your_http_proxy}
+export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
+export RERANK_MODEL_ID="BAAI/bge-reranker-base"
+export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
+export MARIADB_DATABASE="vectordb"
+export MARIADB_USER="chatqna"
+export MARIADB_PASSWORD="password"
+```
+
+Note: Please replace with `host_ip` with you external IP address, do not use localhost.
+
+### Start all the services Docker Containers
+
+> Before running the docker compose command, you need to be in the folder that has the docker compose yaml file
+
+```bash
+cd GenAIExamples/ChatQnA/docker_compose/intel/cpu/xeon/
+docker compose -f compose_mariadb.yaml up -d
+```
+
+### Validate Microservices
+
+Follow the instructions to validate MicroServices.
+For details on how to verify the correctness of the response, refer to [how-to-validate_service](../../hpu/gaudi/how_to_validate_service.md).
+
+1. TEI Embedding Service
+
+   ```bash
+   curl ${host_ip}:6040/embed \
+       -X POST \
+       -d '{"inputs":"What is Deep Learning?"}' \
+       -H 'Content-Type: application/json'
+   ```
+
+2. Retriever Microservice
+
+   To consume the retriever microservice, you need to generate a mock embedding vector by Python script. The length of embedding vector
+   is determined by the embedding model.
+   Here we use the model `EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"`, which vector size is 768.
+
+   Check the vector dimension of your embedding model, set `your_embedding` dimension equals to it.
+
+   ```bash
+   export your_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
+   curl http://${host_ip}:6045/v1/retrieval \
+     -X POST \
+     -d '{"text":"What is the revenue of Nike in 2023?","embedding":"'"${your_embedding}"'"}' \
+     -H 'Content-Type: application/json'
+   ```
+
+3. TEI Reranking Service
+
+   ```bash
+   curl http://${host_ip}:6041/rerank \
+       -X POST \
+       -d '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' \
+       -H 'Content-Type: application/json'
+   ```
+
+4. LLM Backend Service
+
+   In the first startup, this service will take more time to download, load and warm up the model. After it's finished, the service will be ready.
+
+   Try the command below to check whether the LLM service is ready.
+
+   ```bash
+   docker logs vllm-service 2>&1 | grep complete
+   ```
+
+   If the service is ready, you will get the response like below.
+
+   ```text
+   INFO: Application startup complete.
+   ```
+
+   Then try the `cURL` command below to validate vLLM service.
+
+   ```bash
+   curl http://${host_ip}:6042/v1/chat/completions \
+     -X POST \
+     -d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens":17}' \
+     -H 'Content-Type: application/json'
+   ```
+
+5. MegaService
+
+   ```bash
+   curl http://${host_ip}:8912/v1/chatqna -H "Content-Type: application/json" -d '{
+        "messages": "What is the revenue of Nike in 2023?"
+        }'
+   ```
+
+6. Dataprep Microservice（Optional）
+
+   If you want to update the default knowledge base, you can use the following commands:
+
+   Update Knowledge Base via Local File Upload:
+
+   ```bash
+   curl -X POST "http://${host_ip}:6043/v1/dataprep/ingest" \
+        -H "Content-Type: multipart/form-data" \
+        -F "files=@./your_file.pdf"
+   ```
+
+   This command updates a knowledge base by uploading a local file for processing. Update the file path according to your environment.
+
+   Add Knowledge Base via HTTP Links:
+
+   ```bash
+   curl -X POST "http://${host_ip}:6043/v1/dataprep/ingest" \
+        -H "Content-Type: multipart/form-data" \
+        -F 'link_list=["https://opea.dev"]'
+   ```
+
+## Launch the UI
+
+To access the frontend, open the following URL in your browser: http://{host_ip}:5173. By default, the UI runs on port 5173 internally. If you prefer to use a different host port to access the frontend, you can modify the port mapping in the `compose.yaml` file as shown below:
+
+```yaml
+  chatqna-xeon-ui-server:
+    image: opea/chatqna-ui:latest
+    ...
+    ports:
+      - "80:5173"
+```
+
+![project-screenshot](../../../../assets/img/chat_ui_init.png)
+
+Here is an example of running ChatQnA:
+
+![project-screenshot](../../../../assets/img/chat_ui_response.png)

ChatQnA/docker_compose/intel/cpu/xeon/compose_mariadb.yaml

@@ -0,0 +1,185 @@
+# Copyright (C) 2025 MariaDB Foundation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  mariadb-server:
+    image: mariadb:latest
+    container_name: mariadb-server
+    ports:
+      - "3306:3306"
+    environment:
+      - MARIADB_DATABASE=${MARIADB_DATABASE}
+      - MARIADB_USER=${MARIADB_USER}
+      - MARIADB_PASSWORD=${MARIADB_PASSWORD}
+      - MARIADB_RANDOM_ROOT_PASSWORD=1
+    healthcheck:
+      test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"]
+      start_period: 10s
+      interval: 10s
+      timeout: 5s
+      retries: 3
+  dataprep-mariadb-vector:
+    image: ${REGISTRY:-opea}/dataprep:${TAG:-latest}
+    container_name: dataprep-mariadb-vector
+    depends_on:
+      mariadb-server:
+        condition: service_healthy
+      tei-embedding-service:
+        condition: service_started
+    ports:
+      - "6007:5000"
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      DATAPREP_COMPONENT_NAME: "OPEA_DATAPREP_MARIADBVECTOR"
+      MARIADB_CONNECTION_URL: mariadb+mariadbconnector://${MARIADB_USER}:${MARIADB_PASSWORD}@mariadb-server:3306/${MARIADB_DATABASE}
+      TEI_ENDPOINT: http://tei-embedding-service:80
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://localhost:5000/v1/health_check || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 50
+    restart: unless-stopped
+  tei-embedding-service:
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+    container_name: tei-embedding-server
+    ports:
+      - "6006:80"
+    volumes:
+      - "${MODEL_CACHE:-./data}:/data"
+    shm_size: 1g
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+    command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate
+  retriever:
+    image: ${REGISTRY:-opea}/retriever:${TAG:-latest}
+    container_name: retriever-mariadb-vector
+    depends_on:
+      mariadb-server:
+        condition: service_healthy
+    ports:
+      - "7000:7000"
+    ipc: host
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      MARIADB_CONNECTION_URL: mariadb+mariadbconnector://${MARIADB_USER}:${MARIADB_PASSWORD}@mariadb-server:3306/${MARIADB_DATABASE}
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      LOGFLAG: ${LOGFLAG}
+      RETRIEVER_COMPONENT_NAME: "OPEA_RETRIEVER_MARIADBVECTOR"
+    restart: unless-stopped
+  tei-reranking-service:
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+    container_name: tei-reranking-server
+    ports:
+      - "8808:80"
+    volumes:
+      - "${MODEL_CACHE:-./data}:/data"
+    shm_size: 1g
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      HF_HUB_DISABLE_PROGRESS_BARS: 1
+      HF_HUB_ENABLE_HF_TRANSFER: 0
+    command: --model-id ${RERANK_MODEL_ID} --auto-truncate
+  vllm-service:
+    image: ${REGISTRY:-opea}/vllm:${TAG:-latest}
+    container_name: vllm-service
+    ports:
+      - "9009:80"
+    volumes:
+      - "${MODEL_CACHE:-./data}:/root/.cache/huggingface/hub"
+    shm_size: 128g
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      LLM_MODEL_ID: ${LLM_MODEL_ID}
+      VLLM_TORCH_PROFILER_DIR: "/mnt"
+      VLLM_CPU_KVCACHE_SPACE: 40
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:9009/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 100
+    command: --model $LLM_MODEL_ID --host 0.0.0.0 --port 80
+  chatqna-xeon-backend-server:
+    image: ${REGISTRY:-opea}/chatqna:${TAG:-latest}
+    container_name: chatqna-xeon-backend-server
+    depends_on:
+      mariadb-server:
+        condition: service_healthy
+      dataprep-mariadb-vector:
+        condition: service_healthy
+      tei-embedding-service:
+        condition: service_started
+      retriever:
+        condition: service_started
+      tei-reranking-service:
+        condition: service_started
+      vllm-service:
+        condition: service_healthy
+    ports:
+      - "8888:8888"
+    environment:
+      - no_proxy=${no_proxy}
+      - https_proxy=${https_proxy}
+      - http_proxy=${http_proxy}
+      - MEGA_SERVICE_HOST_IP=chatqna-xeon-backend-server
+      - EMBEDDING_SERVER_HOST_IP=tei-embedding-service
+      - EMBEDDING_SERVER_PORT=${EMBEDDING_SERVER_PORT:-80}
+      - RETRIEVER_SERVICE_HOST_IP=retriever
+      - RERANK_SERVER_HOST_IP=tei-reranking-service
+      - RERANK_SERVER_PORT=${RERANK_SERVER_PORT:-80}
+      - LLM_SERVER_HOST_IP=vllm-service
+      - LLM_SERVER_PORT=80
+      - LLM_MODEL=${LLM_MODEL_ID}
+      - LOGFLAG=${LOGFLAG}
+    ipc: host
+    restart: always
+  chatqna-xeon-ui-server:
+    image: ${REGISTRY:-opea}/chatqna-ui:${TAG:-latest}
+    container_name: chatqna-xeon-ui-server
+    depends_on:
+      - chatqna-xeon-backend-server
+    ports:
+      - "5173:5173"
+    environment:
+      - no_proxy=${no_proxy}
+      - https_proxy=${https_proxy}
+      - http_proxy=${http_proxy}
+    ipc: host
+    restart: always
+  chatqna-xeon-nginx-server:
+    image: ${REGISTRY:-opea}/nginx:${TAG:-latest}
+    container_name: chatqna-xeon-nginx-server
+    depends_on:
+      - chatqna-xeon-backend-server
+      - chatqna-xeon-ui-server
+    ports:
+      - "${NGINX_PORT:-80}:80"
+    environment:
+      - no_proxy=${no_proxy}
+      - https_proxy=${https_proxy}
+      - http_proxy=${http_proxy}
+      - FRONTEND_SERVICE_IP=chatqna-xeon-ui-server
+      - FRONTEND_SERVICE_PORT=5173
+      - BACKEND_SERVICE_NAME=chatqna
+      - BACKEND_SERVICE_IP=chatqna-xeon-backend-server
+      - BACKEND_SERVICE_PORT=8888
+      - DATAPREP_SERVICE_IP=dataprep-mariadb-vector
+      - DATAPREP_SERVICE_PORT=5000
+    ipc: host
+    restart: always
+
+networks:
+  default:
+    driver: bridge

ChatQnA/docker_compose/intel/cpu/xeon/grafana/dashboards/download_opea_dashboard.sh

@@ -1,6 +1,8 @@
 # Copyright (C) 2025 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
-rm *.json
+if ls *.json 1> /dev/null 2>&1; then
+    rm *.json
+fi
 wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/chatqna_megaservice_grafana.json
 wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/qdrant_grafana.json
 wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/milvus_grafana.json

ChatQnA/docker_compose/intel/cpu/xeon/set_env.sh

@@ -7,6 +7,9 @@ pushd "../../../../../" > /dev/null
 source .set_env.sh
 popd > /dev/null
 
+export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export HF_TOKEN=${HF_TOKEN}
+export host_ip=${ip_address}
 export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
 export RERANK_MODEL_ID="BAAI/bge-reranker-base"
 export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"

ChatQnA/docker_compose/intel/cpu/xeon/set_env_mariadb.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2025 MariaDB Foundation
+# SPDX-License-Identifier: Apache-2.0
+
+pushd "../../../../../" > /dev/null
+source .set_env.sh
+popd > /dev/null
+
+if [ -z "${HUGGINGFACEHUB_API_TOKEN}" ]; then
+    echo "Error: HUGGINGFACEHUB_API_TOKEN is not set. Please set HUGGINGFACEHUB_API_TOKEN."
+fi
+
+export host_ip=$(hostname -I | awk '{print $1}')
+export MARIADB_DATABASE="vectordb"
+export MARIADB_USER="chatqna"
+export MARIADB_PASSWORD="password"
+export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
+export RERANK_MODEL_ID="BAAI/bge-reranker-base"
+export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
+export LOGFLAG=""
+export no_proxy="$no_proxy,chatqna-xeon-ui-server,chatqna-xeon-backend-server,dataprep-redis-service,tei-embedding-service,retriever,tei-reranking-service,tgi-service,vllm-service,jaeger,prometheus,grafana,node-exporter"
+export LLM_SERVER_PORT=9000
+export NGINX_PORT=80

ChatQnA/docker_compose/intel/hpu/gaudi/README.md

@@ -43,7 +43,7 @@ Some HuggingFace resources, such as some models, are only accessible if you have
 
 ### Configure the Deployment Environment
 
-To set up environment variables for deploying ChatQnA services, source the _setup_env.sh_ script in this directory:
+To set up environment variables for deploying ChatQnA services, source the _setup_env.sh_ script in this directory (If using faqgen or guardrails, source the _set_env_faqgen.sh_):
 
 ```
 source ./set_env.sh

ChatQnA/docker_compose/intel/hpu/gaudi/set_env.sh

@@ -4,12 +4,20 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # Function to prompt for input and set environment variables
+NON_INTERACTIVE=${NON_INTERACTIVE:-false}
+
 prompt_for_env_var() {
   local var_name="$1"
   local prompt_message="$2"
   local default_value="$3"
   local mandatory="$4"
 
+  if [[ "$NON_INTERACTIVE" == "true" ]]; then
+    echo "Non-interactive environment detected. Setting $var_name to default: $default_value"
+    export "$var_name"="$default_value"
+    return
+  fi
+
   if [[ "$mandatory" == "true" ]]; then
     while [[ -z "$value" ]]; do
       read -p "$prompt_message [default: \"${default_value}\"]: " value
@@ -34,7 +42,7 @@ popd > /dev/null
 
 # Prompt the user for each required environment variable
 prompt_for_env_var "EMBEDDING_MODEL_ID" "Enter the EMBEDDING_MODEL_ID" "BAAI/bge-base-en-v1.5" false
-prompt_for_env_var "HUGGINGFACEHUB_API_TOKEN" "Enter the HUGGINGFACEHUB_API_TOKEN" "" true
+prompt_for_env_var "HUGGINGFACEHUB_API_TOKEN" "Enter the HUGGINGFACEHUB_API_TOKEN" "${HF_TOKEN}" true
 prompt_for_env_var "RERANK_MODEL_ID" "Enter the RERANK_MODEL_ID" "BAAI/bge-reranker-base" false
 prompt_for_env_var "LLM_MODEL_ID" "Enter the LLM_MODEL_ID" "meta-llama/Meta-Llama-3-8B-Instruct" false
 prompt_for_env_var "INDEX_NAME" "Enter the INDEX_NAME" "rag-redis" false
@@ -42,34 +50,40 @@ prompt_for_env_var "NUM_CARDS" "Enter the number of Gaudi devices" "1" false
 prompt_for_env_var "host_ip" "Enter the host_ip" "$(curl ifconfig.me)" false
 
 #Query for enabling http_proxy
-prompt_for_env_var "http_proxy" "Enter the http_proxy." "" false
+prompt_for_env_var "http_proxy" "Enter the http_proxy." "${http_proxy}" false
 
 #Query for enabling https_proxy
-prompt_for_env_var "https_proxy" "Enter the https_proxy." "" false
+prompt_for_env_var "http_proxy" "Enter the http_proxy." "${https_proxy}" false
 
 #Query for enabling no_proxy
-prompt_for_env_var "no_proxy" "Enter the no_proxy." "" false
+prompt_for_env_var "no_proxy" "Enter the no_proxy." "${no_proxy}" false
 
 # Query for enabling logging
-read -p "Enable logging? (yes/no): " logging && logging=$(echo "$logging" | tr '[:upper:]' '[:lower:]')
-if [[ "$logging" == "yes" || "$logging" == "y" ]]; then
-  export LOGFLAG=true
+if [[ "$NON_INTERACTIVE" == "true" ]]; then
+  # Query for enabling logging
+  prompt_for_env_var "LOGFLAG" "Enable logging? (yes/no): " "true" false
+  export JAEGER_IP=$(ip route get 8.8.8.8 | grep -oP 'src \K[^ ]+')
+  export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=grpc://$JAEGER_IP:4317
+  export TELEMETRY_ENDPOINT=http://$JAEGER_IP:4318/v1/traces
+  telemetry_flag=true
 else
-  export LOGFLAG=false
-fi
-
-# Query for enabling OpenTelemetry Tracing Endpoint
-read -p "Enable OpenTelemetry Tracing Endpoint? (yes/no): " telemetry && telemetry=$(echo "$telemetry" | tr '[:upper:]' '[:lower:]')
-if [[ "$telemetry" == "yes" || "$telemetry" == "y" ]]; then
-    export JAEGER_IP=$(ip route get 8.8.8.8 | grep -oP 'src \K[^ ]+')
-    export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=grpc://$JAEGER_IP:4317
-    export TELEMETRY_ENDPOINT=http://$JAEGER_IP:4318/v1/traces
-    telemetry_flag=true
-    pushd "grafana/dashboards" > /dev/null
-    source download_opea_dashboard.sh
-    popd > /dev/null
-else
-    telemetry_flag=false
+  # Query for enabling logging
+  read -p "Enable logging? (yes/no): " logging && logging=$(echo "$logging" | tr '[:upper:]' '[:lower:]')
+  if [[ "$logging" == "yes" || "$logging" == "y" ]]; then
+    export LOGFLAG=true
+  else
+    export LOGFLAG=false
+  fi
+  # Query for enabling OpenTelemetry Tracing Endpoint
+  read -p "Enable OpenTelemetry Tracing Endpoint? (yes/no): " telemetry && telemetry=$(echo "$telemetry" | tr '[:upper:]' '[:lower:]')
+  if [[ "$telemetry" == "yes" || "$telemetry" == "y" ]]; then
+      export JAEGER_IP=$(ip route get 8.8.8.8 | grep -oP 'src \K[^ ]+')
+      export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=grpc://$JAEGER_IP:4317
+      export TELEMETRY_ENDPOINT=http://$JAEGER_IP:4318/v1/traces
+      telemetry_flag=true
+  else
+      telemetry_flag=false
+  fi
 fi
 
 # Generate the .env file

ChatQnA/docker_compose/intel/hpu/gaudi/set_env_faqgen.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+pushd "../../../../../" > /dev/null
+source .set_env.sh
+popd > /dev/null
+
+export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export HF_TOKEN=${HF_TOKEN}
+export host_ip=${ip_address}
+export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
+export RERANK_MODEL_ID="BAAI/bge-reranker-base"
+export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
+export INDEX_NAME="rag-redis"
+export NUM_CARDS=1
+export VLLM_SKIP_WARMUP=true
+export LOGFLAG=True
+export http_proxy=${http_proxy}
+export https_proxy=${https_proxy}
+export no_proxy="${ip_address},redis-vector-db,dataprep-redis-service,tei-embedding-service,retriever,tei-reranking-service,tgi-service,vllm-service,guardrails,llm-faqgen,chatqna-gaudi-backend-server,chatqna-gaudi-ui-server,chatqna-gaudi-nginx-server"
+
+export LLM_ENDPOINT_PORT=8010
+export LLM_SERVER_PORT=9001
+export CHATQNA_BACKEND_PORT=8888
+export CHATQNA_REDIS_VECTOR_PORT=6377
+export CHATQNA_REDIS_VECTOR_INSIGHT_PORT=8006
+export CHATQNA_FRONTEND_SERVICE_PORT=5175
+export NGINX_PORT=80
+export FAQGen_COMPONENT_NAME="OpeaFaqGenvLLM"
+export LLM_ENDPOINT="http://${host_ip}:${LLM_ENDPOINT_PORT}"

ChatQnA/tests/README.md

@@ -0,0 +1,123 @@
+# ChatQnA E2E test scripts
+
+## Set the required environment variable
+
+```bash
+export HUGGINGFACEHUB_API_TOKEN="Your_Huggingface_API_Token"
+```
+
+## Run test
+
+On Intel Xeon with TGI:
+
+```bash
+bash test_compose_tgi_on_xeon.sh
+```
+
+On Intel Xeon with vLLM:
+
+```bash
+bash test_compose_on_xeon.sh
+```
+
+On Intel Xeon with MariaDB Vector:
+
+```bash
+bash test_compose_mariadb_on_xeon.sh
+```
+
+On Intel Xeon with Pinecone:
+
+```bash
+bash test_compose_pinecone_on_xeon.sh
+```
+
+On Intel Xeon with Milvus
+
+```bash
+bash test_compose_milvus_on_xeon.sh
+```
+
+On Intel Xeon with Qdrant
+
+```bash
+bash test_compose_qdrant_on_xeon.sh
+```
+
+On Intel Xeon without Rerank:
+
+```bash
+bash test_compose_without_rerank_on_xeon.sh
+```
+
+On Intel Gaudi with TGI:
+
+```bash
+bash test_compose_tgi_on_gaudi.sh
+```
+
+On Intel Gaudi with vLLM:
+
+```bash
+bash test_compose_on_gaudi.sh
+```
+
+On Intel Gaudi with Guardrails:
+
+```bash
+bash test_compose_guardrails_on_gaudi.sh
+```
+
+On Intel Gaudi without Rerank:
+
+```bash
+bash test_compose_without_rerank_on_gaudi.sh
+```
+
+On AMD ROCm with TGI:
+
+```bash
+bash test_compose_on_rocm.sh
+```
+
+On AMD ROCm with vLLM:
+
+```bash
+bash test_compose_vllm_on_rocm.sh
+```
+
+Test FAQ Generation On Intel Xeon with TGI:
+
+```bash
+bash test_compose_faqgen_tgi_on_xeon.sh
+```
+
+Test FAQ Generation On Intel Xeon with vLLM:
+
+```bash
+bash test_compose_faqgen_on_xeon.sh
+```
+
+Test FAQ Generation On Intel Gaudi with TGI:
+
+```bash
+bash test_compose_faqgen_tgi_on_gaudi.sh
+```
+
+Test FAQ Generation On Intel Gaudi with vLLM:
+
+```bash
+bash test_compose_faqgen_on_gaudi.sh
+```
+
+Test FAQ Generation On AMD ROCm with TGI:
+
+```bash
+bash test_compose_faqgen_on_rocm.sh
+```
+
+Test FAQ Generation On AMD ROCm with vLLM:
+
+```bash
+bash test_compose_faqgen_vllm_on_rocm.sh
+```

ChatQnA/tests/test_compose_faqgen_on_gaudi.sh

@@ -24,8 +24,8 @@ function build_docker_images() {
     docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
     popd && sleep 1s
     git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
-    VLLM_VER=v0.6.6.post1+Gaudi-1.20.0
-    git checkout ${VLLM_VER} &> /dev/null && cd ../
+    VLLM_FORK_VER=v0.6.6.post1+Gaudi-1.20.0
+    git checkout ${VLLM_FORK_VER} &> /dev/null && cd ../
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="chatqna chatqna-ui dataprep retriever llm-faqgen vllm-gaudi nginx"
@@ -36,27 +36,7 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/intel/hpu/gaudi
-    export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-    export RERANK_MODEL_ID="BAAI/bge-reranker-base"
-    export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
-    export NUM_CARDS=1
-    export INDEX_NAME="rag-redis"
-    export host_ip=${ip_address}
-    export LLM_ENDPOINT_PORT=8010
-    export LLM_SERVER_PORT=9001
-    export CHATQNA_BACKEND_PORT=8888
-    export CHATQNA_REDIS_VECTOR_PORT=6377
-    export CHATQNA_REDIS_VECTOR_INSIGHT_PORT=8006
-    export CHATQNA_FRONTEND_SERVICE_PORT=5175
-    export NGINX_PORT=80
-    export FAQGen_COMPONENT_NAME="OpeaFaqGenvLLM"
-    export LLM_ENDPOINT="http://${host_ip}:${LLM_ENDPOINT_PORT}"
-    export HF_TOKEN=${HF_TOKEN}
-    export VLLM_SKIP_WARMUP=true
-    export LOGFLAG=True
-    export http_proxy=${http_proxy}
-    export https_proxy=${https_proxy}
-    export no_proxy="${ip_address},redis-vector-db,dataprep-redis-service,tei-embedding-service,retriever,tei-reranking-service,tgi-service,vllm-service,guardrails,llm-faqgen,chatqna-gaudi-backend-server,chatqna-gaudi-ui-server,chatqna-gaudi-nginx-server"
+    source set_env_faqgen.sh
 
     # Start Docker Containers
     docker compose -f compose_faqgen.yaml up -d > ${LOG_PATH}/start_services_with_compose.log

ChatQnA/tests/test_compose_faqgen_on_rocm.sh

@@ -15,44 +15,7 @@ WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 
-export HOST_IP=${ip_address}
-export HOST_IP_EXTERNAL=${ip_address}
-
-export CHATQNA_EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-export CHATQNA_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-export CHATQNA_LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
-export CHATQNA_RERANK_MODEL_ID="BAAI/bge-reranker-base"
-
-export CHATQNA_BACKEND_SERVICE_PORT=8888
-export CHATQNA_FRONTEND_SERVICE_PORT=5173
-export CHATQNA_LLM_FAQGEN_PORT=18011
-export CHATQNA_NGINX_PORT=80
-export CHATQNA_REDIS_DATAPREP_PORT=18103
-export CHATQNA_REDIS_RETRIEVER_PORT=7000
-export CHATQNA_REDIS_VECTOR_INSIGHT_PORT=8001
-export CHATQNA_REDIS_VECTOR_PORT=6379
-export CHATQNA_TEI_EMBEDDING_PORT=18090
-export CHATQNA_TEI_RERANKING_PORT=18808
-export CHATQNA_TGI_SERVICE_PORT=18008
-
-export CHATQNA_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_BACKEND_SERVICE_PORT}/v1/chatqna"
-export CHATQNA_BACKEND_SERVICE_IP=${HOST_IP}
-export CHATQNA_DATAPREP_DELETE_FILE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep/delete"
-export CHATQNA_DATAPREP_GET_FILE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep/get"
-export CHATQNA_DATAPREP_SERVICE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep/ingest"
-export CHATQNA_EMBEDDING_SERVICE_HOST_IP=${HOST_IP}
-export CHATQNA_FRONTEND_SERVICE_IP=${HOST_IP}
-export CHATQNA_LLM_SERVICE_HOST_IP=${HOST_IP}
-export CHATQNA_LLM_ENDPOINT="http://${HOST_IP}:${CHATQNA_TGI_SERVICE_PORT}"
-export CHATQNA_MEGA_SERVICE_HOST_IP=${HOST_IP}
-export CHATQNA_REDIS_URL="redis://${HOST_IP}:${CHATQNA_REDIS_VECTOR_PORT}"
-export CHATQNA_RERANK_SERVICE_HOST_IP=${HOST_IP}
-export CHATQNA_RETRIEVER_SERVICE_HOST_IP=${HOST_IP}
-export CHATQNA_TEI_EMBEDDING_ENDPOINT="http://${HOST_IP}:${CHATQNA_TEI_EMBEDDING_PORT}"
-
-export CHATQNA_BACKEND_SERVICE_NAME=chatqna
-export CHATQNA_INDEX_NAME="rag-redis"
-export FAQGen_COMPONENT_NAME="OpeaFaqGenTgi"
+source $WORKPATH/docker_compose/amd/gpu/rocm/set_env_faqgen.sh
 
 export PATH="~/miniconda3/bin:$PATH"
 

ChatQnA/tests/test_compose_faqgen_on_xeon.sh

@@ -37,26 +37,16 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/intel/cpu/xeon
-    export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-    export RERANK_MODEL_ID="BAAI/bge-reranker-base"
-    export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
-    export INDEX_NAME="rag-redis"
-    export host_ip=${ip_address}
-    export LLM_ENDPOINT_PORT=8010
     export LLM_SERVER_PORT=9001
-    export CHATQNA_BACKEND_PORT=8888
     export CHATQNA_REDIS_VECTOR_PORT=6377
     export CHATQNA_REDIS_VECTOR_INSIGHT_PORT=8006
     export CHATQNA_FRONTEND_SERVICE_PORT=5175
-    export NGINX_PORT=80
-    export FAQGen_COMPONENT_NAME="OpeaFaqGenvLLM"
-    export LLM_ENDPOINT="http://${host_ip}:${LLM_ENDPOINT_PORT}"
-    export HF_TOKEN=${HF_TOKEN}
     export VLLM_SKIP_WARMUP=true
     export LOGFLAG=True
     export http_proxy=${http_proxy}
     export https_proxy=${https_proxy}
     export no_proxy="${ip_address},redis-vector-db,dataprep-redis-service,tei-embedding-service,retriever,tei-reranking-service,tgi-service,vllm-service,guardrails,llm-faqgen,chatqna-xeon-backend-server,chatqna-xeon-ui-server,chatqna-xeon-nginx-server"
+    source set_env.sh
 
     # Start Docker Containers
     docker compose -f compose_faqgen.yaml up -d > ${LOG_PATH}/start_services_with_compose.log

ChatQnA/tests/test_compose_faqgen_tgi_on_gaudi.sh

@@ -33,25 +33,8 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/intel/hpu/gaudi
-    export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-    export RERANK_MODEL_ID="BAAI/bge-reranker-base"
-    export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
-    export INDEX_NAME="rag-redis"
-    export host_ip=${ip_address}
-    export LLM_ENDPOINT_PORT=8010
-    export LLM_SERVER_PORT=9001
-    export CHATQNA_BACKEND_PORT=8888
-    export CHATQNA_REDIS_VECTOR_PORT=6377
-    export CHATQNA_REDIS_VECTOR_INSIGHT_PORT=8006
-    export CHATQNA_FRONTEND_SERVICE_PORT=5175
-    export NGINX_PORT=80
     export FAQGen_COMPONENT_NAME="OpeaFaqGenTgi"
-    export LLM_ENDPOINT="http://${host_ip}:${LLM_ENDPOINT_PORT}"
-    export HF_TOKEN=${HF_TOKEN}
-    export LOGFLAG=True
-    export http_proxy=${http_proxy}
-    export https_proxy=${https_proxy}
-    export no_proxy="${ip_address},redis-vector-db,dataprep-redis-service,tei-embedding-service,retriever,tei-reranking-service,tgi-service,vllm-service,guardrails,llm-faqgen,chatqna-gaudi-backend-server,chatqna-gaudi-ui-server,chatqna-gaudi-nginx-server"
+    source set_env_faqgen.sh
 
     # Start Docker Containers
     docker compose -f compose_faqgen_tgi.yaml up -d > ${LOG_PATH}/start_services_with_compose.log

ChatQnA/tests/test_compose_faqgen_tgi_on_xeon.sh

@@ -37,25 +37,16 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/intel/cpu/xeon
-    export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-    export RERANK_MODEL_ID="BAAI/bge-reranker-base"
-    export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
-    export INDEX_NAME="rag-redis"
-    export host_ip=${ip_address}
-    export LLM_ENDPOINT_PORT=8010
     export LLM_SERVER_PORT=9001
-    export CHATQNA_BACKEND_PORT=8888
     export CHATQNA_REDIS_VECTOR_PORT=6377
     export CHATQNA_REDIS_VECTOR_INSIGHT_PORT=8006
     export CHATQNA_FRONTEND_SERVICE_PORT=5175
-    export NGINX_PORT=80
     export FAQGen_COMPONENT_NAME="OpeaFaqGenTgi"
-    export LLM_ENDPOINT="http://${host_ip}:${LLM_ENDPOINT_PORT}"
-    export HF_TOKEN=${HF_TOKEN}
     export LOGFLAG=True
     export http_proxy=${http_proxy}
     export https_proxy=${https_proxy}
     export no_proxy="${ip_address},redis-vector-db,dataprep-redis-service,tei-embedding-service,retriever,tei-reranking-service,tgi-service,vllm-service,guardrails,llm-faqgen,chatqna-xeon-backend-server,chatqna-xeon-ui-server,chatqna-xeon-nginx-server"
+    source set_env.sh
 
     # Start Docker Containers
     docker compose -f compose_faqgen_tgi.yaml up -d > ${LOG_PATH}/start_services_with_compose.log

ChatQnA/tests/test_compose_faqgen_vllm_on_rocm.sh

@@ -14,41 +14,7 @@ WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 
-export HOST_IP=${ip_address}
-export HOST_IP_EXTERNAL=${ip_address}
-
-export CHATQNA_EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-export CHATQNA_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-export CHATQNA_LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
-export CHATQNA_RERANK_MODEL_ID="BAAI/bge-reranker-base"
-
-export CHATQNA_BACKEND_SERVICE_PORT=8888
-export CHATQNA_FRONTEND_SERVICE_PORT=5173
-export CHATQNA_LLM_FAQGEN_PORT=18011
-export CHATQNA_NGINX_PORT=80
-export CHATQNA_REDIS_DATAPREP_PORT=18103
-export CHATQNA_REDIS_RETRIEVER_PORT=7000
-export CHATQNA_REDIS_VECTOR_INSIGHT_PORT=8001
-export CHATQNA_REDIS_VECTOR_PORT=6379
-export CHATQNA_TEI_EMBEDDING_PORT=18090
-export CHATQNA_TEI_RERANKING_PORT=18808
-export CHATQNA_VLLM_SERVICE_PORT=18008
-
-export CHATQNA_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_BACKEND_SERVICE_PORT}/v1/chatqna"
-export CHATQNA_BACKEND_SERVICE_IP=${HOST_IP_EXTERNAL}
-export CHATQNA_DATAPREP_DELETE_FILE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep/delete"
-export CHATQNA_DATAPREP_GET_FILE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep/get"
-export CHATQNA_DATAPREP_SERVICE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep/ingest"
-export CHATQNA_FRONTEND_SERVICE_IP=${HOST_IP}
-export CHATQNA_MEGA_SERVICE_HOST_IP=${HOST_IP}
-export CHATQNA_REDIS_URL="redis://${HOST_IP}:${CHATQNA_REDIS_VECTOR_PORT}"
-export CHATQNA_TEI_EMBEDDING_ENDPOINT="http://${HOST_IP}:${CHATQNA_TEI_EMBEDDING_PORT}"
-export LLM_ENDPOINT="http://${HOST_IP}:${CHATQNA_VLLM_SERVICE_PORT}"
-
-export CHATQNA_BACKEND_SERVICE_NAME=chatqna
-export CHATQNA_INDEX_NAME="rag-redis"
-export CHATQNA_TYPE="CHATQNA_FAQGEN"
-export FAQGen_COMPONENT_NAME="OpeaFaqGenvLLM"
+source $WORKPATH/docker_compose/amd/gpu/rocm/set_env_faqgen_vllm.sh
 
 function build_docker_images() {
     opea_branch=${opea_branch:-"main"}

ChatQnA/tests/test_compose_guardrails_on_gaudi.sh

@@ -2,7 +2,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-set -e
+set -xe
 IMAGE_REPO=${IMAGE_REPO:-"opea"}
 IMAGE_TAG=${IMAGE_TAG:-"latest"}
 echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
@@ -24,8 +24,8 @@ function build_docker_images() {
     docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
     popd && sleep 1s
     git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
-    VLLM_VER=v0.6.6.post1+Gaudi-1.20.0
-    git checkout ${VLLM_VER} &> /dev/null && cd ../
+    VLLM_FORK_VER=v0.6.6.post1+Gaudi-1.20.0
+    git checkout ${VLLM_FORK_VER} &> /dev/null && cd ../
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="chatqna chatqna-ui dataprep retriever vllm-gaudi guardrails nginx"
@@ -36,14 +36,8 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/intel/hpu/gaudi
-    export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-    export RERANK_MODEL_ID="BAAI/bge-reranker-base"
-    export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
-    export NUM_CARDS=1
-    export INDEX_NAME="rag-redis"
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export host_ip=${ip_address}
     export GURADRAILS_MODEL_ID="meta-llama/Meta-Llama-Guard-2-8B"
+    source set_env_faqgen.sh
 
     # Start Docker Containers
     docker compose -f compose_guardrails.yaml up -d > ${LOG_PATH}/start_services_with_compose.log

ChatQnA/tests/test_compose_mariadb_on_xeon.sh

@@ -0,0 +1,170 @@
+#!/bin/bash
+# Copyright (C) 2025 MariaDB Foundation
+# SPDX-License-Identifier: Apache-2.0
+
+set -xe
+IMAGE_REPO=${IMAGE_REPO:-"opea"}
+IMAGE_TAG=${IMAGE_TAG:-"latest"}
+echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
+echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
+export REGISTRY=${IMAGE_REPO}
+export TAG=${IMAGE_TAG}
+export MODEL_CACHE=${model_cache:-"./data"}
+
+WORKPATH=$(dirname "$PWD")
+LOG_PATH="$WORKPATH/tests"
+ip_address=$(hostname -I | awk '{print $1}')
+
+function build_docker_images() {
+    opea_branch=${opea_branch:-"main"}
+    cd $WORKPATH/docker_image_build
+    git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
+    git clone https://github.com/vllm-project/vllm.git && cd vllm
+    VLLM_VER="v0.8.3"
+    echo "Check out vLLM tag ${VLLM_VER}"
+    git checkout ${VLLM_VER} &> /dev/null
+    # make sure NOT change the pwd
+    cd ../
+
+    echo "Build all the images with --no-cache, check docker_image_build.log for details..."
+    service_list="chatqna chatqna-ui dataprep retriever vllm nginx"
+    docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
+
+    docker images && sleep 1s
+}
+
+function start_services() {
+    cd $WORKPATH/docker_compose/intel/cpu/xeon
+    export MARIADB_PASSWORD="test"
+    source set_env_mariadb.sh
+
+    # Start Docker Containers
+    docker compose -f compose_mariadb.yaml up -d > ${LOG_PATH}/start_services_with_compose.log
+    n=0
+    until [[ "$n" -ge 100 ]]; do
+        docker logs vllm-service > ${LOG_PATH}/vllm_service_start.log 2>&1
+        if grep -q complete ${LOG_PATH}/vllm_service_start.log; then
+            break
+        fi
+        sleep 5s
+        n=$((n+1))
+    done
+}
+
+function validate_service() {
+    local URL="$1"
+    local EXPECTED_RESULT="$2"
+    local SERVICE_NAME="$3"
+    local DOCKER_NAME="$4"
+    local INPUT_DATA="$5"
+
+    local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL")
+    if [ "$HTTP_STATUS" -eq 200 ]; then
+        echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
+
+        local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
+
+        if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
+            echo "[ $SERVICE_NAME ] Content is as expected."
+        else
+            echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
+            docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
+            exit 1
+        fi
+    else
+        echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
+        docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
+        exit 1
+    fi
+    sleep 1s
+}
+
+function validate_microservices() {
+    # Check if the microservices are running correctly.
+    sleep 3m
+
+    # tei for embedding service
+    validate_service \
+        "${ip_address}:6006/embed" \
+        "\[\[" \
+        "tei-embedding" \
+        "tei-embedding-server" \
+        '{"inputs":"What is Deep Learning?"}'
+
+    # retrieval microservice
+    test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
+    validate_service \
+        "${ip_address}:7000/v1/retrieval" \
+        " " \
+        "retrieval" \
+        "retriever-mariadb-vector" \
+        "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}"
+
+    # tei for rerank microservice
+    validate_service \
+        "${ip_address}:8808/rerank" \
+        '{"index":1,"score":' \
+        "tei-rerank" \
+        "tei-reranking-server" \
+        '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}'
+
+    # vllm for llm service
+    validate_service \
+        "${ip_address}:9009/v1/chat/completions" \
+        "content" \
+        "vllm-llm" \
+        "vllm-service" \
+        '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}'
+}
+
+function validate_megaservice() {
+    # Curl the Mega Service
+    validate_service \
+        "${ip_address}:8888/v1/chatqna" \
+        "Nike" \
+        "mega-chatqna" \
+        "chatqna-xeon-backend-server" \
+        '{"messages": "What is the revenue of Nike in 2023?"}'
+
+}
+
+function stop_docker() {
+    cd $WORKPATH/docker_compose/intel/cpu/xeon
+    docker compose -f compose_mariadb.yaml down
+}
+
+function main() {
+
+    echo "::group::stop_docker"
+    stop_docker
+    echo "::endgroup::"
+
+    echo "::group::build_docker_images"
+    if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
+    echo "::endgroup::"
+
+    echo "::group::start_services"
+    start_services
+    echo "::endgroup::"
+
+    echo "::group::validate_microservices"
+    validate_microservices
+    echo "::endgroup::"
+
+    echo "::group::validate_megaservice"
+    validate_megaservice
+    echo "::endgroup::"
+
+    echo "::group::stop_docker"
+    stop_docker
+    echo "::endgroup::"
+
+    docker system prune -f
+
+}
+
+main

ChatQnA/tests/test_compose_milvus_on_xeon.sh

@@ -2,7 +2,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-set -e
+set -xe
 IMAGE_REPO=${IMAGE_REPO:-"opea"}
 IMAGE_TAG=${IMAGE_TAG:-"latest"}
 echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
@@ -39,11 +39,8 @@ function build_docker_images() {
 }
 function start_services() {
     cd $WORKPATH/docker_compose/intel/cpu/xeon/
-    export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-    export RERANK_MODEL_ID="BAAI/bge-reranker-base"
-    export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
     export LOGFLAG=true
+    source set_env.sh
 
     # Start Docker Containers
     docker compose -f compose_milvus.yaml up -d > ${LOG_PATH}/start_services_with_compose.log

ChatQnA/tests/test_compose_on_gaudi.sh

@@ -2,7 +2,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-set -e
+set -xe
 IMAGE_REPO=${IMAGE_REPO:-"opea"}
 IMAGE_TAG=${IMAGE_TAG:-"latest"}
 echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
@@ -24,8 +24,8 @@ function build_docker_images() {
     docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
     popd && sleep 1s
     git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
-    VLLM_VER=v0.6.6.post1+Gaudi-1.20.0
-    git checkout ${VLLM_VER} &> /dev/null && cd ../
+    VLLM_FORK_VER=v0.6.6.post1+Gaudi-1.20.0
+    git checkout ${VLLM_FORK_VER} &> /dev/null && cd ../
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="chatqna chatqna-ui dataprep retriever vllm-gaudi nginx"
@@ -36,16 +36,10 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/intel/hpu/gaudi
-    export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-    export RERANK_MODEL_ID="BAAI/bge-reranker-base"
-    export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
-    export NUM_CARDS=1
-    export INDEX_NAME="rag-redis"
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+    export NON_INTERACTIVE=true
     export host_ip=${ip_address}
-    export JAEGER_IP=$(ip route get 8.8.8.8 | grep -oP 'src \K[^ ]+')
-    export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=grpc://$JAEGER_IP:4317
-    export TELEMETRY_ENDPOINT=http://$JAEGER_IP:4318/v1/traces
+    export telemetry=yes
+    source set_env.sh
 
     # Start Docker Containers
     docker compose -f compose.yaml -f compose.telemetry.yaml up -d > ${LOG_PATH}/start_services_with_compose.log

ChatQnA/tests/test_compose_on_rocm.sh

@@ -15,41 +15,7 @@ WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 
-export HOST_IP=${ip_address}
-export HOST_IP_EXTERNAL=${ip_address}
-
-export CHATQNA_EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-export CHATQNA_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-export CHATQNA_LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
-export CHATQNA_RERANK_MODEL_ID="BAAI/bge-reranker-base"
-
-export CHATQNA_BACKEND_SERVICE_PORT=8888
-export CHATQNA_FRONTEND_SERVICE_PORT=5173
-export CHATQNA_NGINX_PORT=80
-export CHATQNA_REDIS_DATAPREP_PORT=18103
-export CHATQNA_REDIS_RETRIEVER_PORT=7000
-export CHATQNA_REDIS_VECTOR_INSIGHT_PORT=8001
-export CHATQNA_REDIS_VECTOR_PORT=6379
-export CHATQNA_TEI_EMBEDDING_PORT=18090
-export CHATQNA_TEI_RERANKING_PORT=18808
-export CHATQNA_TGI_SERVICE_PORT=18008
-
-export CHATQNA_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_BACKEND_SERVICE_PORT}/v1/chatqna"
-export CHATQNA_BACKEND_SERVICE_IP=${HOST_IP}
-export CHATQNA_DATAPREP_DELETE_FILE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep/delete"
-export CHATQNA_DATAPREP_GET_FILE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep/get"
-export CHATQNA_DATAPREP_SERVICE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep/ingest"
-export CHATQNA_EMBEDDING_SERVICE_HOST_IP=${HOST_IP}
-export CHATQNA_FRONTEND_SERVICE_IP=${HOST_IP}
-export CHATQNA_LLM_SERVICE_HOST_IP=${HOST_IP}
-export CHATQNA_MEGA_SERVICE_HOST_IP=${HOST_IP}
-export CHATQNA_REDIS_URL="redis://${HOST_IP}:${CHATQNA_REDIS_VECTOR_PORT}"
-export CHATQNA_RERANK_SERVICE_HOST_IP=${HOST_IP}
-export CHATQNA_RETRIEVER_SERVICE_HOST_IP=${HOST_IP}
-export CHATQNA_TEI_EMBEDDING_ENDPOINT="http://${HOST_IP}:${CHATQNA_TEI_EMBEDDING_PORT}"
-
-export CHATQNA_BACKEND_SERVICE_NAME=chatqna
-export CHATQNA_INDEX_NAME="rag-redis"
+source $WORKPATH/docker_compose/amd/gpu/rocm/set_env.sh
 
 export PATH="~/miniconda3/bin:$PATH"
 

ChatQnA/tests/test_compose_on_xeon.sh

@@ -2,7 +2,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-set -e
+set -xe
 IMAGE_REPO=${IMAGE_REPO:-"opea"}
 IMAGE_TAG=${IMAGE_TAG:-"latest"}
 echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
@@ -40,15 +40,7 @@ function build_docker_images() {
 function start_services() {
     cd $WORKPATH/docker_compose/intel/cpu/xeon
 
-    export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-    export RERANK_MODEL_ID="BAAI/bge-reranker-base"
-    export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
-    export INDEX_NAME="rag-redis"
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export host_ip=${ip_address}
-    export JAEGER_IP=$(ip route get 8.8.8.8 | grep -oP 'src \K[^ ]+')
-    export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=grpc://$JAEGER_IP:4317
-    export TELEMETRY_ENDPOINT=http://$JAEGER_IP:4318/v1/traces
+    source set_env.sh
 
     # Start Docker Containers
     docker compose -f compose.yaml -f compose.telemetry.yaml up -d > ${LOG_PATH}/start_services_with_compose.log

ChatQnA/tests/test_compose_pinecone_on_xeon.sh

@@ -2,7 +2,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-set -e
+set -xe
 IMAGE_REPO=${IMAGE_REPO:-"opea"}
 IMAGE_TAG=${IMAGE_TAG:-"latest"}
 echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
@@ -41,14 +41,11 @@ function build_docker_images() {
 function start_services() {
     cd $WORKPATH/docker_compose/intel/cpu/xeon/
     export no_proxy=${no_proxy},${ip_address}
-    export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-    export RERANK_MODEL_ID="BAAI/bge-reranker-base"
-    export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
     export PINECONE_API_KEY=${PINECONE_KEY_LANGCHAIN_TEST}
     export PINECONE_INDEX_NAME="langchain-test"
     export INDEX_NAME="langchain-test"
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
     export LOGFLAG=true
+    source set_env.sh
 
     # Start Docker Containers
     docker compose -f compose_pinecone.yaml up -d > ${LOG_PATH}/start_services_with_compose.log

ChatQnA/tests/test_compose_qdrant_on_xeon.sh

@@ -2,7 +2,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-set -e
+set -xe
 IMAGE_REPO=${IMAGE_REPO:-"opea"}
 IMAGE_TAG=${IMAGE_TAG:-"latest"}
 echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
@@ -40,11 +40,8 @@ function build_docker_images() {
 function start_services() {
     cd $WORKPATH/docker_compose/intel/cpu/xeon
 
-    export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-    export RERANK_MODEL_ID="BAAI/bge-reranker-base"
-    export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
     export INDEX_NAME="rag-qdrant"
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+    source set_env.sh
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 

ChatQnA/tests/test_compose_tgi_on_gaudi.sh

@@ -2,7 +2,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-set -e
+set -xe
 IMAGE_REPO=${IMAGE_REPO:-"opea"}
 IMAGE_TAG=${IMAGE_TAG:-"latest"}
 echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
@@ -32,15 +32,10 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/intel/hpu/gaudi
-    export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-    export RERANK_MODEL_ID="BAAI/bge-reranker-base"
-    export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
-    export NUM_CARDS=1
-    export INDEX_NAME="rag-redis"
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export JAEGER_IP=$(ip route get 8.8.8.8 | grep -oP 'src \K[^ ]+')
-    export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=grpc://$JAEGER_IP:4317
-    export TELEMETRY_ENDPOINT=http://$JAEGER_IP:4318/v1/traces
+    export NON_INTERACTIVE=true
+    export host_ip=${ip_address}
+    export telemetry=yes
+    source set_env.sh
 
     # Start Docker Containers
     docker compose -f compose_tgi.yaml -f compose_tgi.telemetry.yaml up -d > ${LOG_PATH}/start_services_with_compose.log

ChatQnA/tests/test_compose_tgi_on_xeon.sh

@@ -2,7 +2,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-set -e
+set -xe
 IMAGE_REPO=${IMAGE_REPO:-"opea"}
 IMAGE_TAG=${IMAGE_TAG:-"latest"}
 echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
@@ -33,14 +33,7 @@ function build_docker_images() {
 function start_services() {
     cd $WORKPATH/docker_compose/intel/cpu/xeon
 
-    export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-    export RERANK_MODEL_ID="BAAI/bge-reranker-base"
-    export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
-    export INDEX_NAME="rag-redis"
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export JAEGER_IP=$(ip route get 8.8.8.8 | grep -oP 'src \K[^ ]+')
-    export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=grpc://$JAEGER_IP:4317
-    export TELEMETRY_ENDPOINT=http://$JAEGER_IP:4318/v1/traces
+    source set_env.sh
 
     # Start Docker Containers
     docker compose -f compose_tgi.yaml -f compose_tgi.telemetry.yaml up -d > ${LOG_PATH}/start_services_with_compose.log

ChatQnA/tests/test_compose_vllm_on_rocm.sh

@@ -2,7 +2,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-set -e
+set -xe
 IMAGE_REPO=${IMAGE_REPO:-"opea"}
 IMAGE_TAG=${IMAGE_TAG:-"latest"}
 echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
@@ -14,42 +14,7 @@ WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
 
-export HOST_IP=${ip_address}
-export HOST_IP_EXTERNAL=${ip_address}
-
-export CHATQNA_EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-export CHATQNA_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-export CHATQNA_LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
-export CHATQNA_RERANK_MODEL_ID="BAAI/bge-reranker-base"
-
-export CHATQNA_BACKEND_SERVICE_PORT=8888
-export CHATQNA_FRONTEND_SERVICE_PORT=5173
-export CHATQNA_NGINX_PORT=80
-export CHATQNA_REDIS_DATAPREP_PORT=18103
-export CHATQNA_REDIS_RETRIEVER_PORT=7000
-export CHATQNA_REDIS_VECTOR_INSIGHT_PORT=8001
-export CHATQNA_REDIS_VECTOR_PORT=6379
-export CHATQNA_TEI_EMBEDDING_PORT=18090
-export CHATQNA_TEI_RERANKING_PORT=18808
-export CHATQNA_VLLM_SERVICE_PORT=18008
-
-export CHATQNA_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_BACKEND_SERVICE_PORT}/v1/chatqna"
-export CHATQNA_BACKEND_SERVICE_IP=${HOST_IP_EXTERNAL}
-export CHATQNA_DATAPREP_DELETE_FILE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep/delete"
-export CHATQNA_DATAPREP_GET_FILE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep/get"
-export CHATQNA_DATAPREP_SERVICE_ENDPOINT="http://${HOST_IP_EXTERNAL}:${CHATQNA_REDIS_DATAPREP_PORT}/v1/dataprep/ingest"
-export CHATQNA_EMBEDDING_SERVICE_HOST_IP=${HOST_IP}
-export CHATQNA_FRONTEND_SERVICE_IP=${HOST_IP}
-export CHATQNA_LLM_SERVICE_HOST_IP=${HOST_IP}
-export CHATQNA_MEGA_SERVICE_HOST_IP=${HOST_IP}
-export CHATQNA_REDIS_URL="redis://${HOST_IP}:${CHATQNA_REDIS_VECTOR_PORT}"
-export CHATQNA_RERANK_SERVICE_HOST_IP=${HOST_IP}
-export CHATQNA_RETRIEVER_SERVICE_HOST_IP=${HOST_IP}
-export CHATQNA_TEI_EMBEDDING_ENDPOINT="http://${HOST_IP}:${CHATQNA_TEI_EMBEDDING_PORT}"
-
-export CHATQNA_BACKEND_SERVICE_NAME=chatqna
-export CHATQNA_INDEX_NAME="rag-redis"
-
+source $WORKPATH/docker_compose/amd/gpu/rocm/set_env_vllm.sh
 
 function build_docker_images() {
     opea_branch=${opea_branch:-"main"}

ChatQnA/tests/test_compose_without_rerank_on_gaudi.sh

@@ -2,7 +2,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-set -e
+set -xe
 IMAGE_REPO=${IMAGE_REPO:-"opea"}
 IMAGE_TAG=${IMAGE_TAG:-"latest"}
 echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
@@ -24,8 +24,8 @@ function build_docker_images() {
     docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
     popd && sleep 1s
     git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
-    VLLM_VER=v0.6.6.post1+Gaudi-1.20.0
-    git checkout ${VLLM_VER} &> /dev/null && cd ../
+    VLLM_FORK_VER=v0.6.6.post1+Gaudi-1.20.0
+    git checkout ${VLLM_FORK_VER} &> /dev/null && cd ../
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="chatqna chatqna-ui dataprep retriever vllm-gaudi nginx"
@@ -36,11 +36,8 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/intel/hpu/gaudi
-    export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-    export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
-    export NUM_CARDS=1
-    export INDEX_NAME="rag-redis"
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+    export NON_INTERACTIVE=true
+    source set_env.sh
 
     # Start Docker Containers
     docker compose -f compose_without_rerank.yaml up -d > ${LOG_PATH}/start_services_with_compose.log

ChatQnA/tests/test_compose_without_rerank_on_xeon.sh

@@ -2,7 +2,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-set -e
+set -xe
 IMAGE_REPO=${IMAGE_REPO:-"opea"}
 IMAGE_TAG=${IMAGE_TAG:-"latest"}
 echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
@@ -41,10 +41,7 @@ function build_docker_images() {
 function start_services() {
     cd $WORKPATH/docker_compose/intel/cpu/xeon
 
-    export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-    export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
-    export INDEX_NAME="rag-redis"
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+    source set_env.sh
 
     # Start Docker Containers
     docker compose -f compose_without_rerank.yaml up -d > ${LOG_PATH}/start_services_with_compose.log

CodeGen/Dockerfile

@@ -1,8 +1,9 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
+ARG IMAGE_REPO=opea
 ARG BASE_TAG=latest
-FROM opea/comps-base:$BASE_TAG
+FROM $IMAGE_REPO/comps-base:$BASE_TAG
 
 COPY ./codegen.py $HOME/codegen.py
 

CodeGen/docker_compose/amd/gpu/rocm/set_env.sh

@@ -5,8 +5,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 ### The IP address or domain name of the server on which the application is running
-export HOST_IP=''
-export EXTERNAL_HOST_IP=''
+export HOST_IP=${ip_address}
+export EXTERNAL_HOST_IP=${ip_address}
 
 ### The port of the TGI service. On this port, the TGI service will accept connections
 export CODEGEN_TGI_SERVICE_PORT=8028
@@ -27,7 +27,7 @@ export CODEGEN_TGI_LLM_ENDPOINT="http://${HOST_IP}:${CODEGEN_TGI_SERVICE_PORT}"
 export CODEGEN_MEGA_SERVICE_HOST_IP=${HOST_IP}
 
 ### The port for CodeGen backend service
-export CODEGEN_BACKEND_SERVICE_PORT=18150
+export CODEGEN_BACKEND_SERVICE_PORT=7778
 
 ### The URL of CodeGen backend service, used by the frontend service
 export CODEGEN_BACKEND_SERVICE_URL="http://${EXTERNAL_HOST_IP}:${CODEGEN_BACKEND_SERVICE_PORT}/v1/codegen"
@@ -36,4 +36,4 @@ export CODEGEN_BACKEND_SERVICE_URL="http://${EXTERNAL_HOST_IP}:${CODEGEN_BACKEND
 export CODEGEN_LLM_SERVICE_HOST_IP=${HOST_IP}
 
 ### The CodeGen service UI port
-export CODEGEN_UI_SERVICE_PORT=18151
+export CODEGEN_UI_SERVICE_PORT=5173

CodeGen/docker_compose/amd/gpu/rocm/set_env_vllm.sh

@@ -5,8 +5,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 ### The IP address or domain name of the server on which the application is running
-export HOST_IP=''
-export EXTERNAL_HOST_IP=''
+export HOST_IP=${ip_address}
+export EXTERNAL_HOST_IP=${ip_address}
 
 ### The port of the vLLM service. On this port, the TGI service will accept connections
 export CODEGEN_VLLM_SERVICE_PORT=8028
@@ -25,7 +25,7 @@ export CODEGEN_LLM_SERVICE_PORT=9000
 export CODEGEN_MEGA_SERVICE_HOST_IP=${HOST_IP}
 
 ### The port for CodeGen backend service
-export CODEGEN_BACKEND_SERVICE_PORT=18150
+export CODEGEN_BACKEND_SERVICE_PORT=7778
 
 ### The URL of CodeGen backend service, used by the frontend service
 export CODEGEN_BACKEND_SERVICE_URL="http://${EXTERNAL_HOST_IP}:${CODEGEN_BACKEND_SERVICE_PORT}/v1/codegen"
@@ -34,4 +34,4 @@ export CODEGEN_BACKEND_SERVICE_URL="http://${EXTERNAL_HOST_IP}:${CODEGEN_BACKEND
 export CODEGEN_LLM_SERVICE_HOST_IP=${HOST_IP}
 
 ### The CodeGen service UI port
-export CODEGEN_UI_SERVICE_PORT=18151
+export CODEGEN_UI_SERVICE_PORT=5173

CodeGen/docker_compose/intel/cpu/xeon/README.md

@@ -6,22 +6,10 @@ This README provides instructions for deploying the CodeGen application using Do
 
 - [Overview](#overview)
 - [Prerequisites](#prerequisites)
-- [Quick Start](#quick-start)
-- [Available Deployment Options](#available-deployment-options)
-  - [Default: vLLM-based Deployment (`--profile codegen-xeon-vllm`)](#default-vllm-based-deployment---profile-codegen-xeon-vllm)
-  - [TGI-based Deployment (`--profile codegen-xeon-tgi`)](#tgi-based-deployment---profile-codegen-xeon-tgi)
-- [Configuration Parameters](#configuration-parameters)
-  - [Environment Variables](#environment-variables)
-  - [Compose Profiles](#compose-profiles)
+- [Quick Start Deployment](#quick-start-deployment)
 - [Building Custom Images (Optional)](#building-custom-images-optional)
 - [Validate Services](#validate-services)
-  - [Check Container Status](#check-container-status)
-  - [Run Validation Script/Commands](#run-validation-scriptcommands)
 - [Accessing the User Interface (UI)](#accessing-the-user-interface-ui)
-  - [Gradio UI (Default)](#gradio-ui-default)
-  - [Svelte UI (Optional)](#svelte-ui-optional)
-  - [React UI (Optional)](#react-ui-optional)
-  - [VS Code Extension (Optional)](#vs-code-extension-optional)
 - [Troubleshooting](#troubleshooting)
 - [Stopping the Application](#stopping-the-application)
 - [Next Steps](#next-steps)
@@ -43,27 +31,37 @@ This guide focuses on running the pre-configured CodeGen service using Docker Co
   cd GenAIExamples/CodeGen/docker_compose/intel/cpu/xeon
   ```
 
-## Quick Start
+## Quick Start Deployment
 
 This uses the default vLLM-based deployment profile (`codegen-xeon-vllm`).
 
 1.  **Configure Environment:**
     Set required environment variables in your shell:
 
-    ```bash
-    # Replace with your host's external IP address (do not use localhost or 127.0.0.1)
-    export HOST_IP="your_external_ip_address"
-    # Replace with your Hugging Face Hub API token
-    export HUGGINGFACEHUB_API_TOKEN="your_huggingface_token"
+        ```bash
+        # Replace with your host's external IP address (do not use localhost or 127.0.0.1)
+        export HOST_IP="your_external_ip_address"
+        # Replace with your Hugging Face Hub API token
+        export HUGGINGFACEHUB_API_TOKEN="your_huggingface_token"
 
-    # Optional: Configure proxy if needed
-    # export http_proxy="your_http_proxy"
-    # export https_proxy="your_https_proxy"
-    # export no_proxy="localhost,127.0.0.1,${HOST_IP}" # Add other hosts if necessary
-    source ../../../set_env.sh
-    ```
+        # Optional: Configure proxy if needed
+        # export http_proxy="your_http_proxy"
+        # export https_proxy="your_https_proxy"
+        # export no_proxy="localhost,127.0.0.1,${HOST_IP}" # Add other hosts if necessary
+        source ../../set_env.sh
+        ```
 
-    _Note: The compose file might read additional variables from a `.env` file or expect them defined elsewhere. Ensure all required variables like ports (`LLM_SERVICE_PORT`, `MEGA_SERVICE_PORT`, etc.) are set if not using defaults from the compose file._
+        _Note: The compose file might read additional variables from set_env.sh. Ensure all required variables like ports (`LLM_SERVICE_PORT`, `MEGA_SERVICE_PORT`, etc.) are set if not using defaults from the compose file._
+
+    For instance, edit the set_env.sh to change the LLM model
+
+        ```
+        export LLM_MODEL_ID="Qwen/Qwen2.5-Coder-7B-Instruct"
+        ```
+        can be changed to other model if needed
+        ```
+        export LLM_MODEL_ID="Qwen/Qwen2.5-Coder-32B-Instruct"
+        ```
 
 2.  **Start Services (vLLM Profile):**
 
@@ -74,17 +72,17 @@ This uses the default vLLM-based deployment profile (`codegen-xeon-vllm`).
 3.  **Validate:**
     Wait several minutes for models to download (especially the first time) and services to initialize. Check container logs (`docker compose logs -f <service_name>`) or proceed to the validation steps below.
 
-## Available Deployment Options
+### Available Deployment Options
 
 The `compose.yaml` file uses Docker Compose profiles to select the LLM serving backend.
 
-### Default: vLLM-based Deployment (`--profile codegen-xeon-vllm`)
+#### Default: vLLM-based Deployment (`--profile codegen-xeon-vllm`)
 
 - **Profile:** `codegen-xeon-vllm`
 - **Description:** Uses vLLM optimized for Intel CPUs as the LLM serving engine. This is the default profile used in the Quick Start.
 - **Services Deployed:** `codegen-vllm-server`, `codegen-llm-server`, `codegen-tei-embedding-server`, `codegen-retriever-server`, `redis-vector-db`, `codegen-dataprep-server`, `codegen-backend-server`, `codegen-gradio-ui-server`.
 
-### TGI-based Deployment (`--profile codegen-xeon-tgi`)
+#### TGI-based Deployment (`--profile codegen-xeon-tgi`)
 
 - **Profile:** `codegen-xeon-tgi`
 - **Description:** Uses Hugging Face Text Generation Inference (TGI) optimized for Intel CPUs as the LLM serving engine.
@@ -95,24 +93,24 @@ The `compose.yaml` file uses Docker Compose profiles to select the LLM serving b
   docker compose --profile codegen-xeon-tgi up -d
   ```
 
-## Configuration Parameters
+### Configuration Parameters
 
-### Environment Variables
+#### Environment Variables
 
 Key parameters are configured via environment variables set before running `docker compose up`.
 
-| Environment Variable                    | Description                                                                                                         | Default (Set Externally)                                                                         |
-| :-------------------------------------- | :------------------------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------------------------------- |
-| `HOST_IP`                               | External IP address of the host machine. **Required.**                                                              | `your_external_ip_address`                                                                       |
-| `HUGGINGFACEHUB_API_TOKEN`              | Your Hugging Face Hub token for model access. **Required.**                                                         | `your_huggingface_token`                                                                         |
-| `LLM_MODEL_ID`                          | Hugging Face model ID for the CodeGen LLM (used by TGI/vLLM service). Configured within `compose.yaml` environment. | `Qwen/Qwen2.5-Coder-7B-Instruct`                                                                 |
-| `EMBEDDING_MODEL_ID`                    | Hugging Face model ID for the embedding model (used by TEI service). Configured within `compose.yaml` environment.  | `BAAI/bge-base-en-v1.5`                                                                          |
-| `LLM_ENDPOINT`                          | Internal URL for the LLM serving endpoint (used by `codegen-llm-server`). Configured in `compose.yaml`.             | `http://codegen-tgi-server:80/generate` or `http://codegen-vllm-server:8000/v1/chat/completions` |
-| `TEI_EMBEDDING_ENDPOINT`                | Internal URL for the Embedding service. Configured in `compose.yaml`.                                               | `http://codegen-tei-embedding-server:80/embed`                                                   |
-| `DATAPREP_ENDPOINT`                     | Internal URL for the Data Preparation service. Configured in `compose.yaml`.                                        | `http://codegen-dataprep-server:80/dataprep`                                                     |
-| `BACKEND_SERVICE_ENDPOINT`              | External URL for the CodeGen Gateway (MegaService). Derived from `HOST_IP` and port `7778`.                         | `http://${HOST_IP}:7778/v1/codegen`                                                              |
-| `*_PORT` (Internal)                     | Internal container ports (e.g., `80`, `6379`). Defined in `compose.yaml`.                                           | N/A                                                                                              |
-| `http_proxy` / `https_proxy`/`no_proxy` | Network proxy settings (if required).                                                                               | `""`                                                                                             |
+| Environment Variable                    | Description                                                                                                         | Default (Set Externally)                       |
+| :-------------------------------------- | :------------------------------------------------------------------------------------------------------------------ | :--------------------------------------------- | ------------------------------------ |
+| `HOST_IP`                               | External IP address of the host machine. **Required.**                                                              | `your_external_ip_address`                     |
+| `HUGGINGFACEHUB_API_TOKEN`              | Your Hugging Face Hub token for model access. **Required.**                                                         | `your_huggingface_token`                       |
+| `LLM_MODEL_ID`                          | Hugging Face model ID for the CodeGen LLM (used by TGI/vLLM service). Configured within `compose.yaml` environment. | `Qwen/Qwen2.5-Coder-7B-Instruct`               |
+| `EMBEDDING_MODEL_ID`                    | Hugging Face model ID for the embedding model (used by TEI service). Configured within `compose.yaml` environment.  | `BAAI/bge-base-en-v1.5`                        |
+| `LLM_ENDPOINT`                          | Internal URL for the LLM serving endpoint (used by `codegen-llm-server`). Configured in `compose.yaml`.             | `http://codegen-vllm                           | tgi-server:9000/v1/chat/completions` |
+| `TEI_EMBEDDING_ENDPOINT`                | Internal URL for the Embedding service. Configured in `compose.yaml`.                                               | `http://codegen-tei-embedding-server:80/embed` |
+| `DATAPREP_ENDPOINT`                     | Internal URL for the Data Preparation service. Configured in `compose.yaml`.                                        | `http://codegen-dataprep-server:80/dataprep`   |
+| `BACKEND_SERVICE_ENDPOINT`              | External URL for the CodeGen Gateway (MegaService). Derived from `HOST_IP` and port `7778`.                         | `http://${HOST_IP}:7778/v1/codegen`            |
+| `*_PORT` (Internal)                     | Internal container ports (e.g., `80`, `6379`). Defined in `compose.yaml`.                                           | N/A                                            |
+| `http_proxy` / `https_proxy`/`no_proxy` | Network proxy settings (if required).                                                                               | `""`                                           |
 
 Most of these parameters are in `set_env.sh`, you can either modify this file or overwrite the env variables by setting them.
 
@@ -120,7 +118,7 @@ Most of these parameters are in `set_env.sh`, you can either modify this file or
 source CodeGen/docker_compose/set_env.sh
 ```
 
-### Compose Profiles
+#### Compose Profiles
 
 Docker Compose profiles (`codegen-xeon-vllm`, `codegen-xeon-tgi`) control which LLM serving backend (vLLM or TGI) and its associated dependencies are started. Only one profile should typically be active.
 
@@ -152,11 +150,11 @@ Check logs for specific services: `docker compose logs <service_name>`
 
 Use `curl` commands to test the main service endpoints. Ensure `HOST_IP` is correctly set in your environment.
 
-1.  **Validate LLM Serving Endpoint (Example for vLLM on default port 8000 internally, exposed differently):**
+1.  **Validate LLM Serving Endpoint (Example for vLLM on default port 9000 internally, exposed differently):**
 
     ```bash
     # This command structure targets the OpenAI-compatible vLLM endpoint
-    curl http://${HOST_IP}:8000/v1/chat/completions \
+    curl http://${HOST_IP}:9000/v1/chat/completions \
        -X POST \
        -H 'Content-Type: application/json' \
        -d '{"model": "Qwen/Qwen2.5-Coder-7B-Instruct", "messages": [{"role": "user", "content": "Implement a basic Python class"}], "max_tokens":32}'
@@ -179,8 +177,8 @@ Multiple UI options can be configured via the `compose.yaml`.
 ### Gradio UI (Default)
 
 Access the default Gradio UI by navigating to:
-`http://{HOST_IP}:8080`
-_(Port `8080` is the default host mapping for `codegen-gradio-ui-server`)_
+`http://{HOST_IP}:5173`
+_(Port `5173` is the default host mapping for `codegen-gradio-ui-server`)_
 
 ![Gradio UI - Code Generation](../../../../assets/img/codegen_gradio_ui_main.png)
 ![Gradio UI - Resource Management](../../../../assets/img/codegen_gradio_ui_dataprep.png)

CodeGen/docker_compose/intel/hpu/gaudi/README.md

@@ -6,23 +6,10 @@ This README provides instructions for deploying the CodeGen application using Do
 
 - [Overview](#overview)
 - [Prerequisites](#prerequisites)
-- [Quick Start](#quick-start)
-- [Available Deployment Options](#available-deployment-options)
-  - [Default: vLLM-based Deployment (`--profile codegen-gaudi-vllm`)](#default-vllm-based-deployment---profile-codegen-gaudi-vllm)
-  - [TGI-based Deployment (`--profile codegen-gaudi-tgi`)](#tgi-based-deployment---profile-codegen-gaudi-tgi)
-- [Configuration Parameters](#configuration-parameters)
-  - [Environment Variables](#environment-variables)
-  - [Compose Profiles](#compose-profiles)
-  - [Docker Compose Gaudi Configuration](#docker-compose-gaudi-configuration)
+- [Quick Start Deployment](#quick-start-deployment)
 - [Building Custom Images (Optional)](#building-custom-images-optional)
 - [Validate Services](#validate-services)
-  - [Check Container Status](#check-container-status)
-  - [Run Validation Script/Commands](#run-validation-scriptcommands)
 - [Accessing the User Interface (UI)](#accessing-the-user-interface-ui)
-  - [Gradio UI (Default)](#gradio-ui-default)
-  - [Svelte UI (Optional)](#svelte-ui-optional)
-  - [React UI (Optional)](#react-ui-optional)
-  - [VS Code Extension (Optional)](#vs-code-extension-optional)
 - [Troubleshooting](#troubleshooting)
 - [Stopping the Application](#stopping-the-application)
 - [Next Steps](#next-steps)
@@ -44,7 +31,7 @@ This guide focuses on running the pre-configured CodeGen service using Docker Co
   cd GenAIExamples/CodeGen/docker_compose/intel/hpu/gaudi
   ```
 
-## Quick Start
+## Quick Start Deployment
 
 This uses the default vLLM-based deployment profile (`codegen-gaudi-vllm`).
 
@@ -61,10 +48,21 @@ This uses the default vLLM-based deployment profile (`codegen-gaudi-vllm`).
     # export http_proxy="your_http_proxy"
     # export https_proxy="your_https_proxy"
     # export no_proxy="localhost,127.0.0.1,${HOST_IP}" # Add other hosts if necessary
-    source ../../../set_env.sh
+    source ../../set_env.sh
     ```
 
-    _Note: Ensure all required variables like ports (`LLM_SERVICE_PORT`, `MEGA_SERVICE_PORT`, etc.) are set if not using defaults from the compose file._
+    _Note: The compose file might read additional variables from set_env.sh. Ensure all required variables like ports (`LLM_SERVICE_PORT`, `MEGA_SERVICE_PORT`, etc.) are set if not using defaults from the compose file._
+    For instance, edit the set_env.sh to change the LLM model
+
+    ```
+    export LLM_MODEL_ID="Qwen/Qwen2.5-Coder-7B-Instruct"
+    ```
+
+    can be changed to other model if needed
+
+    ```
+    export LLM_MODEL_ID="Qwen/Qwen2.5-Coder-32B-Instruct"
+    ```
 
 2.  **Start Services (vLLM Profile):**
 
@@ -104,18 +102,18 @@ The `compose.yaml` file uses Docker Compose profiles to select the LLM serving b
 
 Key parameters are configured via environment variables set before running `docker compose up`.
 
-| Environment Variable                    | Description                                                                                                         | Default (Set Externally)                                                                         |
-| :-------------------------------------- | :------------------------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------------------------------- |
-| `HOST_IP`                               | External IP address of the host machine. **Required.**                                                              | `your_external_ip_address`                                                                       |
-| `HUGGINGFACEHUB_API_TOKEN`              | Your Hugging Face Hub token for model access. **Required.**                                                         | `your_huggingface_token`                                                                         |
-| `LLM_MODEL_ID`                          | Hugging Face model ID for the CodeGen LLM (used by TGI/vLLM service). Configured within `compose.yaml` environment. | `Qwen/Qwen2.5-Coder-7B-Instruct`                                                                 |
-| `EMBEDDING_MODEL_ID`                    | Hugging Face model ID for the embedding model (used by TEI service). Configured within `compose.yaml` environment.  | `BAAI/bge-base-en-v1.5`                                                                          |
-| `LLM_ENDPOINT`                          | Internal URL for the LLM serving endpoint (used by `codegen-llm-server`). Configured in `compose.yaml`.             | `http://codegen-tgi-server:80/generate` or `http://codegen-vllm-server:8000/v1/chat/completions` |
-| `TEI_EMBEDDING_ENDPOINT`                | Internal URL for the Embedding service. Configured in `compose.yaml`.                                               | `http://codegen-tei-embedding-server:80/embed`                                                   |
-| `DATAPREP_ENDPOINT`                     | Internal URL for the Data Preparation service. Configured in `compose.yaml`.                                        | `http://codegen-dataprep-server:80/dataprep`                                                     |
-| `BACKEND_SERVICE_ENDPOINT`              | External URL for the CodeGen Gateway (MegaService). Derived from `HOST_IP` and port `7778`.                         | `http://${HOST_IP}:7778/v1/codegen`                                                              |
-| `*_PORT` (Internal)                     | Internal container ports (e.g., `80`, `6379`). Defined in `compose.yaml`.                                           | N/A                                                                                              |
-| `http_proxy` / `https_proxy`/`no_proxy` | Network proxy settings (if required).                                                                               | `""`                                                                                             |
+| Environment Variable                    | Description                                                                                                         | Default (Set Externally)                       |
+| :-------------------------------------- | :------------------------------------------------------------------------------------------------------------------ | :--------------------------------------------- | ------------------------------------ |
+| `HOST_IP`                               | External IP address of the host machine. **Required.**                                                              | `your_external_ip_address`                     |
+| `HUGGINGFACEHUB_API_TOKEN`              | Your Hugging Face Hub token for model access. **Required.**                                                         | `your_huggingface_token`                       |
+| `LLM_MODEL_ID`                          | Hugging Face model ID for the CodeGen LLM (used by TGI/vLLM service). Configured within `compose.yaml` environment. | `Qwen/Qwen2.5-Coder-7B-Instruct`               |
+| `EMBEDDING_MODEL_ID`                    | Hugging Face model ID for the embedding model (used by TEI service). Configured within `compose.yaml` environment.  | `BAAI/bge-base-en-v1.5`                        |
+| `LLM_ENDPOINT`                          | Internal URL for the LLM serving endpoint (used by `llm-codegen-vllm-server`). Configured in `compose.yaml`.        | http://codegen-vllm                            | tgi-server:9000/v1/chat/completions` |
+| `TEI_EMBEDDING_ENDPOINT`                | Internal URL for the Embedding service. Configured in `compose.yaml`.                                               | `http://codegen-tei-embedding-server:80/embed` |
+| `DATAPREP_ENDPOINT`                     | Internal URL for the Data Preparation service. Configured in `compose.yaml`.                                        | `http://codegen-dataprep-server:80/dataprep`   |
+| `BACKEND_SERVICE_ENDPOINT`              | External URL for the CodeGen Gateway (MegaService). Derived from `HOST_IP` and port `7778`.                         | `http://${HOST_IP}:7778/v1/codegen`            |
+| `*_PORT` (Internal)                     | Internal container ports (e.g., `80`, `6379`). Defined in `compose.yaml`.                                           | N/A                                            |
+| `http_proxy` / `https_proxy`/`no_proxy` | Network proxy settings (if required).                                                                               | `""`                                           |
 
 Most of these parameters are in `set_env.sh`, you can either modify this file or overwrite the env variables by setting them.
 
@@ -172,11 +170,11 @@ Check logs: `docker compose logs <service_name>`. Pay attention to `vllm-gaudi-s
 
 Use `curl` commands targeting the main service endpoints. Ensure `HOST_IP` is correctly set.
 
-1.  **Validate LLM Serving Endpoint (Example for vLLM on default port 8000 internally, exposed differently):**
+1.  **Validate LLM Serving Endpoint (Example for vLLM on default port 9000 internally, exposed differently):**
 
     ```bash
     # This command structure targets the OpenAI-compatible vLLM endpoint
-    curl http://${HOST_IP}:8000/v1/chat/completions \
+    curl http://${HOST_IP}:9000/v1/chat/completions \
        -X POST \
        -H 'Content-Type: application/json' \
        -d '{"model": "Qwen/Qwen2.5-Coder-7B-Instruct", "messages": [{"role": "user", "content": "Implement a basic Python class"}], "max_tokens":32}'
@@ -197,8 +195,8 @@ UI options are similar to the Xeon deployment.
 ### Gradio UI (Default)
 
 Access the default Gradio UI:
-`http://{HOST_IP}:8080`
-_(Port `8080` is the default host mapping)_
+`http://{HOST_IP}:5173`
+_(Port `5173` is the default host mapping)_
 
 ![Gradio UI](../../../../assets/img/codegen_gradio_ui_main.png)
 

CodeGen/docker_compose/intel/set_env.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+pushd "../../" > /dev/null
+source .set_env.sh
+popd > /dev/null
+
+export HOST_IP=$(hostname -I | awk '{print $1}')
+export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+if [ -z "${HUGGINGFACEHUB_API_TOKEN}" ]; then
+    echo "Error: HUGGINGFACEHUB_API_TOKEN is not set. Please set HUGGINGFACEHUB_API_TOKEN"
+fi
+
+if [ -z "${HOST_IP}" ]; then
+    echo "Error: HOST_IP is not set. Please set HOST_IP first."
+fi
+
+export no_proxy=${no_proxy},${HOST_IP}
+export http_proxy=${http_proxy}
+export https_proxy=${https_proxy}
+
+export LLM_MODEL_ID="Qwen/Qwen2.5-Coder-7B-Instruct"
+export LLM_SERVICE_PORT=9000
+export LLM_ENDPOINT="http://${HOST_IP}:8028"
+export LLM_SERVICE_HOST_IP=${HOST_IP}
+export TGI_LLM_ENDPOINT="http://${HOST_IP}:8028"
+
+export MEGA_SERVICE_PORT=7778
+export MEGA_SERVICE_HOST_IP=${HOST_IP}
+export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:7778/v1/codegen"
+
+export REDIS_DB_PORT=6379
+export REDIS_INSIGHTS_PORT=8001
+export REDIS_RETRIEVER_PORT=7000
+export REDIS_URL="redis://${HOST_IP}:${REDIS_DB_PORT}"
+export RETRIEVAL_SERVICE_HOST_IP=${HOST_IP}
+export RETRIEVER_COMPONENT_NAME="OPEA_RETRIEVER_REDIS"
+export INDEX_NAME="CodeGen"
+
+export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
+export EMBEDDER_PORT=6000
+export TEI_EMBEDDER_PORT=8090
+export TEI_EMBEDDING_HOST_IP=${HOST_IP}
+export TEI_EMBEDDING_ENDPOINT="http://${HOST_IP}:${TEI_EMBEDDER_PORT}"
+
+export DATAPREP_REDIS_PORT=6007
+export DATAPREP_ENDPOINT="http://${HOST_IP}:${DATAPREP_REDIS_PORT}/v1/dataprep"
+export LOGFLAG=false
+export MODEL_CACHE=${model_cache:-"./data"}
+export NUM_CARDS=1

CodeGen/docker_compose/set_env.sh

@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (C) 2024 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
-pushd "../../" > /dev/null
-source .set_env.sh
-popd > /dev/null
-
-export host_ip=$(hostname -I | awk '{print $1}')
-if [ -z "${HUGGINGFACEHUB_API_TOKEN}" ]; then
-    echo "Error: HUGGINGFACEHUB_API_TOKEN is not set. Please set HUGGINGFACEHUB_API_TOKEN"
-fi
-
-if [ -z "${host_ip}" ]; then
-    echo "Error: host_ip is not set. Please set host_ip first."
-fi
-
-export no_proxy=${no_proxy},${host_ip}
-export http_proxy=${http_proxy}
-export https_proxy=${https_proxy}
-
-export LLM_MODEL_ID="Qwen/Qwen2.5-Coder-32B-Instruct"
-export LLM_SERVICE_PORT=9000
-export LLM_ENDPOINT="http://${host_ip}:8028"
-export LLM_SERVICE_HOST_IP=${host_ip}
-export TGI_LLM_ENDPOINT="http://${host_ip}:8028"
-
-export MEGA_SERVICE_PORT=7778
-export MEGA_SERVICE_HOST_IP=${host_ip}
-export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:7778/v1/codegen"
-
-export REDIS_DB_PORT=6379
-export REDIS_INSIGHTS_PORT=8001
-export REDIS_RETRIEVER_PORT=7000
-export REDIS_URL="redis://${host_ip}:${REDIS_DB_PORT}"
-export RETRIEVAL_SERVICE_HOST_IP=${host_ip}
-export RETRIEVER_COMPONENT_NAME="OPEA_RETRIEVER_REDIS"
-export INDEX_NAME="CodeGen"
-
-export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-export EMBEDDER_PORT=6000
-export TEI_EMBEDDER_PORT=8090
-export TEI_EMBEDDING_HOST_IP=${host_ip}
-export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:${TEI_EMBEDDER_PORT}"
-
-export DATAPREP_REDIS_PORT=6007
-export DATAPREP_ENDPOINT="http://${host_ip}:${DATAPREP_REDIS_PORT}/v1/dataprep"
-export LOGFLAG=false
-export MODEL_CACHE="./data"
-export NUM_CARDS=1

CodeGen/docker_image_build/build.yaml

@@ -5,6 +5,8 @@ services:
   codegen:
     build:
       args:
+        IMAGE_REPO: ${REGISTRY}
+        BASE_TAG: ${TAG}
         http_proxy: ${http_proxy}
         https_proxy: ${https_proxy}
         no_proxy: ${no_proxy}
@@ -39,6 +41,7 @@ services:
     build:
       context: GenAIComps
       dockerfile: comps/third_parties/vllm/src/Dockerfile.amd_gpu
+    extends: codegen
     image: ${REGISTRY:-opea}/vllm-rocm:${TAG:-latest}
   vllm:
     build:

CodeGen/tests/README.md

@@ -0,0 +1,33 @@
+# CodeGen E2E test scripts
+
+## Set the required environment variable
+
+```bash
+export HUGGINGFACEHUB_API_TOKEN="Your_Huggingface_API_Token"
+```
+
+## Run test
+
+On Intel Xeon with TGI:
+
+```bash
+bash test_compose_on_xeon.sh
+```
+
+On Intel Gaudi with TGI:
+
+```bash
+bash test_compose_on_gaudi.sh
+```
+
+On AMD ROCm with TGI:
+
+```bash
+bash test_compose_on_rocm.sh
+```
+
+On AMD ROCm with vLLM:
+
+```bash
+bash test_compose_vllm_on_rocm.sh
+```

CodeGen/tests/test_compose_on_gaudi.sh

@@ -10,42 +10,26 @@ echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
 export REGISTRY=${IMAGE_REPO}
 export TAG=${IMAGE_TAG}
 export MODEL_CACHE=${model_cache:-"./data"}
-export REDIS_DB_PORT=6379
-export REDIS_INSIGHTS_PORT=8001
-export REDIS_RETRIEVER_PORT=7000
-export EMBEDDER_PORT=6000
-export TEI_EMBEDDER_PORT=8090
-export DATAPREP_REDIS_PORT=6007
 
 WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
-
-export http_proxy=${http_proxy}
-export https_proxy=${https_proxy}
-export no_proxy=${no_proxy},${ip_address}
-
+source $WORKPATH/docker_compose/intel/set_env.sh
 function build_docker_images() {
     opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
 
     cd $WORKPATH/docker_image_build
     git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
 
     # Download Gaudi vllm of latest tag
     git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
-    VLLM_VER=v0.6.6.post1+Gaudi-1.20.0
-    echo "Check out vLLM tag ${VLLM_VER}"
-    git checkout ${VLLM_VER} &> /dev/null && cd ../
+    VLLM_FORK_VER=v0.6.6.post1+Gaudi-1.20.0
+    echo "Check out vLLM tag ${VLLM_FORK_VER}"
+    git checkout ${VLLM_FORK_VER} &> /dev/null && cd ../
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="codegen codegen-gradio-ui llm-textgen vllm-gaudi dataprep retriever embedding"
@@ -60,28 +44,6 @@ function start_services() {
 
     cd $WORKPATH/docker_compose/intel/hpu/gaudi
 
-    export LLM_MODEL_ID="Qwen/Qwen2.5-Coder-7B-Instruct"
-    export LLM_ENDPOINT="http://${ip_address}:8028"
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export MEGA_SERVICE_PORT=7778
-    export MEGA_SERVICE_HOST_IP=${ip_address}
-    export LLM_SERVICE_HOST_IP=${ip_address}
-    export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:${MEGA_SERVICE_PORT}/v1/codegen"
-    export NUM_CARDS=1
-    export host_ip=${ip_address}
-
-    export REDIS_URL="redis://${host_ip}:${REDIS_DB_PORT}"
-    export RETRIEVAL_SERVICE_HOST_IP=${host_ip}
-    export RETRIEVER_COMPONENT_NAME="OPEA_RETRIEVER_REDIS"
-    export INDEX_NAME="CodeGen"
-
-    export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-    export TEI_EMBEDDING_HOST_IP=${host_ip}
-    export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:${TEI_EMBEDDER_PORT}"
-    export DATAPREP_ENDPOINT="http://${host_ip}:${DATAPREP_REDIS_PORT}/v1/dataprep"
-
-    export INDEX_NAME="CodeGen"
-
     # Start Docker Containers
     docker compose --profile ${compose_profile} up -d | tee ${LOG_PATH}/start_services_with_compose.log
 
@@ -250,24 +212,36 @@ function main() {
         stop_docker "${docker_compose_profiles[${i}]}"
     done
 
-    # build docker images
+    echo "::group::build_docker_images"
     if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
+    echo "::endgroup::"
 
     # loop all profiles
     for ((i = 0; i < len_profiles; i++)); do
         echo "Process [${i}]: ${docker_compose_profiles[$i]}, ${docker_llm_container_names[${i}]}"
+
+        echo "::group::start_services"
         start_services "${docker_compose_profiles[${i}]}" "${docker_llm_container_names[${i}]}"
+        echo "::endgroup::"
         docker ps -a
 
+        echo "::group::validate_microservices"
         validate_microservices "${docker_llm_container_names[${i}]}"
+        echo "::endgroup::"
+
+        echo "::group::validate_megaservice"
         validate_megaservice
+        echo "::endgroup::"
+
+        echo "::group::validate_gradio"
         validate_gradio
+        echo "::endgroup::"
 
         stop_docker "${docker_compose_profiles[${i}]}"
         sleep 5s
     done
 
-    echo y | docker system prune
+    docker system prune -f
 }
 
 main

CodeGen/tests/test_compose_on_rocm.sh

@@ -17,19 +17,13 @@ ip_address=$(hostname -I | awk '{print $1}')
 
 function build_docker_images() {
     opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
 
     cd $WORKPATH/docker_image_build
     git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="codegen codegen-ui llm-textgen"
@@ -41,18 +35,7 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/amd/gpu/rocm/
-
-    export CODEGEN_LLM_MODEL_ID="Qwen/Qwen2.5-Coder-7B-Instruct"
-    export CODEGEN_TGI_SERVICE_PORT=8028
-    export CODEGEN_TGI_LLM_ENDPOINT="http://${ip_address}:${CODEGEN_TGI_SERVICE_PORT}"
-    export CODEGEN_LLM_SERVICE_PORT=9000
-    export CODEGEN_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export CODEGEN_MEGA_SERVICE_HOST_IP=${ip_address}
-    export CODEGEN_LLM_SERVICE_HOST_IP=${ip_address}
-    export CODEGEN_BACKEND_SERVICE_PORT=7778
-    export CODEGEN_BACKEND_SERVICE_URL="http://${ip_address}:${CODEGEN_BACKEND_SERVICE_PORT}/v1/codegen"
-    export CODEGEN_UI_SERVICE_PORT=5173
-    export HOST_IP=${ip_address}
+    source set_env.sh
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 
@@ -164,18 +147,35 @@ function stop_docker() {
 
 function main() {
 
+    echo "::group::stop_docker"
     stop_docker
+    echo "::endgroup::"
 
+    echo "::group::build_docker_images"
     if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
+    echo "::endgroup::"
+
+    echo "::group::start_services"
     start_services
+    echo "::endgroup::"
 
+    echo "::group::validate_microservices"
     validate_microservices
-    validate_megaservice
-    validate_frontend
+    echo "::endgroup::"
 
+    echo "::group::validate_megaservice"
+    validate_megaservice
+    echo "::endgroup::"
+
+    echo "::group::validate_frontend"
+    validate_frontend
+    echo "::endgroup::"
+
+    echo "::group::stop_docker"
     stop_docker
-    echo y | docker system prune
-    cd $WORKPATH
+    echo "::endgroup::"
+
+    docker system prune -f
 
 }
 

CodeGen/tests/test_compose_on_xeon.sh

@@ -10,36 +10,21 @@ echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
 export REGISTRY=${IMAGE_REPO}
 export TAG=${IMAGE_TAG}
 export MODEL_CACHE=${model_cache:-"./data"}
-export REDIS_DB_PORT=6379
-export REDIS_INSIGHTS_PORT=8001
-export REDIS_RETRIEVER_PORT=7000
-export EMBEDDER_PORT=6000
-export TEI_EMBEDDER_PORT=8090
-export DATAPREP_REDIS_PORT=6007
 
 WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')
-
-export http_proxy=${http_proxy}
-export https_proxy=${https_proxy}
-export no_proxy=${no_proxy},${ip_address}
+source $WORKPATH/docker_compose/intel/set_env.sh
 
 function build_docker_images() {
     opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
 
     cd $WORKPATH/docker_image_build
     git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
 
     git clone https://github.com/vllm-project/vllm.git && cd vllm
     VLLM_VER="v0.8.3"
@@ -62,25 +47,6 @@ function start_services() {
 
     cd $WORKPATH/docker_compose/intel/cpu/xeon/
 
-    export LLM_MODEL_ID="Qwen/Qwen2.5-Coder-7B-Instruct"
-    export LLM_ENDPOINT="http://${ip_address}:8028"
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export MEGA_SERVICE_PORT=7778
-    export MEGA_SERVICE_HOST_IP=${ip_address}
-    export LLM_SERVICE_HOST_IP=${ip_address}
-    export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:${MEGA_SERVICE_PORT}/v1/codegen"
-    export host_ip=${ip_address}
-
-    export REDIS_URL="redis://${host_ip}:${REDIS_DB_PORT}"
-    export RETRIEVAL_SERVICE_HOST_IP=${host_ip}
-    export RETRIEVER_COMPONENT_NAME="OPEA_RETRIEVER_REDIS"
-    export INDEX_NAME="CodeGen"
-
-    export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
-    export TEI_EMBEDDING_HOST_IP=${host_ip}
-    export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:${TEI_EMBEDDER_PORT}"
-    export DATAPREP_ENDPOINT="http://${host_ip}:${DATAPREP_REDIS_PORT}/v1/dataprep"
-
     # Start Docker Containers
     docker compose --profile ${compose_profile} up -d > ${LOG_PATH}/start_services_with_compose.log
 
@@ -256,17 +222,28 @@ function main() {
     for ((i = 0; i < len_profiles; i++)); do
         echo "Process [${i}]: ${docker_compose_profiles[$i]}, ${docker_llm_container_names[${i}]}"
         docker ps -a
-        start_services "${docker_compose_profiles[${i}]}" "${docker_llm_container_names[${i}]}"
 
+        echo "::group::start_services"
+        start_services "${docker_compose_profiles[${i}]}" "${docker_llm_container_names[${i}]}"
+        echo "::endgroup::"
+
+        echo "::group::validate_microservices"
         validate_microservices "${docker_llm_container_names[${i}]}"
+        echo "::endgroup::"
+
+        echo "::group::validate_megaservice"
         validate_megaservice
+        echo "::endgroup::"
+
+        echo "::group::validate_gradio"
         validate_gradio
+        echo "::endgroup::"
 
         stop_docker "${docker_compose_profiles[${i}]}"
         sleep 5s
     done
 
-    echo y | docker system prune
+    docker system prune -f
 }
 
 main

CodeGen/tests/test_compose_vllm_on_rocm.sh

@@ -17,19 +17,13 @@ ip_address=$(hostname -I | awk '{print $1}')
 
 function build_docker_images() {
     opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
 
     cd $WORKPATH/docker_image_build
     git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="vllm-rocm llm-textgen codegen codegen-ui"
@@ -40,18 +34,7 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/amd/gpu/rocm/
-
-    export CODEGEN_LLM_MODEL_ID="Qwen/Qwen2.5-Coder-7B-Instruct"
-    export CODEGEN_VLLM_SERVICE_PORT=8028
-    export CODEGEN_VLLM_ENDPOINT="http://${ip_address}:${CODEGEN_VLLM_SERVICE_PORT}"
-    export CODEGEN_LLM_SERVICE_PORT=9000
-    export CODEGEN_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export CODEGEN_MEGA_SERVICE_HOST_IP=${ip_address}
-    export CODEGEN_LLM_SERVICE_HOST_IP=${ip_address}
-    export CODEGEN_BACKEND_SERVICE_PORT=7778
-    export CODEGEN_BACKEND_SERVICE_URL="http://${ip_address}:${CODEGEN_BACKEND_SERVICE_PORT}/v1/codegen"
-    export CODEGEN_UI_SERVICE_PORT=5173
-    export HOST_IP=${ip_address}
+    source set_env_vllm.sh
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 
@@ -164,17 +147,35 @@ function stop_docker() {
 
 function main() {
 
+    echo "::group::stop_docker"
     stop_docker
+    echo "::endgroup::"
+
+    echo "::group::build_docker_images"
     if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
+    echo "::endgroup::"
+
+    echo "::group::start_services"
     start_services
+    echo "::endgroup::"
 
+    echo "::group::validate_microservices"
     validate_microservices
-    validate_megaservice
-    validate_frontend
+    echo "::endgroup::"
 
+    echo "::group::validate_megaservice"
+    validate_megaservice
+    echo "::endgroup::"
+
+    echo "::group::validate_frontend"
+    validate_frontend
+    echo "::endgroup::"
+
+    echo "::group::stop_docker"
     stop_docker
-    echo y | docker system prune
-    cd $WORKPATH
+    echo "::endgroup::"
+
+    docker system prune -f
 
 }
 

CodeTrans/Dockerfile

@@ -1,8 +1,9 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
+ARG IMAGE_REPO=opea
 ARG BASE_TAG=latest
-FROM opea/comps-base:$BASE_TAG
+FROM $IMAGE_REPO/comps-base:$BASE_TAG
 
 COPY ./code_translation.py $HOME/code_translation.py
 

CodeTrans/README.md

@@ -22,12 +22,11 @@ This Code Translation use case demonstrates Text Generation Inference across mul
 
 The table below lists currently available deployment options. They outline in detail the implementation of this example on selected hardware.
 
-| Category               | Deployment Option    | Description                                                       |
-| ---------------------- | -------------------- | ----------------------------------------------------------------- |
-| On-premise Deployments | Docker compose       | [CodeTrans deployment on Xeon](./docker_compose/intel/cpu/xeon)   |
-|                        |                      | [CodeTrans deployment on Gaudi](./docker_compose/intel/hpu/gaudi) |
-|                        |                      | [CodeTrans deployment on AMD ROCm](./docker_compose/amd/gpu/rocm) |
-|                        | Kubernetes           | [Helm Charts](./kubernetes/helm)                                  |
-|                        |                      | [GMC](./kubernetes/gmc)                                           |
-|                        | Azure                | Work-in-progress                                                  |
-|                        | Intel Tiber AI Cloud | Work-in-progress                                                  |
+| Category               | Deployment Option    | Description                                                                 |
+| ---------------------- | -------------------- | --------------------------------------------------------------------------- |
+| On-premise Deployments | Docker compose       | [CodeTrans deployment on Xeon](./docker_compose/intel/cpu/xeon/README.md)   |
+|                        |                      | [CodeTrans deployment on Gaudi](./docker_compose/intel/hpu/gaudi/README.md) |
+|                        |                      | [CodeTrans deployment on AMD ROCm](./docker_compose/amd/gpu/rocm/README.md) |
+|                        | Kubernetes           | [Helm Charts](./kubernetes/helm/README.md)                                  |
+|                        | Azure                | Work-in-progress                                                            |
+|                        | Intel Tiber AI Cloud | Work-in-progress                                                            |

CodeTrans/README_miscellaneous.md

@@ -44,3 +44,38 @@ Some HuggingFace resources, such as some models, are only accessible if the deve
 
 2. (Docker only) If all microservices work well, check the port ${host_ip}:7777, the port may be allocated by other users, you can modify the `compose.yaml`.
 3. (Docker only) If you get errors like "The container name is in use", change container name in `compose.yaml`.
+
+## Monitoring OPEA Services with Prometheus and Grafana Dashboard
+
+OPEA microservice deployment can easily be monitored through Grafana dashboards using data collected via Prometheus. Follow the [README](https://github.com/opea-project/GenAIEval/blob/main/evals/benchmark/grafana/README.md) to setup Prometheus and Grafana servers and import dashboards to monitor the OPEA services.
+
+![example dashboards](./assets/img/example_dashboards.png)
+![tgi dashboard](./assets/img/tgi_dashboard.png)
+
+## Tracing with OpenTelemetry and Jaeger
+
+> NOTE: This feature is disabled by default. Please use the compose.telemetry.yaml file to enable this feature.
+
+OPEA microservice and [TGI](https://huggingface.co/docs/text-generation-inference/en/index)/[TEI](https://huggingface.co/docs/text-embeddings-inference/en/index) serving can easily be traced through [Jaeger](https://www.jaegertracing.io/) dashboards in conjunction with [OpenTelemetry](https://opentelemetry.io/) Tracing feature. Follow the [README](https://github.com/opea-project/GenAIComps/tree/main/comps/cores/telemetry#tracing) to trace additional functions if needed.
+
+Tracing data is exported to http://{EXTERNAL_IP}:4318/v1/traces via Jaeger.
+Users could also get the external IP via below command.
+
+```bash
+ip route get 8.8.8.8 | grep -oP 'src \K[^ ]+'
+```
+
+Access the Jaeger dashboard UI at http://{EXTERNAL_IP}:16686
+
+For TGI serving on Gaudi, users could see different services like opea, TEI and TGI.
+![Screenshot from 2024-12-27 11-58-18](https://github.com/user-attachments/assets/6126fa70-e830-4780-bd3f-83cb6eff064e)
+
+Here is a screenshot for one tracing of TGI serving request.
+![Screenshot from 2024-12-27 11-26-25](https://github.com/user-attachments/assets/3a7c51c6-f422-41eb-8e82-c3df52cd48b8)
+
+There are also OPEA related tracings. Users could understand the time breakdown of each service request by looking into each opea:schedule operation.
+![image](https://github.com/user-attachments/assets/6137068b-b374-4ff8-b345-993343c0c25f)
+
+There could be asynchronous function such as `llm/MicroService_asyn_generate` and user needs to check the trace of the asynchronous function in another operation like
+opea:llm_generate_stream.
+![image](https://github.com/user-attachments/assets/a973d283-198f-4ce2-a7eb-58515b77503e)

CodeTrans/docker_compose/intel/cpu/xeon/README.md

@@ -46,7 +46,7 @@ export http_proxy="Your_HTTP_Proxy"           # http proxy if any
 export https_proxy="Your_HTTPs_Proxy"         # https proxy if any
 export no_proxy=localhost,127.0.0.1,$host_ip  # additional no proxies if needed
 export NGINX_PORT=${your_nginx_port}          # your usable port for nginx, 80 for example
-source ./set_env.sh
+source docker_compose/intel/set_env.sh
 ```
 
 Consult the section on [CodeTrans Service configuration](#codetrans-configuration) for information on how service specific configuration parameters affect deployments.

CodeTrans/docker_compose/intel/hpu/gaudi/README.md

@@ -46,7 +46,7 @@ export http_proxy="Your_HTTP_Proxy"           # http proxy if any
 export https_proxy="Your_HTTPs_Proxy"         # https proxy if any
 export no_proxy=localhost,127.0.0.1,$host_ip  # additional no proxies if needed
 export NGINX_PORT=${your_nginx_port}          # your usable port for nginx, 80 for example
-source ./set_env.sh
+source docker_compose/intel/set_env.sh
 ```
 
 Consult the section on [CodeTrans Service configuration](#codetrans-configuration) for information on how service specific configuration parameters affect deployments.

CodeTrans/docker_compose/intel/set_env.sh

@@ -2,7 +2,7 @@
 
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
-pushd "../../" > /dev/null
+pushd "../../../" > /dev/null
 source .set_env.sh
 popd > /dev/null
 

CodeTrans/docker_image_build/build.yaml

@@ -5,6 +5,8 @@ services:
   codetrans:
     build:
       args:
+        IMAGE_REPO: ${REGISTRY:-opea}
+        BASE_TAG: ${TAG:-latest}
         http_proxy: ${http_proxy}
         https_proxy: ${https_proxy}
         no_proxy: ${no_proxy}
@@ -45,4 +47,5 @@ services:
     build:
       context: GenAIComps
       dockerfile: comps/third_parties/vllm/src/Dockerfile.amd_gpu
+    extends: codetrans
     image: ${REGISTRY:-opea}/vllm-rocm:${TAG:-latest}

CodeTrans/tests/test_compose_on_gaudi.sh

@@ -17,22 +17,17 @@ ip_address=$(hostname -I | awk '{print $1}')
 
 function build_docker_images() {
     opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
 
     cd $WORKPATH/docker_image_build
     git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
+
     git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
-    VLLM_VER=v0.6.6.post1+Gaudi-1.20.0
-    git checkout ${VLLM_VER} &> /dev/null && cd ../
+    VLLM_FORK_VER=v0.6.6.post1+Gaudi-1.20.0
+    git checkout ${VLLM_FORK_VER} &> /dev/null && cd ../
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="codetrans codetrans-ui llm-textgen vllm-gaudi nginx"
@@ -42,12 +37,12 @@ function build_docker_images() {
 }
 
 function start_services() {
-    cd $WORKPATH/docker_compose
+    cd $WORKPATH/docker_compose/intel
     export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
     export NGINX_PORT=80
     export host_ip=${ip_address}
     source set_env.sh
-    cd intel/hpu/gaudi
+    cd hpu/gaudi
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 
@@ -160,17 +155,35 @@ function stop_docker() {
 
 function main() {
 
+    echo "::group::stop_docker"
     stop_docker
+    echo "::endgroup::"
 
+    echo "::group::build_docker_images"
     if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
+    echo "::endgroup::"
+
+    echo "::group::start_services"
     start_services
+    echo "::endgroup::"
 
+    echo "::group::validate_microservices"
     validate_microservices
-    validate_megaservice
-    validate_frontend
+    echo "::endgroup::"
 
+    echo "::group::validate_megaservice"
+    validate_megaservice
+    echo "::endgroup::"
+
+    echo "::group::validate_frontend"
+    validate_frontend
+    echo "::endgroup::"
+
+    echo "::group::stop_docker"
     stop_docker
-    echo y | docker system prune
+    echo "::endgroup::"
+
+    docker system prune -f
 
 }
 

CodeTrans/tests/test_compose_on_rocm.sh

@@ -18,19 +18,13 @@ ip_address=$(hostname -I | awk '{print $1}')
 
 function build_docker_images() {
     opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
 
     cd $WORKPATH/docker_image_build
     git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="codetrans codetrans-ui llm-textgen nginx"
@@ -161,17 +155,35 @@ function stop_docker() {
 
 function main() {
 
+    echo "::group::stop_docker"
     stop_docker
+    echo "::endgroup::"
 
+    echo "::group::build_docker_images"
     if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
+    echo "::endgroup::"
+
+    echo "::group::start_services"
     start_services
+    echo "::endgroup::"
 
+    echo "::group::validate_microservices"
     validate_microservices
-    validate_megaservice
-    validate_frontend
+    echo "::endgroup::"
 
+    echo "::group::validate_megaservice"
+    validate_megaservice
+    echo "::endgroup::"
+
+    echo "::group::validate_frontend"
+    validate_frontend
+    echo "::endgroup::"
+
+    echo "::group::stop_docker"
     stop_docker
-    echo y | docker system prune
+    echo "::endgroup::"
+
+    docker system prune -f
 
 }
 

CodeTrans/tests/test_compose_on_xeon.sh

@@ -17,19 +17,14 @@ ip_address=$(hostname -I | awk '{print $1}')
 
 function build_docker_images() {
     opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
 
     cd $WORKPATH/docker_image_build
     git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
+
     git clone https://github.com/vllm-project/vllm.git && cd vllm
     VLLM_VER="v0.8.3"
     echo "Check out vLLM tag ${VLLM_VER}"
@@ -44,13 +39,13 @@ function build_docker_images() {
 }
 
 function start_services() {
-    cd $WORKPATH/docker_compose
+    cd $WORKPATH/docker_compose/intel
     export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
 
     export NGINX_PORT=80
     export host_ip=${ip_address}
     source set_env.sh
-    cd intel/cpu/xeon/
+    cd cpu/xeon/
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 
@@ -163,17 +158,35 @@ function stop_docker() {
 
 function main() {
 
+    echo "::group::stop_docker"
     stop_docker
+    echo "::endgroup::"
 
+    echo "::group::build_docker_images"
     if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
+    echo "::endgroup::"
+
+    echo "::group::start_services"
     start_services
+    echo "::endgroup::"
 
+    echo "::group::validate_microservices"
     validate_microservices
-    validate_megaservice
-    validate_frontend
+    echo "::endgroup::"
 
+    echo "::group::validate_megaservice"
+    validate_megaservice
+    echo "::endgroup::"
+
+    echo "::group::validate_frontend"
+    validate_frontend
+    echo "::endgroup::"
+
+    echo "::group::stop_docker"
     stop_docker
-    echo y | docker system prune
+    echo "::endgroup::"
+
+    docker system prune -f
 
 }
 

CodeTrans/tests/test_compose_tgi_on_gaudi.sh

@@ -17,19 +17,13 @@ ip_address=$(hostname -I | awk '{print $1}')
 
 function build_docker_images() {
     opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
 
     cd $WORKPATH/docker_image_build
     git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="codetrans codetrans-ui llm-textgen nginx"
@@ -40,13 +34,13 @@ function build_docker_images() {
 }
 
 function start_services() {
-    cd $WORKPATH/docker_compose
+    cd $WORKPATH/docker_compose/intel
     export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
 
     export NGINX_PORT=80
     export host_ip=${ip_address}
     source set_env.sh
-    cd intel/hpu/gaudi/
+    cd hpu/gaudi/
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 
@@ -167,17 +161,35 @@ function stop_docker() {
 
 function main() {
 
+    echo "::group::stop_docker"
     stop_docker
+    echo "::endgroup::"
 
+    echo "::group::build_docker_images"
     if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
+    echo "::endgroup::"
+
+    echo "::group::start_services"
     start_services
+    echo "::endgroup::"
 
+    echo "::group::validate_microservices"
     validate_microservices
-    validate_megaservice
-    validate_frontend
+    echo "::endgroup::"
 
+    echo "::group::validate_megaservice"
+    validate_megaservice
+    echo "::endgroup::"
+
+    echo "::group::validate_frontend"
+    validate_frontend
+    echo "::endgroup::"
+
+    echo "::group::stop_docker"
     stop_docker
-    echo y | docker system prune
+    echo "::endgroup::"
+
+    docker system prune -f
 
 }
 

CodeTrans/tests/test_compose_tgi_on_xeon.sh

@@ -17,19 +17,13 @@ ip_address=$(hostname -I | awk '{print $1}')
 
 function build_docker_images() {
     opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
 
     cd $WORKPATH/docker_image_build
     git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="codetrans codetrans-ui llm-textgen nginx"
@@ -40,13 +34,13 @@ function build_docker_images() {
 }
 
 function start_services() {
-    cd $WORKPATH/docker_compose
+    cd $WORKPATH/docker_compose/intel
     export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
 
     export NGINX_PORT=80
     export host_ip=${ip_address}
     source set_env.sh
-    cd intel/cpu/xeon/
+    cd cpu/xeon/
 
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 
@@ -167,17 +161,35 @@ function stop_docker() {
 
 function main() {
 
+    echo "::group::stop_docker"
     stop_docker
+    echo "::endgroup::"
 
+    echo "::group::build_docker_images"
     if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
+    echo "::endgroup::"
+
+    echo "::group::start_services"
     start_services
+    echo "::endgroup::"
 
+    echo "::group::validate_microservices"
     validate_microservices
-    validate_megaservice
-    validate_frontend
+    echo "::endgroup::"
 
+    echo "::group::validate_megaservice"
+    validate_megaservice
+    echo "::endgroup::"
+
+    echo "::group::validate_frontend"
+    validate_frontend
+    echo "::endgroup::"
+
+    echo "::group::stop_docker"
     stop_docker
-    echo y | docker system prune
+    echo "::endgroup::"
+
+    docker system prune -f
 
 }
 

CodeTrans/tests/test_compose_vllm_on_rocm.sh

@@ -17,19 +17,13 @@ ip_address=$(hostname -I | awk '{print $1}')
 
 function build_docker_images() {
     opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
 
     cd $WORKPATH/docker_image_build
     git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="vllm-rocm llm-textgen codetrans codetrans-ui nginx"
@@ -160,17 +154,35 @@ function stop_docker() {
 
 function main() {
 
+    echo "::group::stop_docker"
     stop_docker
+    echo "::endgroup::"
 
+    echo "::group::build_docker_images"
     if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
+    echo "::endgroup::"
+
+    echo "::group::start_services"
     start_services
+    echo "::endgroup::"
 
+    echo "::group::validate_microservices"
     validate_microservices
-    validate_megaservice
-    validate_frontend
+    echo "::endgroup::"
 
+    echo "::group::validate_megaservice"
+    validate_megaservice
+    echo "::endgroup::"
+
+    echo "::group::validate_frontend"
+    validate_frontend
+    echo "::endgroup::"
+
+    echo "::group::stop_docker"
     stop_docker
-    echo y | docker system prune
+    echo "::endgroup::"
+
+    docker system prune -f
 
 }
 

DBQnA/docker_compose/amd/gpu/rocm/set_env.sh

@@ -3,8 +3,13 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-export host_ip=""
-export DBQNA_HUGGINGFACEHUB_API_TOKEN=""
+pushd "../../" > /dev/null
+ls -l
+source .set_env.sh
+popd > /dev/null
+
+export host_ip=${ip_address}
+export DBQNA_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
 export DBQNA_TGI_SERVICE_PORT=8008
 export DBQNA_TGI_LLM_ENDPOINT="http://${host_ip}:${DBQNA_TGI_SERVICE_PORT}"
 export DBQNA_LLM_MODEL_ID="mistralai/Mistral-7B-Instruct-v0.3"
@@ -14,3 +19,4 @@ export POSTGRES_PASSWORD="testpwd"
 export POSTGRES_DB="chinook"
 export DBQNA_TEXT_TO_SQL_PORT=9090
 export DBQNA_UI_PORT=5174
+export build_texttosql_url="${ip_address}:${DBQNA_TEXT_TO_SQL_PORT}/v1"

DBQnA/docker_compose/intel/cpu/xeon/set_env.sh

@@ -2,26 +2,19 @@
 
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
+
 pushd "../../../../../" > /dev/null
 source .set_env.sh
 popd > /dev/null
 
-#export host_ip=$(hostname -I | awk '{print $1}')
-
-if [ -z "${HUGGINGFACEHUB_API_TOKEN}" ]; then
-    echo "Error: HUGGINGFACEHUB_API_TOKEN is not set. Please set HUGGINGFACEHUB_API_TOKEN."
-fi
-
-if [ -z "${host_ip}" ]; then
-    echo "Error: host_ip is not set. Please set host_ip first."
-fi
+export host_ip=${ip_address}
 export no_proxy=$no_proxy,$host_ip,dbqna-xeon-react-ui-server,text2sql-service,tgi-service,postgres-container
 export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-export TGI_PORT=8008
-export TGI_LLM_ENDPOINT="http://${host_ip}:${TGI_PORT}"
-export LLM_MODEL_ID="mistralai/Mistral-7B-Instruct-v0.3"
+export HF_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
 export POSTGRES_USER=postgres
 export POSTGRES_PASSWORD=testpwd
 export POSTGRES_DB=chinook
+export TGI_PORT=8008
 export TEXT2SQL_PORT=9090
-"set_env.sh" 27L, 974B
+export TGI_LLM_ENDPOINT="http://${host_ip}:${TGI_PORT}"
+export LLM_MODEL_ID="mistralai/Mistral-7B-Instruct-v0.3"